diff --git a/bazaar/core/management/commands/auto_update.py b/bazaar/core/management/commands/auto_update.py new file mode 100644 index 00000000..e69de29b diff --git a/bazaar/core/migrations/0007_bookmark.py b/bazaar/core/migrations/0007_bookmark.py new file mode 100644 index 00000000..b6099b8a --- /dev/null +++ b/bazaar/core/migrations/0007_bookmark.py @@ -0,0 +1,39 @@ +# Generated by Django 6.0.8 on 2026-10-05 13:51 + +import django.db.models.deletion +import uuid +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0006_alter_yara_content_alter_yara_title"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="Bookmark", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("sample", models.CharField(max_length=256)), + ( + "owner", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + ), + ] diff --git a/bazaar/core/migrations/0008_comment.py b/bazaar/core/migrations/0008_comment.py new file mode 100644 index 00000000..7ed2fdb1 --- /dev/null +++ b/bazaar/core/migrations/0008_comment.py @@ -0,0 +1,40 @@ +# Generated by Django 6.0.8 on 2026-10-06 12:59 + +import django.db.models.deletion +import uuid +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0007_bookmark"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="Comment", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("sample", models.CharField(max_length=256)), + ("content", models.CharField(max_length=4096)), + ( + "owner", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + ), + ] diff --git a/bazaar/core/migrations/0009_comment_unique_comment_per_sample_per_owner.py b/bazaar/core/migrations/0009_comment_unique_comment_per_sample_per_owner.py new file mode 100644 index 00000000..2f206909 --- /dev/null +++ b/bazaar/core/migrations/0009_comment_unique_comment_per_sample_per_owner.py @@ -0,0 +1,21 @@ +# Generated by Django 6.0.8 on 2026-10-06 17:36 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0008_comment"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddConstraint( + model_name="comment", + constraint=models.UniqueConstraint( + fields=("sample", "owner"), name="unique_comment_per_sample_per_owner" + ), + ), + ] diff --git a/bazaar/core/migrations/0010_notification.py b/bazaar/core/migrations/0010_notification.py new file mode 100644 index 00000000..0de07331 --- /dev/null +++ b/bazaar/core/migrations/0010_notification.py @@ -0,0 +1,40 @@ +# Generated by Django 6.0.8 on 2026-10-07 08:15 + +import django.db.models.deletion +import uuid +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0009_comment_unique_comment_per_sample_per_owner"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="Notification", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("text_content", models.CharField(max_length=128)), + ("redirect_link", models.CharField(max_length=128)), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + ), + ] diff --git a/bazaar/core/models.py b/bazaar/core/models.py index 1473773b..43624bbf 100644 --- a/bazaar/core/models.py +++ b/bazaar/core/models.py @@ -1,12 +1,45 @@ +import uuid + import yara from django.conf import settings from django.core.exceptions import ValidationError from django.db import models -import uuid - from elasticsearch import Elasticsearch +class Notification(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + text_content = models.CharField(max_length=128) + redirect_link = models.CharField(max_length=128) + + +class Comment(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + sample = models.CharField(max_length=256) + owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + content = models.CharField(max_length=4096) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["sample", "owner"], + name="unique_comment_per_sample_per_owner", + ), + ] + + +class Bookmark(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + sample = models.CharField(max_length=256) + owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + + @staticmethod + def get_es_index(user): + es_index = f'bookmark_user_{user.id}' + return es_index + + def check_yara_rule(rule): try: yara.compile(source=rule) diff --git a/bazaar/core/modules/pithus.py b/bazaar/core/modules/pithus.py index 28a138f8..2c61b7d0 100644 --- a/bazaar/core/modules/pithus.py +++ b/bazaar/core/modules/pithus.py @@ -19,13 +19,15 @@ from django.conf import settings from django.core.files.storage import default_storage from django.utils import timezone +from django.contrib.auth import get_user_model + from elasticsearch import Elasticsearch from elasticsearch.helpers.actions import scan from bazaar.core.fingerprinting import ApplicationSignature from bazaar.core.utils import strings_from_apk -from bazaar.core.models import Yara +from bazaar.core.models import Yara, Notification es = Elasticsearch( @@ -355,7 +357,12 @@ def yara_analysis(sha256, rule_id=-1): return {'status': 'success', 'info': ''} -def retrohunt(rule_id): +def retrohunt(rule_id, user_id): + + user_model = get_user_model() + if user_id is not None: + user = user_model.objects.get(pk=user_id) + rule = None try: rule = Yara.objects.get(id=rule_id) @@ -367,7 +374,13 @@ def retrohunt(rule_id): index=settings.ELASTICSEARCH_APK_INDEX, ): _id = report.get('_source').get('sha256') - execute_single_yara_rule(rule.id, _id) + res = execute_single_yara_rule(rule.id, _id) + if user_id is not None and user is not None and user.is_authenticated and res['status'] == 'success': + Notification.objects.create( + user=user, + text_content=f"Retrohunt found sample matching [{rule.title}]", + redirect_link="/workspace" + ) del rule gc.collect() diff --git a/bazaar/core/services/__init__.py b/bazaar/core/services/__init__.py index d41d6548..b8084415 100644 --- a/bazaar/core/services/__init__.py +++ b/bazaar/core/services/__init__.py @@ -3,3 +3,4 @@ from bazaar.core.services import search as SearchService # noqa: F401 from bazaar.core.services import rules as RulesService # noqa: F401 from bazaar.core.services import genom as GenomService # noqa: F401 +from bazaar.core.services import notifications as NotificationsService # noqa: F401 diff --git a/bazaar/core/services/notifications.py b/bazaar/core/services/notifications.py new file mode 100644 index 00000000..ad23f2cf --- /dev/null +++ b/bazaar/core/services/notifications.py @@ -0,0 +1,35 @@ + +from bazaar.core.models import Notification + + +def get_user_notifications(request): + if not request.user.is_authenticated: + return [] + + return Notification.objects.filter(user=request.user) + + +def add_user_notification(user, text_content, redirect_link=None): + if not user.is_authenticated: + return False + + Notification.objects.create(user=user, text_content=text_content, redirect_link=redirect_link) + return True + + +def delete_user_notification(request, notif_id): + if not request.user.is_authenticated: + return False + notifications = Notification.objects.filter(user=request.user, id=notif_id) + for notif in notifications: + notif.delete() + return True + + +def delete_all_user_notifications(request): + if not request.user.is_authenticated: + return False + notifications = Notification.objects.filter(user=request.user) + for notif in notifications: + notif.delete() + return True diff --git a/bazaar/core/utils.py b/bazaar/core/utils.py index 220e5e72..0c5794e3 100644 --- a/bazaar/core/utils.py +++ b/bazaar/core/utils.py @@ -460,6 +460,10 @@ def normalize(data): return x +def transform_results(results): + return [doc['_source'] for doc in results['hits']['hits']] + + def transform_hl_results(results): ret = [] for doc in results['hits']['hits']: diff --git a/bazaar/front/forms.py b/bazaar/front/forms.py index 9a5d8fbb..345c0901 100644 --- a/bazaar/front/forms.py +++ b/bazaar/front/forms.py @@ -7,6 +7,7 @@ from django.forms import ModelForm from bazaar.core.models import Yara +from bazaar.core.models import Comment class SimilaritySearchForm(forms.Form): @@ -51,3 +52,14 @@ class YaraCreateForm(ModelForm): class Meta: model = Yara fields = ['title', 'content', 'is_private'] + + +class CommentForm(forms.ModelForm): + class Meta: + model = Comment + fields = ["content"] + widgets = { + "content": forms.Textarea(attrs={ + "rows": 5, + }), + } diff --git a/bazaar/front/urls.py b/bazaar/front/urls.py index 28b02052..66d3939a 100644 --- a/bazaar/front/urls.py +++ b/bazaar/front/urls.py @@ -2,8 +2,10 @@ from django.views.generic.base import TemplateView from bazaar.front.view import HomeView, ReportView, basic_upload_view, similarity_search_view, export_report_view, \ - download_sample_view, my_rules_view, my_rule_edit_view, my_rule_create_view, my_rule_delete_view, og_card_view, \ - my_retrohunt_view, get_andgrocfg_code, get_genom, basic_url_download_view, report_status_view, instance_status + download_sample_view, my_rule_edit_view, my_rule_create_view, my_rule_delete_view, og_card_view, \ + my_retrohunt_view, get_andgrocfg_code, get_genom, basic_url_download_view, report_status_view, instance_status, \ + remove_bookmark_sample_view, add_bookmark_sample_view, workspace_view, add_comment_view, delete_comment_view, \ + notifications_view, notification_delete_view app_name = "front" urlpatterns = [ @@ -15,16 +17,23 @@ path("report//card", view=og_card_view, name="og_card"), path("report//status", view=report_status_view, name="report_status"), path("apk/", view=basic_upload_view, name="basic_upload"), - path("url/", view=basic_url_download_view, name="basic_url_download"), path("apk/", view=download_sample_view, name="download_sample"), - path("similar/", view=similarity_search_view, name="similarity_search"), - path("similar/", view=similarity_search_view, name="similarity_search"), - path("rules/", view=my_rules_view, name="my_rules"), - path("rules/new", view=my_rule_create_view, name="my_rule_create"), - path("rules//edit", view=my_rule_edit_view, name="my_rule_edit"), + path("apk//bookmark/add", view=add_bookmark_sample_view, name="add_bookmark_sample"), + path("apk//bookmark/remove", view=remove_bookmark_sample_view, + name="remove_bookmark_sample"), path("rules//delete", view=my_rule_delete_view, name="my_rule_delete"), + path("rules//edit", view=my_rule_edit_view, name="my_rule_edit"), path("rules//retro", view=my_retrohunt_view, name="my_rule_retro"), + path("rules/new", view=my_rule_create_view, name="my_rule_create"), + path("similar/", view=similarity_search_view, name="similarity_search"), + path("similar/", view=similarity_search_view, name="similarity_search"), path("androcfg//", view=get_andgrocfg_code, name="get_andgrocfg_code"), path("androcfg/all", view=get_genom, name="get_genom"), - path("status", view=instance_status, name="instance_status") + path("status", view=instance_status, name="instance_status"), + path("url/", view=basic_url_download_view, name="basic_url_download"), + path("workspace/", view=workspace_view, name="workspace"), + path("comment//add", view=add_comment_view, name="add_comment"), + path("comment//delete", view=delete_comment_view, name="delete_comment"), + path("notifications", view=notifications_view, name="notifications_url"), + path("notifications//delete", view=notification_delete_view, name="notifications_url") ] diff --git a/bazaar/front/view.py b/bazaar/front/view.py index c9c7d16d..185b6a77 100644 --- a/bazaar/front/view.py +++ b/bazaar/front/view.py @@ -1,40 +1,43 @@ import logging +from collections import Counter from tempfile import NamedTemporaryFile +from androcfg.code_style import U39bStyle from django.conf import settings from django.contrib import messages from django.core.cache import cache from django.core.files.storage import default_storage -from django.http import JsonResponse, HttpResponse +from django.http import HttpResponse, JsonResponse, HttpResponseNotAllowed from django.http.response import HttpResponseBadRequest -from django.shortcuts import render, redirect +from django.shortcuts import redirect, render from django.utils import timezone from django.utils.decorators import method_decorator from django.views.decorators.cache import cache_page from django.views.decorators.csrf import csrf_exempt from django.views.generic import View from django_q.tasks import async_task - -from rest_framework.authtoken.models import Token -from rest_framework.reverse import reverse_lazy +from elasticsearch import Elasticsearch from pygments import highlight from pygments.formatters import HtmlFormatter from pygments.lexers.jvm import JavaLexer -from androcfg.code_style import U39bStyle +from rest_framework.authtoken.models import Token +from rest_framework.reverse import reverse_lazy from bazaar.core.services import ReportService from bazaar.core.services import ApkService from bazaar.core.services import SearchService from bazaar.core.services import RulesService from bazaar.core.services import GenomService +from bazaar.core.services import NotificationsService -from bazaar.core.models import Yara +from bazaar.core.models import Yara, Bookmark, Comment, Notification from bazaar.core.modules.pithus import retrohunt from bazaar.core.utils import get_matching_items_by_dexofuzzy -from bazaar.front.forms import SearchForm, BasicUploadForm, SimilaritySearchForm, BasicUrlDownloadForm +from bazaar.front.forms import SearchForm, BasicUploadForm, SimilaritySearchForm, BasicUrlDownloadForm, CommentForm from bazaar.front.og import generate_og_card from bazaar.front.utils import get_similarity_matrix, generate_world_map, \ get_sample_timeline, get_andro_cfg_storage_path +from bazaar.core.utils import transform_hl_results, transform_results from .forms import YaraCreateForm @@ -53,6 +56,7 @@ def get(self, request, *args, **kwargs): f = SearchForm(request.GET) form_to_show = f + if not request.GET: form_to_show = SearchForm() if f.is_valid(): @@ -72,7 +76,7 @@ def get(self, request, *args, **kwargs): 'list_results': list_results, 'report_example': report_example, 'q': q, 'matrix': matrix, - 'max_size': settings.MAX_APK_UPLOAD_SIZE + 'max_size': settings.MAX_APK_UPLOAD_SIZE, }) @@ -104,7 +108,6 @@ def get(self, request, *args, **kwargs): # Find similar sample based on dexofuzzy similar_samples = None - similar_samples_extended = None try: dexofuzzy_hash = result['dexofuzzy']['apk'] if dexofuzzy_hash: @@ -125,8 +128,6 @@ def get(self, request, *args, **kwargs): vt = None res.append((apk[0]['source']['app_name'], apk[0]['source']['handle'], sha256, vt, score)) - similar_samples_extended = res - # Find public hunting results hunting_matches = Yara.find_public_hunting_matches(sha) @@ -138,6 +139,12 @@ def get(self, request, *args, **kwargs): # Get timeline timeline = get_sample_timeline(sha) + # Check if the sample is already bookmarked + if request.user.is_authenticated: + is_bookmarked = get_user_bookmark_by_hash(request, sha) + else: + is_bookmarked = False + return render(request, 'front/report.html', { 'result': result, 'status': status, @@ -145,8 +152,9 @@ def get(self, request, *args, **kwargs): 'timeline': timeline, 'cache_key': f'{cache_key}_tpl', 'hunting_matches': hunting_matches, - 'similar_samples': similar_samples_extended, - 'cache_retention_time': cache_retention_time}) + 'similar_samples': similar_samples, + 'cache_retention_time': cache_retention_time, + 'is_bookmarked': is_bookmarked}) except Exception as e: logging.exception(e) return redirect(reverse_lazy('front:home')) @@ -186,6 +194,11 @@ def basic_upload_view(request): apk = request.FILES['apk'] try: sha256 = ApkService.upload_apk(apk) + if request.user.is_authenticated: + # Todo: make bookmarking a user setting + Bookmark.objects.create(sample=sha256, owner=request.user).save() + except Exception as e: + logging.exception(e) except Exception as e: messages.warning(request, e) return redirect(reverse_lazy('front:home')) @@ -253,18 +266,202 @@ def og_card_view(request, sha256): return HttpResponse(fp.read(), content_type="image/png") -def my_rules_view(request): +def notifications_view(request): if not request.user.is_authenticated: return redirect(reverse_lazy('front:home')) - my_rules = None + notifications = NotificationsService.get_user_notifications(request) + return render(request, 'front/workspace/notifications.html', + context={'notifications': notifications}) + + +def notification_delete_view(request, notif_id): + if not request.user.is_authenticated: + return redirect(reverse_lazy('front:home')) + + if request.method != "POST": + return HttpResponseNotAllowed(["POST"]) + + notifs = Notification.objects.filter( + id=notif_id, + user=request.user, + ) + if notifs is not None: + for notif in notifs: + notif.delete() + + return JsonResponse({ + "success": True, + "message": "Comment deleted.", + }) + + +def get_user_bookmarks(request): + return Bookmark.objects.filter(owner=request.user) + + +def enrich_bookmarks(request, bookmarks): + res = [] + for sample in bookmarks: + apk = get_sample_light(sample.sample) + apk[0]["comment"] = Comment.objects.filter(owner=request.user, sample=sample.sample).first() + res.append(apk) + return res + + +def get_user_bookmark_by_hash(request, sha): + bookmark = Bookmark.objects.filter(owner=request.user, sample=sha) + return bookmark + + +def add_bookmark_sample_view(request, sha256): + if not request.user.is_authenticated: + return redirect(reverse_lazy('front:home')) + + if request.method == 'GET': + user_bookmarks = get_user_bookmarks(request) + if not user_bookmarks.filter(sample=sha256): + new_bookmark = Bookmark.objects.create(sample=sha256, owner=request.user) + try: + new_bookmark.save() + user_bookmarks = get_user_bookmarks(request) + except Exception as e: + logging.exception(e) + + return redirect(reverse_lazy('front:report', [sha256])) + + +def remove_bookmark_sample_view(request, sha256): + if not request.user.is_authenticated: + return redirect(reverse_lazy('front:home')) + + if request.method == 'GET': + bookmark = get_user_bookmark_by_hash(request, sha256) + try: + bookmark.delete() + except Exception as e: + logging.exception(e) + + return redirect(reverse_lazy('front:report', [sha256])) + + +def get_last_samples(): + query = { + "query": { + "query_string": { + "default_field": "sha256", + "query": "*" + } + }, + "highlight": { + "fields": { + "*": {"pre_tags": [""], "post_tags": [""]} + } + }, + "aggs": { + "permissions": { + "terms": {"field": "permissions.keyword"} + }, + "domains": { + "terms": {"field": "domains_analysis._name.keyword"} + }, + "android_features": { + "terms": {"field": "features.keyword"} + } + }, + "sort": {"analysis_date": "desc"}, + "_source": ["apk_hash", "sha256", "uploaded_at", "icon_base64", "handle", "app_name", + "version_code", "size", "dexofuzzy.apk", "quark.threat_level", "vt", "vt_report", "malware_bazaar", + "is_signed", "frosting_data.is_frosted", "features", "andro_cfg.genom"], + "size": 5, + } + es = Elasticsearch(settings.ELASTICSEARCH_HOSTS) + try: + raw_results = es.search(index=settings.ELASTICSEARCH_APK_INDEX, body=query) + results = transform_results(raw_results) + return results + except Exception: + return [] + + +def workspace_view(request): + if not request.user.is_authenticated: + return redirect(reverse_lazy('front:home')) + + my_bookmarks = None + if request.method == 'GET': + # TODO: refactor so there is only one call for all the rules, + # then filter by user and by score and output those in the tables + apk_count = get_count_apks() + malicious = get_malicious_samples_count() + count = { + "apks": apk_count, + "malicious": malicious + } + my_rules = get_rules(request) + my_bookmarks = enrich_bookmarks(request, get_user_bookmarks(request)) + last_samples = get_last_samples() + highest_matching_rules = get_best_rules() + trending_malwares = get_trending_malware() + trends = { + 'samples': last_samples, 'rules': highest_matching_rules, 'trending_malwares': trending_malwares + } + owner = request.user if request.method == 'GET': my_rules = RulesService.get_rules(owner) token, _ = Token.objects.get_or_create(user=owner) + return render(request, 'front/workspace/workspace_base.html', + context={'my_rules': my_rules, 'my_token': token, + 'bookmarked_samples': my_bookmarks, 'trends': trends, + 'count': count}) + + +def add_comment_view(request, sha256): + if not request.user.is_authenticated: + return redirect(reverse_lazy('front:home')) + + if request.method != "POST": + return HttpResponseNotAllowed(["POST"]) + form = CommentForm(request.POST) + + comments = Comment.objects.filter( + sample=sha256, + owner=request.user, + ) + if comments is not None: + for comment in comments: + comment.delete() + + if form.is_valid(): + comment = form.save(commit=False) + comment.sample = sha256 + comment.owner = request.user + comment.save() + + return HttpResponse(status=200) + - return render(request, 'front/yara_rules/my_rules.html', context={'my_rules': my_rules, 'my_token': token.key}) +def delete_comment_view(request, comment_id): + if not request.user.is_authenticated: + return redirect(reverse_lazy('front:home')) + + if request.method != "POST": + return HttpResponseNotAllowed(["POST"]) + + comments = Comment.objects.filter( + id=comment_id, + owner=request.user, + ) + if comments is not None: + for comment in comments: + comment.delete() + + return JsonResponse({ + "success": True, + "message": "Comment deleted.", + }) def my_rule_create_view(request): @@ -282,10 +479,10 @@ def my_rule_create_view(request): new_rule.save() messages.success(request, 'Your rule has been created!') except Exception: - return render(request, 'front/yara_rules/my_rule_edit.html', {'form': new_rule}) - return redirect(reverse_lazy('front:my_rules')) + return render(request, 'front/workspace/my_rule_edit.html', {'form': new_rule}) + return redirect(reverse_lazy('front:workspace')) - return render(request, 'front/yara_rules/my_rule_edit.html', {'form': new_rule}) + return render(request, 'front/workspace/my_rule_edit.html', {'form': new_rule}) def my_rule_edit_view(request, uuid): @@ -295,7 +492,7 @@ def my_rule_edit_view(request, uuid): if request.method == 'GET': rule = Yara.objects.get(id=uuid) new_rule = YaraCreateForm(instance=rule) - return render(request, 'front/yara_rules/my_rule_edit.html', {'form': new_rule, 'edit': True}) + return render(request, 'front/workspace/my_rule_edit.html', {'form': new_rule, 'edit': True}) elif request.method == 'POST': rule = Yara.objects.get(id=uuid) @@ -308,8 +505,8 @@ def my_rule_edit_view(request, uuid): RulesService.delete_es_matches(request.user, rule) messages.success(request, 'Your rule has been updated!') except Exception: - return render(request, 'front/yara_rules/my_rule_edit.html', {'form': new_rule}) - return redirect(reverse_lazy('front:my_rules')) + return render(request, 'front/workspace/my_rule_edit.html', {'form': new_rule}) + return redirect(reverse_lazy('front:workspace')) else: return HttpResponseBadRequest() @@ -324,10 +521,272 @@ def my_rule_delete_view(request, uuid=None): RulesService.delete_es_matches(request.user, rule) rule.delete() messages.success(request, 'Your rule has been deleted.') - return redirect(reverse_lazy('front:my_rules')) - except Exception: + return redirect(reverse_lazy('front:workspace')) + except Exception as e: messages.warning(request, 'An error occured while deleting your rule.') - return redirect(reverse_lazy('front:my_rules')) + logging.exception(e) + return redirect(reverse_lazy('front:workspace')) + + +def delete_es_matches(request, rule): + es = Elasticsearch(settings.ELASTICSEARCH_HOSTS) + public_es_index, private_es_index = Yara.get_es_index_names(request.user) + q = {'query': { + 'match': { + 'rule': rule.id, + } + }} + if rule.is_private: + try: + es.delete_by_query(index=private_es_index, body=q) + except Exception as e: + logging.exception(e) + elif not rule.is_private: + try: + es.delete_by_query(index=public_es_index, body=q) + except Exception as e: + logging.exception(e) + else: + pass + + return + + +def get_count_apks(): + query = { + "aggs": { + "type_value": { + "value_count": { + "field": "apk_hash.keyword" + } + } + } + } + es = Elasticsearch(settings.ELASTICSEARCH_HOSTS) + + try: + count = es.search(index=settings.ELASTICSEARCH_APK_INDEX, body=query)['aggregations']['type_value']['value'] + return count + except Exception: + return -1 + + +def get_trending_malware(): + query = { + "query": { + "bool": { + "must": [], + "filter": [ + { + "bool": { + "should": [ + { + "range": { + "vt_report.attributes.popular_threat_classification.popular_threat_name.count": { # noqa + "gt": 1 + } + } + } + ], + "minimum_should_match": 1 + } + } + ], + "should": [], + "must_not": [] + } + }, + "size": 5000, + "_source": "vt_report.attributes.popular_threat_classification.popular_threat_name.value", + "sort": [ + { + "vt_report.attributes.popular_threat_classification.popular_threat_name.count": { + "order": "desc" + } + } + ] + } + es = Elasticsearch(settings.ELASTICSEARCH_HOSTS) + try: + results = es.search(index=settings.ELASTICSEARCH_APK_INDEX, body=query)['hits']['hits'] + return Counter( + value['_source']['vt_report']['attributes'] + ['popular_threat_classification']['popular_threat_name'][0]['value'] for value in results + ).most_common(5) + + except Exception: + return [] + + +def get_malicious_samples_count(): + query = { + "query": { + "bool": { + "must": [], + "filter": [ + { + "bool": { + "should": [ + { + "range": { + "vt.malicious": { + "gt": 1 + } + } + } + ], + "minimum_should_match": 1 + } + } + ], + "should": [], + "must_not": [] + } + }, + "size": 5000, + "_source": "vt", + "sort": [ + { + "vt.malicious": { + "order": "desc" + } + } + ] + } + es = Elasticsearch(settings.ELASTICSEARCH_HOSTS) + try: + results = es.search(index=settings.ELASTICSEARCH_APK_INDEX, body=query)['hits']['hits'] + res = Counter(value['_source']['vt']['malicious'] for value in results) + # FIXME: total() doesn't seem to work, idk why + return sum(res.values()) + except Exception: + return -1 + + +def get_best_rules(): + es = Elasticsearch(settings.ELASTICSEARCH_HOSTS) + yara_rules = Yara.objects.all() + my_rules = [] + public_es_index, _ = Yara.get_es_index_names() + # TODO: note fo later: in an ideal world, it would be nice to have the matches.matching_files + # be set on keyword=true for Elastic search so we can retreive the result sorted + # instead of sorting them on our own. + # See: https://stackoverflow.com/questions/63784500/text-fields-are-not-optimised-for-operations-that-require-per-document-elastic # noqa + q = { + "query": { + "bool": { + "must": [], + "filter": [ + { + "bool": { + "should": [ + { + "range": { + "matches.matching_files": { + "gt": 1 + } + } + } + ], + "minimum_should_match": 1 + } + } + ], + "should": [], + "must_not": [] + } + }, + "size": 5000, + } + public_matches = None + try: + public_matches = es.search(index=public_es_index, body=q)['hits']['hits'] + for rule in yara_rules: + my_rule = { + 'rule': rule, + 'matching_date': '', + 'matches': [], + } + if public_matches: + for match in public_matches: + if match['_source']['rule'] == str(rule.id): + m = match['_source'] + m['sample'] = get_sample_light(match['_source']['matches']['apk_id']) + my_rule['matches'].append(m) + if len(my_rule['matches']) > 1: + my_rules.append(my_rule) + + return my_rules + except Exception: + return [] + + +def get_rules(request): + es = Elasticsearch(settings.ELASTICSEARCH_HOSTS) + yara_rules = Yara.objects.filter(owner=request.user) + public_es_index, private_es_index = Yara.get_es_index_names(request.user) + q = { + 'query': { + 'terms': { + 'owner': [request.user.id] + } + }, + 'size': 5000, + } + public_matches, private_matches = None, None + try: + private_matches = es.search(index=private_es_index, body=q)['hits']['hits'] + except Exception: + pass + + try: + public_matches = es.search(index=public_es_index, body=q)['hits']['hits'] + except Exception: + pass + + my_rules = [] + for rule in yara_rules: + my_rule = { + 'rule': rule, + 'matching_date': '', + 'matches': [], + } + if rule.is_private and private_matches: + for match in private_matches: + if match['_source']['rule'] == str(rule.id): + m = match['_source'] + m['sample'] = get_sample_light(match['_source']['matches']['apk_id']) + my_rule['matches'].append(m) + elif not rule.is_private and public_matches: + for match in public_matches: + if match['_source']['rule'] == str(rule.id): + m = match['_source'] + m['sample'] = get_sample_light(match['_source']['matches']['apk_id']) + my_rule['matches'].append(m) + my_rules.append(my_rule) + + return my_rules + + +def get_sample_light(sha256): + query = { + "query": { + "match": { + "apk_hash": sha256 + } + }, + "_source": ["apk_hash", "sha256", "uploaded_at", "icon_base64", "handle", "app_name", + "version_code", "size", "dexofuzzy.apk", "quark.threat_level", "vt", "malware_bazaar", + "is_signed", "frosting_data.is_frosted", "features"], + "size": 1, + } + es = Elasticsearch(settings.ELASTICSEARCH_HOSTS) + try: + results = es.search(index=settings.ELASTICSEARCH_APK_INDEX, body=query) + results = transform_hl_results(results) + return results + except Exception: + return [] def my_retrohunt_view(request, uuid): @@ -335,12 +794,13 @@ def my_retrohunt_view(request, uuid): return redirect(reverse_lazy('front:home')) # TODO: add a cap on user use try: - async_task(retrohunt, request) + async_task(retrohunt, uuid, request.user.pk) messages.success(request, 'The retrohunt has been launched.') - except Exception: + except Exception as e: + logging.exception(e) messages.warning(request, 'An error occured launching retrohunt.') - return redirect(reverse_lazy('front:my_rules')) + return redirect(reverse_lazy('front:workspace')) def get_andgrocfg_code(request, sha256, foo): diff --git a/bazaar/static/css/project.css b/bazaar/static/css/project.css index bec5d4f5..235a931c 100644 --- a/bazaar/static/css/project.css +++ b/bazaar/static/css/project.css @@ -7316,3 +7316,17 @@ samp { .inline-doc { color: #868e96; } + +.notification-bell { + position: relative; + display: inline-block; } + +.notification-dot { + position: absolute; + top: 5px; + right: 5px; + width: 8px; + height: 8px; + background-color: #dc3545; + border-radius: 50%; + border: 2px solid #fff; } diff --git a/bazaar/static/sass/project.scss b/bazaar/static/sass/project.scss index e8712b71..97a7de9c 100644 --- a/bazaar/static/sass/project.scss +++ b/bazaar/static/sass/project.scss @@ -83,3 +83,20 @@ samp { color: $gray-600; } + + +.notification-bell { + position: relative; + display: inline-block; +} + +.notification-dot { + position: absolute; + top: 5px; + right: 5px; + width: 8px; + height: 8px; + background-color: #dc3545; + border-radius: 50%; + border: 2px solid #fff; +} diff --git a/bazaar/templates/base.html b/bazaar/templates/base.html index bfbac9d6..4b32299e 100644 --- a/bazaar/templates/base.html +++ b/bazaar/templates/base.html @@ -43,9 +43,11 @@ {% url 'front:home' as home_url %} {% url 'about' as about_url %} {% url 'front:instance_status' as instance_status %} -{% url 'front:my_rules' as my_rules_url %} +{% url 'front:workspace' as workspace_url %} {% url 'account_logout' as account_logout_url %} {% url 'account_login' as account_login_url %} +{% url 'front:notifications_url' as notifications_url %} +
@@ -69,29 +71,44 @@ Status {% if request.user.is_authenticated %} - -
  • - - {% trans "My hunting" %} +
  • + + {% trans "Workspace" %}
    • - {% endif %} - + +
    -
    +
    {% if messages %} {% for message in messages %}
    {{ message }} @@ -137,4 +154,4 @@ {% endblock inline_js %} - + \ No newline at end of file diff --git a/bazaar/templates/front/index.html b/bazaar/templates/front/index.html index 28e3cbfb..11a530fa 100644 --- a/bazaar/templates/front/index.html +++ b/bazaar/templates/front/index.html @@ -169,4 +169,3 @@

    No results

    {% endif %}
    {% endblock whole_content %} - diff --git a/bazaar/templates/front/report.html b/bazaar/templates/front/report.html index b2dd5e3c..6f992b2d 100644 --- a/bazaar/templates/front/report.html +++ b/bazaar/templates/front/report.html @@ -39,6 +39,15 @@ + {% if is_bookmarked %} + + + {% else %} + + + {% endif %}
    {% endif %}
    diff --git a/bazaar/templates/front/yara_rules/my_rule_edit.html b/bazaar/templates/front/workspace/my_rule_edit.html similarity index 100% rename from bazaar/templates/front/yara_rules/my_rule_edit.html rename to bazaar/templates/front/workspace/my_rule_edit.html diff --git a/bazaar/templates/front/workspace/notifications.html b/bazaar/templates/front/workspace/notifications.html new file mode 100644 index 00000000..885b5513 --- /dev/null +++ b/bazaar/templates/front/workspace/notifications.html @@ -0,0 +1,45 @@ +{% extends "base.html" %} +{% load static %} +{% load cache %} + +{% block content %} +
    +
    +

    Notifications:

    + + {% for notif in notifications %} + + + {% empty %} + + + {% endfor %} +
    + {{ notif.text_content }} + + Nothing to display
    +
    +
    + + +{% endblock %} \ No newline at end of file diff --git a/bazaar/templates/front/workspace/workspace_base.html b/bazaar/templates/front/workspace/workspace_base.html new file mode 100644 index 00000000..c5b79194 --- /dev/null +++ b/bazaar/templates/front/workspace/workspace_base.html @@ -0,0 +1,367 @@ +{% extends "base.html" %} +{% load static %} +{% load cache %} + +{% block content %} +
    + + +
    +
    +

    Dashboard

    +
    +

    Total of samples on this instance

    +

    {{count.apks}} samples

    +

    {{count.malicious}} malicious samples

    +
    +
    + + +
    +

    Bookmarks

    + + {% for results in bookmarked_samples %} + {% for result in results %} + + + + + + + {% endfor %} + {% endfor %} +
    {% include "front/m_result_flat.html" with result=result %} + +
    +     +
    + +
    + +
    +
    +

    Yara hunting + + +
    + New rule +     +

    +
    +
    + {% for rule in my_rules %} +
    +
    +
    +
    + + {{ rule.rule.title }} +
    +
    + + + + + + + + + +
    +
    + + +
    + +
    +
    + {% if rule.matches|length == 0 %} +

    No matches

    + {% endif %} + + {% for m in rule.matches %} +
    +
    +
    +
    + Hunting information: +
    +
    + {% if m.matches.inner_rules %} + + {% endif %} + {% for ir in m.matches.inner_rules %} + {{ ir }} + {% endfor %} +
    + {% if m.matches.matching_files %} +
    matching files:
    + {% endif %} + {% for f in m.matches.matching_files %} + {{ f }}
    + {% endfor %} +
    +
    +
    +
    + {% for result in m.sample %} + {% include "front/m_result_flat.html" %} + {% endfor %} +
    +
    + {% endfor %} +
    +
    +
    + {% endfor %} +
    +
    + + + + +
    +
    +
    + Your API key: {{ my_token }} +
    +
    +
    +
    + +
    + + + + + + + + +{% endblock %} \ No newline at end of file diff --git a/bazaar/templates/front/yara_rules/my_rules.html b/bazaar/templates/front/yara_rules/my_rules.html deleted file mode 100644 index cdf58d1b..00000000 --- a/bazaar/templates/front/yara_rules/my_rules.html +++ /dev/null @@ -1,132 +0,0 @@ -{% extends "base.html" %} -{% load static %} -{% load cache %} - -{% block content %} -
    -
    -
    -
    - Your API key: {{ my_token }} -
    -
    -
    -
    -
    -

    My hunting

    -
    -
    - - -
    - New rule -     -
    -
    -
    -
    -
    -
    -
    - {% for rule in my_rules %} -
    -
    -
    -
    - - {{ rule.rule.title }} -
    -
    - - - - - - - - - -
    -
    - - -
    - -
    -
    - {% if rule.matches|length == 0 %} -

    No matches

    - {% endif %} - - {% for m in rule.matches %} -
    -
    -
    -
    - Hunting information: -
    -
    - {% if m.matches.inner_rules %} - - {% endif %} - {% for ir in m.matches.inner_rules %} - {{ ir }} - {% endfor %} -
    - {% if m.matches.matching_files %} -
    matching files:
    - {% endif %} - {% for f in m.matches.matching_files %} - {{ f }}
    - {% endfor %} -
    -
    -
    -
    - {% for result in m.sample %} - {% include "front/m_result_flat.html" %} - {% endfor %} -
    -
    - {% endfor %} -
    -
    -
    - {% endfor %} -
    -
    -
    -
    - - - -{% endblock %} diff --git a/bazaar/users/migrations/0002_alter_user_first_name_alter_user_id.py b/bazaar/users/migrations/0002_alter_user_first_name_alter_user_id.py index 6a76e9fc..abd7ef81 100644 --- a/bazaar/users/migrations/0002_alter_user_first_name_alter_user_id.py +++ b/bazaar/users/migrations/0002_alter_user_first_name_alter_user_id.py @@ -1,4 +1,4 @@ -# Generated by Django 6.0.1 on 2026-02-17 10:46 +# Generated by Django 6.0.8 on 2026-10-05 13:59 from django.db import migrations, models diff --git a/bazaar/utils/context_processors.py b/bazaar/utils/context_processors.py index 3c535141..5503acc8 100644 --- a/bazaar/utils/context_processors.py +++ b/bazaar/utils/context_processors.py @@ -1,4 +1,5 @@ from django.conf import settings +from bazaar.core.models import Notification def settings_context(_request): @@ -6,3 +7,14 @@ def settings_context(_request): # Note: we intentionally do NOT expose the entire settings # to prevent accidental leaking of sensitive information return {"DEBUG": settings.DEBUG} + + +def notifications(request): + if not request.user.is_authenticated: + return {"has_notifications": False} + + return { + "has_notifications": Notification.objects.filter( + user=request.user, + ).exists() + } diff --git a/config/settings/base.py b/config/settings/base.py index 1683483f..02875cc1 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -211,6 +211,7 @@ "django.template.context_processors.tz", "django.contrib.messages.context_processors.messages", "bazaar.utils.context_processors.settings_context", + "bazaar.utils.context_processors.notifications", ], }, } diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 54cc4571..e2683455 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -19,7 +19,7 @@ download_sample_view, export_report_view, og_card_view, - my_rules_view, + workspace_view, my_rule_create_view, my_rule_edit_view, my_rule_delete_view, @@ -64,7 +64,7 @@ similarity_search_view, f"/similar/{sha256}", [], {"sha256": sha256}, marks=pytest.mark.skip(reason="No auth required") ), - (my_rules_view, "/rules/", [], {}), + (workspace_view, "/workspace/", [], {}), (my_rule_create_view, "/rules/new", [], {}), (my_rule_edit_view, f"/rules/{uuid}/edit", [uuid], {}), (my_rule_delete_view, f"/rules/{uuid}/delete", [], {"uuid": uuid}), diff --git a/tests/test_urls.py b/tests/test_urls.py index 86db641b..2c582167 100644 --- a/tests/test_urls.py +++ b/tests/test_urls.py @@ -62,8 +62,8 @@ def test_similarity_search_sha256(): def test_my_rules(): - assert reverse("front:my_rules") == "/rules/" - assert resolve("/rules/").view_name == "front:my_rules" + assert reverse("front:workspace") == "/workspace/" + assert resolve("/workspace/").view_name == "front:workspace" def test_my_rule_create(): diff --git a/tests/test_views.py b/tests/test_views.py index 6aea1731..085487dd 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -22,11 +22,11 @@ download_sample_view, export_report_view, og_card_view, - my_rules_view, my_rule_create_view, my_rule_edit_view, my_rule_delete_view, my_retrohunt_view, + workspace_view, ) @@ -212,21 +212,21 @@ def test_og_card_view(mock_gen, report_data, rf: RequestFactory): # Test Rules -def test_my_rules_view_unauth(rf: RequestFactory): - request = rf.get("/rules/") +def test_workspaces_anon(rf: RequestFactory): + request = rf.get("/workspace/") request.user = AnonymousUser() - response = my_rules_view(request) + response = workspace_view(request) assert response.status_code == 302 assert response.url == "/" @pytest.mark.django_db -def test_my_rules_view(user: User, rf: RequestFactory): - request = rf.get("/rules/") +def test_workspaces(user: User, rf: RequestFactory): + request = rf.get("/workspace/") request.user = user - response = my_rules_view(request) + response = workspace_view(request) assert response.status_code == 200 @@ -250,7 +250,7 @@ def test_my_rule_create_view(yara_rule, user: User, rf: RequestFactory): response = my_rule_create_view(request) assert response.status_code == 302 - assert response.url == "/rules/" + assert response.url == "/workspace/" def test_my_rule_edit_view_unauth(rf: RequestFactory): @@ -304,7 +304,7 @@ def test_my_rule_edit_view_post(mock_es_delete, mock_yara_get, yara_rule, user: response = my_rule_edit_view(request, uuid) assert response.status_code == 302 - assert response.url == '/rules/' + assert response.url == '/workspace/' def test_my_rule_delete_view_unauth(rf: RequestFactory): @@ -341,7 +341,7 @@ def test_my_rule_delete_view(mock_es_delete, mock_yara_delete, mock_yara_get, us response = my_rule_delete_view(request, uuid) assert mock_yara_delete.call_count == 1 assert response.status_code == 302 - assert response.url == '/rules/' + assert response.url == '/workspace/' def test_my_retrohunt_view_unauth(rf: RequestFactory): @@ -369,4 +369,4 @@ def test_my_retrohunt_view(mock_async, mock_messages, user: User, rf: RequestFac response = my_retrohunt_view(request, uuid) assert mock_messages.call_count == 1 assert response.status_code == 302 - assert response.url == '/rules/' + assert response.url == '/workspace/'