From 37e97579926327102b9c16265af788a825d1f307 Mon Sep 17 00:00:00 2001 From: trpyn Date: Mon, 12 Oct 2026 18:24:44 +0200 Subject: [PATCH 1/2] fix analysis stuck --- bazaar/core/management/commands/remove_apk.py | 48 +++++++++++++++++++ bazaar/core/mobsf.py | 15 +++++- bazaar/core/modules/mobsf.py | 2 +- bazaar/core/modules/quarkengine.py | 3 +- bazaar/core/modules/similarity.py | 5 +- bazaar/core/tasks.py | 8 ++-- local.yml | 2 +- production.yml | 2 +- 8 files changed, 72 insertions(+), 13 deletions(-) create mode 100644 bazaar/core/management/commands/remove_apk.py diff --git a/bazaar/core/management/commands/remove_apk.py b/bazaar/core/management/commands/remove_apk.py new file mode 100644 index 0000000..d4dd6e5 --- /dev/null +++ b/bazaar/core/management/commands/remove_apk.py @@ -0,0 +1,48 @@ +from time import sleep +import logging +from elasticsearch import Elasticsearch + +from django.conf import settings +from django.core.management.base import BaseCommand +from django_q.tasks import async_task +from django.core.files.storage import default_storage + +from elasticsearch.helpers.actions import scan + +indexes = [ + 'apk_analysis', + 'apk_analysis_tmp', + 'analysis_tasks', + 'vt_reports', + 'google_play_details', + 'dexofuzzy_apk', + 'ssdeep_apk', + 'ssdeep_manifest', +] +es = Elasticsearch( + settings.ELASTICSEARCH_HOSTS, + basic_auth=(settings.ELASTICSEARCH_USER, settings.ELASTICSEARCH_PASSWORD) +) + + +class Command(BaseCommand): + help = 'Remove an APK and its report data' + + def add_arguments(self, parser): + parser.add_argument('hash', type=str) + + def handle(self, *args, **options): + sha256 = options['hash'] + + if default_storage.exists(sha256): + print("APK is saved to default_storage, removing…") + default_storage.delete(sha256) + if default_storage.exists(f"andro_cfg_{sha256}"): + print("AndroCFG data saved to default_storage, removing…") + default_storage.delete(f"andro_cfg_{sha256}") + + for i in indexes: + try: + es.delete(index=i, id=sha256) + except: + pass diff --git a/bazaar/core/mobsf.py b/bazaar/core/mobsf.py index 724aa30..72fd18d 100644 --- a/bazaar/core/mobsf.py +++ b/bazaar/core/mobsf.py @@ -1,5 +1,6 @@ import logging import requests +import time from django.conf import settings logger = logging.getLogger(__name__) @@ -48,7 +49,15 @@ def scan(self, data): """ logger.debug(f"Requesting {self.__server} to scan {data['hash']}") headers = {'Authorization': self.__apikey} - r = requests.post(f'{self.__server}/api/v1/scan', data=data, headers=headers, timeout=10) + max_attempts = 100 + for attempt in range(max_attempts): + try: + r = requests.post(f'{self.__server}/api/v1/scan', data=data, headers=headers, timeout=10) + break + except (requests.ConnectionError, requests.Timeout, requests.ReadTimeout) as e: + if attempt == max_attempts - 1: + raise + time.sleep(5) return r.json() def scans(self, page=1, page_size=100): @@ -69,7 +78,9 @@ def report_json(self, data): headers = {'Authorization': self.__apikey} data = {'hash': data['hash']} r = requests.post(f'{self.__server}/api/v1/report_json', data=data, headers=headers, timeout=10) - + while r.status_code != 200: + time.sleep(3) + r = requests.post(f'{self.__server}/api/v1/report_json', data=data, headers=headers, timeout=10) return r.json() def delete_scan(self, data): diff --git a/bazaar/core/modules/mobsf.py b/bazaar/core/modules/mobsf.py index 61eb8c6..bc9517e 100644 --- a/bazaar/core/modules/mobsf.py +++ b/bazaar/core/modules/mobsf.py @@ -112,7 +112,7 @@ def analysis(sha256): retry_on_conflict=5) except Exception as e: - logging.warning(e) + logging.exception(e) es.update(index=settings.ELASTICSEARCH_TASKS_INDEX, id=sha256, body={'doc': {'mobsf_analysis': -1}}, retry_on_conflict=5) diff --git a/bazaar/core/modules/quarkengine.py b/bazaar/core/modules/quarkengine.py index d564682..65fb542 100644 --- a/bazaar/core/modules/quarkengine.py +++ b/bazaar/core/modules/quarkengine.py @@ -29,8 +29,9 @@ def run_freshquark(): def analysis(sha256): - es.update(index=settings.ELASTICSEARCH_TASKS_INDEX, id=sha256, body={'doc': {'quark_analysis': 5}}, + es.update(index=settings.ELASTICSEARCH_TASKS_INDEX, id=sha256, body={'doc': {'quark_analysis': 1}}, retry_on_conflict=5) + with NamedTemporaryFile() as f: f.write(default_storage.open(sha256).read()) f.seek(0) diff --git a/bazaar/core/modules/similarity.py b/bazaar/core/modules/similarity.py index 068c395..5272ab3 100644 --- a/bazaar/core/modules/similarity.py +++ b/bazaar/core/modules/similarity.py @@ -95,11 +95,10 @@ def analysis(sha256): logging.error('Got an error %s', e) es.update(index=settings.ELASTICSEARCH_APK_INDEX, id=sha256, body={'doc': doc}, retry_on_conflict=5) + es.update(index=settings.ELASTICSEARCH_TASKS_INDEX, id=sha256, body={'doc': {'ssdeep_analysis': 2}}, + retry_on_conflict=5) del apk, f, doc gc.collect() - es.update(index=settings.ELASTICSEARCH_TASKS_INDEX, id=sha256, body={'doc': {'ssdeep_analysis': 2}}, - retry_on_conflict=5) - return {'status': 'success', 'info': ''} diff --git a/bazaar/core/tasks.py b/bazaar/core/tasks.py index cd3eafa..63d94e6 100644 --- a/bazaar/core/tasks.py +++ b/bazaar/core/tasks.py @@ -29,10 +29,10 @@ def _prepare(sha256): tasks = { - 'apkid_analysis': 0, - 'ssdeep_analysis': 0, - 'extract_classes': 0, - 'quark_analysis': 0, + 'apkid_analysis': 1, + 'ssdeep_analysis': 1, + 'extract_classes': 1, + 'quark_analysis': 1, 'analysis_date': timezone.now() } diff --git a/local.yml b/local.yml index 946683b..64b2ab2 100644 --- a/local.yml +++ b/local.yml @@ -98,7 +98,7 @@ services: start_period: 20s mobsf: - image: opensecurity/mobile-security-framework-mobsf:v4.5.2 + image: opensecurity/mobile-security-framework-mobsf:v4.5.3 hostname: mobsf ports: - 8181:8000 diff --git a/production.yml b/production.yml index 2377353..5925885 100644 --- a/production.yml +++ b/production.yml @@ -103,7 +103,7 @@ services: mobsf: - image: opensecurity/mobile-security-framework-mobsf:v4.5.2 + image: opensecurity/mobile-security-framework-mobsf:v4.5.3 hostname: mobsf environment: - "MOBSF_API_ONLY=1" From eb383dd5d7703100b0ccb05e98b7b570a4f9a720 Mon Sep 17 00:00:00 2001 From: trpyn Date: Mon, 12 Oct 2026 18:28:03 +0200 Subject: [PATCH 2/2] flake8 --- bazaar/core/management/commands/remove_apk.py | 6 +----- bazaar/core/mobsf.py | 2 +- bazaar/core/modules/similarity.py | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/bazaar/core/management/commands/remove_apk.py b/bazaar/core/management/commands/remove_apk.py index d4dd6e5..f70a99f 100644 --- a/bazaar/core/management/commands/remove_apk.py +++ b/bazaar/core/management/commands/remove_apk.py @@ -1,13 +1,9 @@ -from time import sleep -import logging from elasticsearch import Elasticsearch from django.conf import settings from django.core.management.base import BaseCommand -from django_q.tasks import async_task from django.core.files.storage import default_storage -from elasticsearch.helpers.actions import scan indexes = [ 'apk_analysis', @@ -44,5 +40,5 @@ def handle(self, *args, **options): for i in indexes: try: es.delete(index=i, id=sha256) - except: + except Exception: pass diff --git a/bazaar/core/mobsf.py b/bazaar/core/mobsf.py index 72fd18d..f352f6c 100644 --- a/bazaar/core/mobsf.py +++ b/bazaar/core/mobsf.py @@ -54,7 +54,7 @@ def scan(self, data): try: r = requests.post(f'{self.__server}/api/v1/scan', data=data, headers=headers, timeout=10) break - except (requests.ConnectionError, requests.Timeout, requests.ReadTimeout) as e: + except (requests.ConnectionError, requests.Timeout, requests.ReadTimeout): if attempt == max_attempts - 1: raise time.sleep(5) diff --git a/bazaar/core/modules/similarity.py b/bazaar/core/modules/similarity.py index 5272ab3..18f265e 100644 --- a/bazaar/core/modules/similarity.py +++ b/bazaar/core/modules/similarity.py @@ -96,7 +96,7 @@ def analysis(sha256): es.update(index=settings.ELASTICSEARCH_APK_INDEX, id=sha256, body={'doc': doc}, retry_on_conflict=5) es.update(index=settings.ELASTICSEARCH_TASKS_INDEX, id=sha256, body={'doc': {'ssdeep_analysis': 2}}, - retry_on_conflict=5) + retry_on_conflict=5) del apk, f, doc gc.collect()