diff --git a/bazaar/core/management/commands/remove_apk.py b/bazaar/core/management/commands/remove_apk.py new file mode 100644 index 0000000..f70a99f --- /dev/null +++ b/bazaar/core/management/commands/remove_apk.py @@ -0,0 +1,44 @@ +from elasticsearch import Elasticsearch + +from django.conf import settings +from django.core.management.base import BaseCommand +from django.core.files.storage import default_storage + + +indexes = [ + 'apk_analysis', + 'apk_analysis_tmp', + 'analysis_tasks', + 'vt_reports', + 'google_play_details', + 'dexofuzzy_apk', + 'ssdeep_apk', + 'ssdeep_manifest', +] +es = Elasticsearch( + settings.ELASTICSEARCH_HOSTS, + basic_auth=(settings.ELASTICSEARCH_USER, settings.ELASTICSEARCH_PASSWORD) +) + + +class Command(BaseCommand): + help = 'Remove an APK and its report data' + + def add_arguments(self, parser): + parser.add_argument('hash', type=str) + + def handle(self, *args, **options): + sha256 = options['hash'] + + if default_storage.exists(sha256): + print("APK is saved to default_storage, removing…") + default_storage.delete(sha256) + if default_storage.exists(f"andro_cfg_{sha256}"): + print("AndroCFG data saved to default_storage, removing…") + default_storage.delete(f"andro_cfg_{sha256}") + + for i in indexes: + try: + es.delete(index=i, id=sha256) + except Exception: + pass diff --git a/bazaar/core/mobsf.py b/bazaar/core/mobsf.py index 724aa30..f352f6c 100644 --- a/bazaar/core/mobsf.py +++ b/bazaar/core/mobsf.py @@ -1,5 +1,6 @@ import logging import requests +import time from django.conf import settings logger = logging.getLogger(__name__) @@ -48,7 +49,15 @@ def scan(self, data): """ logger.debug(f"Requesting {self.__server} to scan {data['hash']}") headers = {'Authorization': self.__apikey} - r = requests.post(f'{self.__server}/api/v1/scan', data=data, headers=headers, timeout=10) + max_attempts = 100 + for attempt in range(max_attempts): + try: + r = requests.post(f'{self.__server}/api/v1/scan', data=data, headers=headers, timeout=10) + break + except (requests.ConnectionError, requests.Timeout, requests.ReadTimeout): + if attempt == max_attempts - 1: + raise + time.sleep(5) return r.json() def scans(self, page=1, page_size=100): @@ -69,7 +78,9 @@ def report_json(self, data): headers = {'Authorization': self.__apikey} data = {'hash': data['hash']} r = requests.post(f'{self.__server}/api/v1/report_json', data=data, headers=headers, timeout=10) - + while r.status_code != 200: + time.sleep(3) + r = requests.post(f'{self.__server}/api/v1/report_json', data=data, headers=headers, timeout=10) return r.json() def delete_scan(self, data): diff --git a/bazaar/core/modules/mobsf.py b/bazaar/core/modules/mobsf.py index 61eb8c6..bc9517e 100644 --- a/bazaar/core/modules/mobsf.py +++ b/bazaar/core/modules/mobsf.py @@ -112,7 +112,7 @@ def analysis(sha256): retry_on_conflict=5) except Exception as e: - logging.warning(e) + logging.exception(e) es.update(index=settings.ELASTICSEARCH_TASKS_INDEX, id=sha256, body={'doc': {'mobsf_analysis': -1}}, retry_on_conflict=5) diff --git a/bazaar/core/modules/quarkengine.py b/bazaar/core/modules/quarkengine.py index d564682..65fb542 100644 --- a/bazaar/core/modules/quarkengine.py +++ b/bazaar/core/modules/quarkengine.py @@ -29,8 +29,9 @@ def run_freshquark(): def analysis(sha256): - es.update(index=settings.ELASTICSEARCH_TASKS_INDEX, id=sha256, body={'doc': {'quark_analysis': 5}}, + es.update(index=settings.ELASTICSEARCH_TASKS_INDEX, id=sha256, body={'doc': {'quark_analysis': 1}}, retry_on_conflict=5) + with NamedTemporaryFile() as f: f.write(default_storage.open(sha256).read()) f.seek(0) diff --git a/bazaar/core/modules/similarity.py b/bazaar/core/modules/similarity.py index 068c395..18f265e 100644 --- a/bazaar/core/modules/similarity.py +++ b/bazaar/core/modules/similarity.py @@ -95,11 +95,10 @@ def analysis(sha256): logging.error('Got an error %s', e) es.update(index=settings.ELASTICSEARCH_APK_INDEX, id=sha256, body={'doc': doc}, retry_on_conflict=5) + es.update(index=settings.ELASTICSEARCH_TASKS_INDEX, id=sha256, body={'doc': {'ssdeep_analysis': 2}}, + retry_on_conflict=5) del apk, f, doc gc.collect() - es.update(index=settings.ELASTICSEARCH_TASKS_INDEX, id=sha256, body={'doc': {'ssdeep_analysis': 2}}, - retry_on_conflict=5) - return {'status': 'success', 'info': ''} diff --git a/bazaar/core/tasks.py b/bazaar/core/tasks.py index cd3eafa..63d94e6 100644 --- a/bazaar/core/tasks.py +++ b/bazaar/core/tasks.py @@ -29,10 +29,10 @@ def _prepare(sha256): tasks = { - 'apkid_analysis': 0, - 'ssdeep_analysis': 0, - 'extract_classes': 0, - 'quark_analysis': 0, + 'apkid_analysis': 1, + 'ssdeep_analysis': 1, + 'extract_classes': 1, + 'quark_analysis': 1, 'analysis_date': timezone.now() } diff --git a/local.yml b/local.yml index 946683b..64b2ab2 100644 --- a/local.yml +++ b/local.yml @@ -98,7 +98,7 @@ services: start_period: 20s mobsf: - image: opensecurity/mobile-security-framework-mobsf:v4.5.2 + image: opensecurity/mobile-security-framework-mobsf:v4.5.3 hostname: mobsf ports: - 8181:8000 diff --git a/production.yml b/production.yml index 2377353..5925885 100644 --- a/production.yml +++ b/production.yml @@ -103,7 +103,7 @@ services: mobsf: - image: opensecurity/mobile-security-framework-mobsf:v4.5.2 + image: opensecurity/mobile-security-framework-mobsf:v4.5.3 hostname: mobsf environment: - "MOBSF_API_ONLY=1"