fix analysis stuck

bazaar/core/management/commands/remove_apk.py

@@ -0,0 +1,44 @@
+from elasticsearch import Elasticsearch
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.core.files.storage import default_storage
+
+
+indexes = [
+    'apk_analysis',
+    'apk_analysis_tmp',
+    'analysis_tasks',
+    'vt_reports',
+    'google_play_details',
+    'dexofuzzy_apk',
+    'ssdeep_apk',
+    'ssdeep_manifest',
+]
+es = Elasticsearch(
+    settings.ELASTICSEARCH_HOSTS,
+    basic_auth=(settings.ELASTICSEARCH_USER, settings.ELASTICSEARCH_PASSWORD)
+)
+
+
+class Command(BaseCommand):
+    help = 'Remove an APK and its report data'
+
+    def add_arguments(self, parser):
+        parser.add_argument('hash', type=str)
+
+    def handle(self, *args, **options):
+        sha256 = options['hash']
+
+        if default_storage.exists(sha256):
+            print("APK is saved to default_storage, removing…")
+            default_storage.delete(sha256)
+        if default_storage.exists(f"andro_cfg_{sha256}"):
+            print("AndroCFG data saved to default_storage, removing…")
+            default_storage.delete(f"andro_cfg_{sha256}")
+
+        for i in indexes:
+            try:
+                es.delete(index=i, id=sha256)
+            except Exception:
+                pass

bazaar/core/mobsf.py

@@ -1,5 +1,6 @@
 import logging
 import requests
+import time
 from django.conf import settings
 
 logger = logging.getLogger(__name__)
@@ -48,7 +49,15 @@ def scan(self, data):
         """
         logger.debug(f"Requesting {self.__server} to scan {data['hash']}")
         headers = {'Authorization': self.__apikey}
-        r = requests.post(f'{self.__server}/api/v1/scan', data=data, headers=headers, timeout=10)
+        max_attempts = 100
+        for attempt in range(max_attempts):
+            try:
+                r = requests.post(f'{self.__server}/api/v1/scan', data=data, headers=headers, timeout=10)
+                break
+            except (requests.ConnectionError, requests.Timeout, requests.ReadTimeout):
+                if attempt == max_attempts - 1:
+                    raise
+                time.sleep(5)
         return r.json()
 
     def scans(self, page=1, page_size=100):
@@ -69,7 +78,9 @@ def report_json(self, data):
         headers = {'Authorization': self.__apikey}
         data = {'hash': data['hash']}
         r = requests.post(f'{self.__server}/api/v1/report_json', data=data, headers=headers, timeout=10)
-
+        while r.status_code != 200:
+            time.sleep(3)
+            r = requests.post(f'{self.__server}/api/v1/report_json', data=data, headers=headers, timeout=10)
         return r.json()
 
     def delete_scan(self, data):

bazaar/core/modules/mobsf.py

@@ -112,7 +112,7 @@ def analysis(sha256):
                   retry_on_conflict=5)
 
     except Exception as e:
-        logging.warning(e)
+        logging.exception(e)
         es.update(index=settings.ELASTICSEARCH_TASKS_INDEX, id=sha256, body={'doc': {'mobsf_analysis': -1}},
                   retry_on_conflict=5)
 

bazaar/core/modules/quarkengine.py

@@ -29,8 +29,9 @@ def run_freshquark():
 
 
 def analysis(sha256):
-    es.update(index=settings.ELASTICSEARCH_TASKS_INDEX, id=sha256, body={'doc': {'quark_analysis': 5}},
+    es.update(index=settings.ELASTICSEARCH_TASKS_INDEX, id=sha256, body={'doc': {'quark_analysis': 1}},
               retry_on_conflict=5)
+
     with NamedTemporaryFile() as f:
         f.write(default_storage.open(sha256).read())
         f.seek(0)

bazaar/core/modules/similarity.py

@@ -95,11 +95,10 @@ def analysis(sha256):
                     logging.error('Got an error %s', e)
 
         es.update(index=settings.ELASTICSEARCH_APK_INDEX, id=sha256, body={'doc': doc}, retry_on_conflict=5)
+        es.update(index=settings.ELASTICSEARCH_TASKS_INDEX, id=sha256, body={'doc': {'ssdeep_analysis': 2}},
+                  retry_on_conflict=5)
 
     del apk, f, doc
     gc.collect()
 
-    es.update(index=settings.ELASTICSEARCH_TASKS_INDEX, id=sha256, body={'doc': {'ssdeep_analysis': 2}},
-              retry_on_conflict=5)
-
     return {'status': 'success', 'info': ''}

bazaar/core/tasks.py

@@ -29,10 +29,10 @@
 
 def _prepare(sha256):
     tasks = {
-        'apkid_analysis': 0,
-        'ssdeep_analysis': 0,
-        'extract_classes': 0,
-        'quark_analysis': 0,
+        'apkid_analysis': 1,
+        'ssdeep_analysis': 1,
+        'extract_classes': 1,
+        'quark_analysis': 1,
         'analysis_date': timezone.now()
     }
 

local.yml

@@ -98,7 +98,7 @@ services:
       start_period: 20s
 
   mobsf:
-    image: opensecurity/mobile-security-framework-mobsf:v4.5.2
+    image: opensecurity/mobile-security-framework-mobsf:v4.5.3
     hostname: mobsf
     ports:
       - 8181:8000

production.yml

@@ -103,7 +103,7 @@ services:
 
 
   mobsf:
-    image: opensecurity/mobile-security-framework-mobsf:v4.5.2
+    image: opensecurity/mobile-security-framework-mobsf:v4.5.3
     hostname: mobsf
     environment:
       - "MOBSF_API_ONLY=1"