Fixing --eta

Author: stamparm

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.7.55"
+VERSION = "1.10.7.56"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -249,6 +249,9 @@
 # Maximum number of techniques used in inject.py/getValue() per one value
 MAX_TECHNIQUES_PER_VALUE = 2
 
+# Fraction of the currently displayed progress-bar ETA kept when a fresh estimate arrives (eases the on-screen countdown toward the new value instead of snapping); 0 disables smoothing, higher is smoother but laggier
+ETA_DISPLAY_SMOOTHING = 0.5
+
 # In case of missing piece of partial union dump, buffered array must be flushed after certain size
 MAX_BUFFERED_PARTIAL_UNION_LENGTH = 1024
 

lib/request/inject.py

@@ -8,6 +8,7 @@
 from __future__ import print_function
 
 import re
+import threading
 import time
 
 from lib.core.agent import agent
@@ -16,6 +17,7 @@
 from lib.core.common import dataToStdout
 from lib.core.common import unArrayizeValue
 from lib.core.datatype import AttribDict
+from lib.utils.progress import ProgressBar
 from lib.utils.safe2bin import safecharencode
 from lib.core.common import Backend
 from lib.core.common import calculateDeltaSeconds
@@ -58,12 +60,14 @@
 from lib.core.exception import SqlmapNotVulnerableException
 from lib.core.exception import SqlmapUserQuitException
 from lib.core.settings import GET_VALUE_UPPERCASE_KEYWORDS
+from lib.core.settings import IS_TTY
 from lib.core.settings import INFERENCE_MARKER
 from lib.core.settings import MAX_TECHNIQUES_PER_VALUE
 from lib.core.settings import SQL_SCALAR_REGEX
 from lib.core.settings import UNICODE_ENCODING
 from lib.core.threads import getCurrentThreadData
 from lib.core.threads import runThreads
+from lib.core.threads import setDaemon
 from lib.core.unescaper import unescaper
 from lib.request.connect import Connect as Request
 from lib.request.direct import direct
@@ -405,6 +409,25 @@ def _verifyInferredValue(expression, value):
     except Exception:
         return True
 
+def valueParallelEligible():
+    """
+    Whether blind enumeration/dumping should take the value-parallel path (one whole value per worker)
+    rather than the classic char loop. It is chosen for concurrency ('--threads') and, independently, to
+    render a single whole-job progress bar/ETA ('--eta'). A concurrency-safe channel - boolean or the
+    HTTP/2 timeless oracle - qualifies with either. Classic time-based qualifies only single-threaded under
+    '--eta': concurrent SLEEP measurements interfere, so it must stay sequential (where it is safe, and
+    still gets the whole-job ETA that matters most for the slowest channel). Callers add their own extra
+    guards (e.g. '--dns-domain', per-column comments).
+    """
+
+    concurrencySafe = isTechniqueAvailable(PAYLOAD.TECHNIQUE.BOOLEAN) or kb.get("timeless") is not None
+
+    if conf.threads > 1:
+        return concurrencySafe
+    if conf.eta:
+        return concurrencySafe or isTechniqueAvailable(PAYLOAD.TECHNIQUE.TIME)
+    return False
+
 def _threadedInferenceValues(exprBuilder, indices, context=None, charsetType=None, dump=False):
     """
     Value-parallel blind retrieval.
@@ -446,6 +469,13 @@ def _threadedInferenceValues(exprBuilder, indices, context=None, charsetType=Non
     results = [None] * len(indices)
     cursor = iter(xrange(len(indices)))
 
+    # With '--eta' show a single value-level bar (values completed / total) instead of the per-value
+    # stream: it is monotonic and carries a meaningful ETA across the whole set, and - unlike the
+    # per-char bar drawn inside bisection() - it survives the worker's disableStdOut (drawn forced,
+    # below), so '--eta' is honoured under '--threads' too. Skipped for trivial single-value sets.
+    etaProgress = ProgressBar(maxValue=len(indices)) if (conf.eta and len(indices) > 1) else None
+    completed = [0]
+
     def inferenceThread():
         threadData = getCurrentThreadData()
         # Each per-value bisection streams its characters to stdout and mirrors them into
@@ -489,13 +519,36 @@ def inferenceThread():
             with kb.locks.value:
                 results[slot] = value
 
+            if etaProgress is not None:
+                with kb.locks.io:
+                    completed[0] += 1
+                    threadData.disableStdOut = False        # let the value-level bar through (per-char streaming stays muted)
+                    try:
+                        etaProgress.progress(completed[0])
+                    finally:
+                        threadData.disableStdOut = True
+
             # Stream each retrieved value as it completes (they arrive out of order under threads, exactly
             # like the error/union dumps), so a dump shows its data live rather than a silent counter.
-            if conf.verbose >= 1 and not kb.bruteMode and not isNoneValue(value):
+            elif conf.verbose >= 1 and not kb.bruteMode and not isNoneValue(value):
                 with kb.locks.io:
                     rendered = safecharencode(unArrayizeValue(value))
                     dataToStdout("[%s] [INFO] retrieved: %s\n" % (time.strftime("%X"), "'%s'" % rendered if dump else rendered), forceOutput=True)
 
+    # Keep the '--eta' countdown live between value updates: a value (esp. time-based) can take many
+    # seconds, so a daemon redraws the bar every second with the ETA decremented by elapsed time (it runs
+    # on its own thread, so it is not muted by the workers' disableStdOut, and shares kb.locks.io with them).
+    etaTickerStop = threading.Event()
+    etaTicker = None
+    if etaProgress is not None and IS_TTY:
+        def _etaTicker():
+            while not etaTickerStop.wait(1.0):
+                with kb.locks.io:
+                    etaProgress.tick()
+        etaTicker = threading.Thread(target=_etaTicker, name="eta-ticker")
+        setDaemon(etaTicker)
+        etaTicker.start()
+
     # Save/restore the calling thread's state: with a single thread runThreads runs the worker
     # INLINE on this thread, so the worker's disableStdOut/shared mutations must not leak out.
     savedPartRun = kb.partRun
@@ -505,6 +558,9 @@ def inferenceThread():
     try:
         runThreads(min(conf.threads or 1, len(indices)) or 1, inferenceThread)
     finally:
+        etaTickerStop.set()
+        if etaTicker is not None:
+            etaTicker.join(timeout=2)
         kb.partRun = savedPartRun
         mainThreadData.disableStdOut = savedStdOut
         mainThreadData.shared = savedShared

lib/utils/progress.py

@@ -13,6 +13,8 @@
 from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
+from lib.core.settings import ETA_DISPLAY_SMOOTHING
+from lib.core.settings import IS_TTY
 
 class ProgressBar(object):
     """
@@ -26,7 +28,9 @@ def __init__(self, minValue=0, maxValue=10, totalWidth=None):
         self._span = max(self._max - self._min, 0.001)
         self._width = totalWidth if totalWidth else conf.progressWidth
         self._amount = 0
-        self._start = None
+        self._start = time.time()   # begin timing at construction, so the first completed item already yields an estimate
+        self._eta = None            # last estimated seconds-remaining and when it was computed, so tick()
+        self._etaAt = None          # can keep the countdown live between (possibly slow) item updates
         self.update()
 
     def _convertSeconds(self, value):
@@ -73,24 +77,49 @@ def update(self, newAmount=0):
 
     def progress(self, newAmount):
         """
-        This method saves item delta time and shows updated progress bar with calculated eta
+        Redraw the bar with an ETA from the average time per completed item so far, applied to the items
+        still remaining: (elapsed / done) * (max - newAmount). The remaining-item count is (max - newAmount)
+        - i.e. at 1/3 it estimates the 2 items left, at 2/3 the 1 left - not just the current item.
         """
 
-        if self._start is None or newAmount > self._max:
-            self._start = time.time()
-            eta = None
+        now = time.time()
+        if newAmount > self._max:               # counter rollover/reset -> restart timing
+            self._start = now
+            self._eta = None
+
+        done = newAmount - self._min
+        elapsed = now - self._start
+        target = (elapsed / done) * (self._max - newAmount) if (done > 0 and elapsed > 0) else None
+
+        if target is None:
+            self._eta = None
+        elif self._eta is None:
+            self._eta = target                  # first estimate: nothing to ease from
         else:
-            delta = time.time() - self._start
-            eta = (self._max - self._min) * (1.0 * delta / newAmount) - delta
+            current = max(0, self._eta - (now - self._etaAt))   # what is on screen now (already decremented by tick())
+            self._eta = ETA_DISPLAY_SMOOTHING * current + (1 - ETA_DISPLAY_SMOOTHING) * target   # ease into the fresh estimate
 
+        self._etaAt = now
         self.update(newAmount)
+        self.draw(self._eta)
+
+    def tick(self):
+        """
+        Redraw the current bar with its ETA decremented by real elapsed time, so the countdown stays
+        live between updates (e.g. during a long time-based wait) instead of freezing at the last estimate
+        """
+
+        eta = None if self._eta is None else max(0, self._eta - (time.time() - self._etaAt))
         self.draw(eta)
 
     def draw(self, eta=None):
         """
         This method draws the progress bar if it has changed
         """
 
+        if not IS_TTY:      # a progress bar is a terminal animation; suppress it when piped/redirected (as done for other '\r' output)
+            return
+
         dataToStdout("\r%s %d/%d%s" % (self._progBar, self._amount, self._max, ("  (ETA %s)" % (self._convertSeconds(int(eta)) if eta is not None else "??:??"))))
         if self._amount >= self._max:
             dataToStdout("\r%s\r" % (" " * self._width))

plugins/generic/databases.py

@@ -403,12 +403,13 @@ def getTables(self, bruteForce=None):
 
                     # Value-parallel, prediction-assisted name enumeration for the DBMSes using the
                     # generic "<db>, <index>" blind template. Retrieves whole names concurrently (one per
-                    # worker, each decoded sequentially so wordlist prediction applies). Used only with
-                    # '--threads' (like getColumns below); single-thread stays on the classic getValue loop,
-                    # which still gets predictive inference via getPartRun. The special templates stay serial.
+                    # worker, each decoded sequentially so wordlist prediction applies). Used with '--threads'
+                    # and under '--eta' (one whole-job bar/ETA) per valueParallelEligible(); plain single-thread
+                    # stays on the classic getValue loop, which still gets predictive inference via getPartRun.
+                    # The special templates stay serial.
                     genericTemplate = Backend.getIdentifiedDbms() not in (DBMS.SYBASE, DBMS.MAXDB, DBMS.ACCESS, DBMS.MCKOI, DBMS.EXTREMEDB, DBMS.SQLITE, DBMS.FIREBIRD, DBMS.HSQLDB, DBMS.INFORMIX, DBMS.FRONTBASE, DBMS.VIRTUOSO, DBMS.SPANNER)
 
-                    if genericTemplate and conf.threads > 1 and isTechniqueAvailable(PAYLOAD.TECHNIQUE.BOOLEAN):
+                    if genericTemplate and inject.valueParallelEligible():
                         for table in (inject._threadedInferenceValues(lambda index: _query % (unsafeSQLIdentificatorNaming(db), index), indexRange, context="Tables") or []):
                             if not isNoneValue(table):
                                 kb.hintValue = table
@@ -899,9 +900,11 @@ def columnNameQuery(index):
 
                 # Value-parallel column-NAME enumeration: the same axis/mechanism as getTables (one name per
                 # worker, decoded sequentially - no length probe, predictive inference applies, names stream
-                # live). Serial fallback for single-thread and when also fetching per-column comments.
+                # live, and under '--eta' a single whole-job bar/ETA is drawn). Eligibility is shared via
+                # valueParallelEligible(); serial fallback only when also fetching per-column comments (those
+                # need an extra per-column query).
                 columnNames = None
-                if conf.threads > 1 and not conf.getComments and isTechniqueAvailable(PAYLOAD.TECHNIQUE.BOOLEAN):
+                if not conf.getComments and inject.valueParallelEligible():
                     columnNames = inject._threadedInferenceValues(columnNameQuery, indexList, context="Columns")
 
                 for position, index in enumerate(indexList):

plugins/generic/entries.py

@@ -444,9 +444,11 @@ def cellQuery(column, index):
                             # value's characters across threads) and the per-column Huffman model + low-cardinality
                             # guessing engage under concurrency. Used for the boolean channel with '--threads', and
                             # for the HTTP/2 timeless oracle (its per-thread connections make concurrency safe and
-                            # deterministic - unlike classic time-based, whose char-parallel loop interleaves its
-                            # per-thread output). Single-thread and classic time-based keep the char-parallel loop.
-                            if conf.threads > 1 and not conf.dnsDomain and (isTechniqueAvailable(PAYLOAD.TECHNIQUE.BOOLEAN) or kb.get("timeless") is not None):
+                            # deterministic). Also used under '--eta' - including single-threaded time-based, the
+                            # slowest channel - to drive one whole-job progress bar/ETA (how long the dump takes)
+                            # instead of a per-cell counter. Eligibility (channel x threads/eta) is valueParallelEligible();
+                            # '--dns-domain' keeps the classic loop (its OOB fast path bypasses bisection).
+                            if not conf.dnsDomain and inject.valueParallelEligible():
                                 # One value-parallel pass over every (non-empty) cell, so there is a single
                                 # thread pool and values stream live as they complete - out of order, exactly
                                 # like the error/union dumps - instead of a silent progress counter.