[pull] master from sqlmapproject:master

data/txt/sha256sums.txt

@@ -167,16 +167,16 @@ df768bcb9838dc6c46dab9b4a877056cb4742bd6cfaaf438c4a3712c5cc0d264  extra/shutils/
 d69e84f1648cdb907f5d2dd454f03874a4613752b07867510145d51d84b3c56f  lib/controller/handler.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/controller/__init__.py
 9da83429449d78797c18bb79ff425aa1eddf5b26b9987d25d042eb0998053675  lib/core/agent.py
-12d0f1f28796b6fbf5629a3fd335b4098eac0583f832d1aa650efa22bf52e782  lib/core/bigarray.py
-e55201cbaa05aac7091cccda361963a74635172bdd9d403feeadf400e09a14cf  lib/core/common.py
+905e49d6e030a60f7767c71e0726e0def57c50542210afd9be1cdec122d2d1ce  lib/core/bigarray.py
+cd22e671c7c96ca8e0e23e1578780e7390dbb50055dabf7bd44f933318c2a9b0  lib/core/common.py
 8f1272487e1adfcc8c755a2f56f0c6d21eac5e685a73a9a159482f9dc9142bc5  lib/core/compat.py
 742bce10b97034966021ec60c7ac294db4af4fe7893613d63172a02c29f009f8  lib/core/convert.py
 c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6  lib/core/data.py
-6acb645b1f285b21673c70824b03f6209acc5993b50e50da5ed2c713a30626f5  lib/core/datatype.py
+6c8d40d6bbab4a60d09eb03324a3352d85df1a741c62044e73701e92172d1d38  lib/core/datatype.py
 70fb2528e580b22564899595b0dff6b1bc257c6a99d2022ce3996a3d04e68e4e  lib/core/decorators.py
 147823c37596bd6a56d677697781f34b8d1d1671d5a2518fbc9468d623c6d07d  lib/core/defaults.py
 2f44a1bfe6f18aafe64147b99e69aa93cf438c0e7befe59f4e2aee9065c8b7b6  lib/core/dicts.py
-2592b0fd38c272c0b0d49878f4449437eb8ba8ff7536bb39b2ac9a2511010f7c  lib/core/dump.py
+12155385c1c4f763c1e8fcb92165015b913620ae1fec1e8de303e4fe841e5a69  lib/core/dump.py
 6b6514202c6ca2d29069176bccf10492927d83e6ede06c9f4b4fcc6164e61856  lib/core/enums.py
 5387168e5dfedd94ae22af7bb255f27d6baaca50b24179c6b98f4f325f5cc7b4  lib/core/exception.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/core/__init__.py
@@ -186,15 +186,15 @@ c1a9edb894033f1cef0a15a05cca196f816df3465444134af171870dedbe1538  lib/core/optio
 ccc4a717e887652b1fcce073d9409d9c59a3b28548c703a9e453d15845f90cd7  lib/core/patch.py
 49c0fa7e3814dfda610d665ee02b12df299b28bc0b6773815b4395514ddf8dec  lib/core/profiling.py
 03db48f02c3d07a047ddb8fe33a757b6238867352d8ddda2a83e4fec09a98d04  lib/core/readlineng.py
-48797d6c34dd9bb8a53f7f3794c85f4288d82a9a1d6be7fcf317d388cb20d4b3  lib/core/replication.py
+9bf174058f15d14e24e94f9aaf42df045119d3617c6c54bd2f3af79b462f331d  lib/core/replication.py
 0b8c38a01bb01f843d94a6c5f2075ee47520d0c4aa799cecea9c3e2c5a4a23a6  lib/core/revision.py
 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c  lib/core/session.py
-f47aed1aa1a986dc2d3789358a34f79aa1111e7ab8747c4fba4dfe3443f778a0  lib/core/settings.py
+aefc6278c2eee19a2411d19afe85ee78a30214750903b2321d04b2a6a2881b59  lib/core/settings.py
 cd5a66deee8963ba8e7e9af3dd36eb5e8127d4d68698811c29e789655f507f82  lib/core/shell.py
 bcb5d8090d5e3e0ef2a586ba09ba80eef0c6d51feb0f611ed25299fbb254f725  lib/core/subprocessng.py
 70ea3768f1b3062b22d20644df41c86238157ec80dd43da40545c620714273c6  lib/core/target.py
 d213562601682fd72603a22f35e5af4e3f41e23bfb143e1584a4fa212a232635  lib/core/testing.py
-e3e653364d08d04d7492aa40a2bd29c6a28f4d78fecdd6c10f21f6cb28b98b4c  lib/core/threads.py
+95656c44bab1771f4808030dd6a17eae5b129cb1234443f00b19695c7b712b86  lib/core/threads.py
 b9aacb840310173202f79c2ba125b0243003ee6b44c92eca50424f2bdfc83c02  lib/core/unescaper.py
 53e396902cb2546eaa09e77073fcba8be8827ee9ce055cfc899e81b0e6ad4d6d  lib/core/update.py
 2400e465fa4d13e4c32795910878c71ff212e4361b46428d57ce43983f5e997c  lib/core/wordlist.py
@@ -211,10 +211,10 @@ c2f34e27578742e729c2fa9c1d4f0a0d8f8f7f4cf0fc14c62ec817a260c71dec  lib/parse/site
 1be3da334411657461421b8a26a0f2ff28e1af1e28f1e963c6c92768f9b0847c  lib/request/basicauthhandler.py
 369484a2999d29f49bf839a329d1686ed94f6ea27c695e027fe08c8da51f30a3  lib/request/basic.py
 bc61bc944b81a7670884f82231033a6ac703324b34b071c9834886a92e249d0e  lib/request/chunkedhandler.py
-390cc4882ba9c76e16a5376ba6d856079e7cb47a3e4ee11925139e637ce05050  lib/request/comparison.py
+d4bb0869b03602a0c8f9e0e0fd217753f14ddadf848fc9f3c65a74d03feb9958  lib/request/comparison.py
 b9e2db44d265909792f6cc821ff910727b14aa2d5063c74b0f2ea6d40c4f3d9d  lib/request/connect.py
 8e06682280fce062eef6174351bfebcb6040e19976acff9dc7b3699779783498  lib/request/direct.py
-cf019248253a5d7edb7bc474aa020b9e8625d73008a463c56ba2b539d7f2d8ec  lib/request/dns.py
+05198477dbdeb6c405059eb21cbbcf9cb6804cc54a0f2a1d11741bfc6cbb7ca2  lib/request/dns.py
 92c81cc31ff4a396723242058fb2152c9e9745f8412d01ea74480b048a53af6c  lib/request/httpshandler.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/request/__init__.py
 7a0ac2522213e756348fd871a7af74cc963bdc82f9d7ade57be5de42b5bf7cab  lib/request/inject.py
@@ -236,7 +236,7 @@ f522436fbd14bdab090a1d305fcac0361800cb8e36c8cbcb47933298376a71e0  lib/takeover/r
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/techniques/blind/__init__.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/techniques/dns/__init__.py
 3df9839fb92a81d46b6194d7adacb43f391efb78b071783c132e8d596ecbfaf1  lib/techniques/dns/test.py
-2934514a60cbcd48675053a73f785b4c7bfe606b51c34ae81a86818362ec4672  lib/techniques/dns/use.py
+74ca78082dcd20b3faf07cc944cd65ea552996df40e6fb58d0a011b262528456  lib/techniques/dns/use.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/techniques/error/__init__.py
 5bbef46c16e34fd80e3f9f0e9aa255ce2e39be0d0e57479e25890b041c7efc7d  lib/techniques/error/use.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/techniques/__init__.py
@@ -574,7 +574,7 @@ dcdeed9ee285e63cf06baf8347e3db7f210ef25a63869bab78ce1ec6898ae191  tamper/unional
 ce1b6bf8f296de27014d6f21aa8b3df9469d418740cd31c93d1f5e36d6c509cf  tamper/xforwardedfor.py
 44401cad3e39ae9fb899ed5d0e2fdd0879561de05c3117f17f3b0db54f4e3724  tests/__init__.py
 bfb553602eb5d20b4ab5928dbcf8e6a3e7e5ff69f7d30d1f53ef6d323c237f6c  tests/test_agent.py
-d4d7d3525d25ce72bf38bd38b5fdf61144e381993d63be7dc72b2b4811ffab67  tests/test_bigarray.py
+feb763ddcbf4f32822372ca53f8c71c754af7b72510ef06e1e9c77927fc90b10  tests/test_bigarray.py
 27ad87c0ea377e0657bd6f6a4eaa0e9756aa9d28ec0483bdadeb3f66dcc4660d  tests/test_charset.py
 9e678a56e16211c49ab4995b6c658d3f122bfa3b357d9e17ff38f5a489ace6ad  tests/test_cloak.py
 a48c411fea864e6bcd6a1c7e1a35094b8cda8d15088fd9e7b0270542ae20daa9  tests/test_common_helpers.py
@@ -587,6 +587,8 @@ a48c411fea864e6bcd6a1c7e1a35094b8cda8d15088fd9e7b0270542ae20daa9  tests/test_com
 9c0a0cd0b2d52a53f75c98c60f87a022354b7c3dc4baaf3fe1e272a0af5b7f0a  tests/test_dialectdbms.py
 e40a49cfa73c45b3c3c6d1d1d00738861e270cb7a07b28f5a5356f9c7c800cf2  tests/test_dialect.py
 993a2d4d87c4fbaf261663b069629acc95ee4405aa0c42cf5a8f39649fdb0fff  tests/test_dicts.py
+a38f3257aa218fa706ddb903c181715b2286619c46aea0097b7d365d18c410c5  tests/test_dns_engine.py
+703faac01f38224ba85bd0fc398d939ea034f1d7fd641cdc15da4f77ec049443  tests/test_dns_server.py
 9cd5841349bc4db818658d12184929a96f7f279eff1f53ad18a54dbefbd6b276  tests/test_dump_jsonl.py
 2bbe4b01f79992cfa8884651fc0a28dbd0e3abb0cbea9eb7eadf1f98ca3c3420  tests/test_encoding.py
 bb6991260a994fcbe79e05febaa34affd5631d02299fbc626820addd5f6ea4f4  tests/test_error_engine.py
@@ -600,12 +602,12 @@ cde0bea1263ae857561f91ed2bd515e972b716743f017d31b1718a8546c72759  tests/test_pag
 4bac34af2abddce003756d6776e89b2fda220bb7603ef3761f4f37ee29f9c369  tests/test_payload_marking.py
 6bfc8201724078bd9d6d559916ef73c9ff97e19b0f2948f37e588a49b027795f  tests/test_payloads_structure.py
 a6d013104601c0414628aff3d8b5b69bee3e6733781d8f8da880457d8b44bd3a  tests/test_property.py
-5c95e7863190e440234f231864fb1219c35207132762858cc95181c57086bafc  tests/test_replication.py
+2dfefb4bfaee3868152835502ec43da317c4f274b1d55cd2ef21e4f7390c9bea  tests/test_replication.py
 67a5241aeebc20eb1c20cfc490422a59af5179040824e5731bd785db2e6bf750  tests/test_report.py
 cec98d72992c0799229a780fa7f0d7f3fb01ec2d708187ce0e4a05c8612f291b  tests/test_safe2bin.py
 a1c6cda1e5b483f61e6a4f8ddd0b06a15ddaa3fd2119bfb9dbd9cc970d7a751d  tests/test_settings_regex.py
 d3d991331096e16e5019de3d652e9fff92c09bd9f97c50b1c2c3ceb0ed49b17e  tests/test_sqlparse.py
-41907c873663401f979b87eaff3efc8d52e0ce96cbe1eef7aa70c6d3af8cd5cf  tests/test_strings.py
+8bcbf1091134dd0a62f6201f8b3645ed87b5ff2f7ba40a87231a29dac412591f  tests/test_strings.py
 f3a628db8a3e05baee580c02132e95b164695e4b3ee1785707e3ea148702449a  tests/test_tamper.py
 b3e13febe9e0ff6f97334f2868655bfdbaa18755e464a6dc4c6d424f513bad02  tests/test_targeturl.py
 639851dc68f62b559b200b09c308e64e453f414969940005bac75dc0ab07a6b6  tests/test_texthelpers.py

lib/core/bigarray.py

@@ -172,6 +172,12 @@ def pop(self):
                 except OSError:
                     pass
 
+                # Note: the formerly on-disk chunk is now an in-memory list (and its file has been
+                # removed), so any cache entry still pointing at it is stale; dropping it prevents
+                # serving outdated data if that chunk index is later re-dumped (e.g. append after pop)
+                if isinstance(self.cache, Cache) and self.cache.index == idx:
+                    self.cache = None
+
         return self.chunks[-1].pop()
 
     def index(self, value):

lib/core/common.py

@@ -2532,19 +2532,23 @@ def readCachedFileContent(filename, mode='r'):
     True
     """
 
-    if filename not in kb.cache.content:
+    content = kb.cache.content.get(filename)
+
+    if content is None:
         with kb.locks.cache:
-            if filename not in kb.cache.content:
+            content = kb.cache.content.get(filename)
+            if content is None:
                 checkFile(filename)
                 try:
                     with openFile(filename, mode) as f:
-                        kb.cache.content[filename] = f.read()
+                        content = f.read()
+                        kb.cache.content[filename] = content
                 except (IOError, OSError, MemoryError) as ex:
                     errMsg = "something went wrong while trying "
                     errMsg += "to read the content of file '%s' ('%s')" % (filename, getSafeExString(ex))
                     raise SqlmapSystemException(errMsg)
 
-    return kb.cache.content[filename]
+    return content
 
 def average(values):
     """
@@ -5342,6 +5346,8 @@ def splitFields(fields, delimiter=','):
     ['foo', 'bar', 'max(foo,bar)']
     >>> splitFields("a, 'b, c', d")
     ['a', "'b, c'", 'd']
+    >>> splitFields('a; b; max(c; d)', delimiter=';')
+    ['a', 'b', 'max(c;d)']
     """
 
     # collapse "<delimiter> " -> "<delimiter>" but only OUTSIDE quoted string literals, so a
@@ -5372,11 +5378,11 @@ def splitFields(fields, delimiter=','):
         index += 1
 
     fields = "".join(normalized)
-    commas = [-1, len(fields)]
-    commas.extend(zeroDepthSearch(fields, ','))
-    commas = sorted(commas)
+    splits = [-1, len(fields)]
+    splits.extend(zeroDepthSearch(fields, delimiter))
+    splits = sorted(splits)
 
-    return [fields[x + 1:y] for (x, y) in _zip(commas, commas[1:])]
+    return [fields[x + 1:y] for (x, y) in _zip(splits, splits[1:])]
 
 def pollProcess(process, suppress_errors=False):
     """

lib/core/datatype.py

@@ -151,7 +151,7 @@ def __getitem__(self, key):
     def get(self, key, default=None):
         try:
             return self.__getitem__(key)
-        except:
+        except (KeyError, TypeError):
             return default
 
     def __setitem__(self, key, value):

lib/core/dump.py

@@ -94,9 +94,9 @@ def _write(self, data, newline=True, console=True, content_type=None):
             except IOError as ex:
                 errMsg = "error occurred while writing to log file ('%s')" % getSafeExString(ex)
                 raise SqlmapGenericException(errMsg)
-
-            if multiThreadMode:
-                self._lock.release()
+            finally:
+                if multiThreadMode:
+                    self._lock.release()
 
         kb.dataOutputFlag = True
 

lib/core/replication.py

@@ -23,8 +23,11 @@ class Replication(object):
     """
 
     def __init__(self, dbpath):
+        self.dbpath = dbpath
+        self.connection = None
+        self.cursor = None
+
         try:
-            self.dbpath = dbpath
             self.connection = sqlite3.connect(dbpath)
             self.connection.isolation_level = None
             self.cursor = self.connection.cursor()
@@ -120,8 +123,17 @@ def createTable(self, tblname, columns=None, typeless=False):
         return Replication.Table(parent=self, name=tblname, columns=columns, typeless=typeless)
 
     def __del__(self):
-        self.cursor.close()
-        self.connection.close()
+        try:
+            if self.cursor is not None:
+                self.cursor.close()
+        except Exception:
+            pass
+
+        try:
+            if self.connection is not None:
+                self.connection.close()
+        except Exception:
+            pass
 
     # sqlite data types
     NULL = DataType('NULL')

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.6.134"
+VERSION = "1.10.6.143"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -809,6 +809,11 @@
 # Reference: http://www.tcpipguide.com/free/t_DNSLabelsNamesandSyntaxRules.htm
 MAX_DNS_LABEL = 63
 
+# Maximum number of (most recent) DNS resolution requests retained by the DNS server (bounded so
+# that unrelated/stray traffic to the listening :53 socket cannot grow memory without limit; the
+# value is popped right after it is triggered, so only recent entries ever matter)
+MAX_DNS_REQUESTS = 1000
+
 # Alphabet used for prefix and suffix strings of name resolution requests in DNS technique (excluding hexadecimal chars for not mixing with inner content)
 DNS_BOUNDARIES_ALPHABET = re.sub(r"[a-fA-F]", "", string.ascii_letters)
 

lib/core/threads.py

@@ -27,7 +27,6 @@
 from lib.core.exception import SqlmapUserQuitException
 from lib.core.exception import SqlmapValueException
 from lib.core.settings import MAX_NUMBER_OF_THREADS
-from lib.core.settings import PYVERSION
 
 shared = AttribDict()
 
@@ -93,7 +92,7 @@ def getCurrentThreadName():
     Returns current's thread name
     """
 
-    return threading.current_thread().getName()
+    return threading.current_thread().name
 
 def exceptionHandledFunction(threadFunction, silent=False):
     try:
@@ -107,17 +106,14 @@ def exceptionHandledFunction(threadFunction, silent=False):
 
         if not silent and kb.get("threadContinue") and not kb.get("multipleCtrlC") and not isinstance(ex, (SqlmapUserQuitException, SqlmapSkipTargetException)):
             errMsg = getSafeExString(ex) if isinstance(ex, SqlmapBaseException) else "%s: %s" % (type(ex).__name__, getSafeExString(ex))
-            logger.error("thread %s: '%s'" % (threading.currentThread().getName(), errMsg))
+            logger.error("thread %s: '%s'" % (threading.current_thread().name, errMsg))
 
             if conf.get("verbose") > 1 and not isinstance(ex, SqlmapConnectionException):
                 traceback.print_exc()
 
 def setDaemon(thread):
     # Reference: http://stackoverflow.com/questions/190010/daemon-threads-explanation
-    if PYVERSION >= "2.6":
-        thread.daemon = True
-    else:
-        thread.setDaemon(True)
+    thread.daemon = True
 
 def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardException=True, threadChoice=False, startThreadMsg=True):
     threads = []
@@ -233,7 +229,7 @@ def _threadFunction():
     except (SqlmapConnectionException, SqlmapValueException) as ex:
         print()
         kb.threadException = True
-        logger.error("thread %s: '%s'" % (threading.currentThread().getName(), ex))
+        logger.error("thread %s: '%s'" % (threading.current_thread().name, ex))
 
         if conf.get("verbose") > 1 and isinstance(ex, SqlmapValueException):
             traceback.print_exc()
@@ -249,7 +245,7 @@ def _threadFunction():
 
                 kb.threadException = True
                 errMsg = unhandledExceptionMessage()
-                logger.error("thread %s: %s" % (threading.currentThread().getName(), errMsg))
+                logger.error("thread %s: %s" % (threading.current_thread().name, errMsg))
                 traceback.print_exc()
 
     finally:

lib/request/comparison.py

@@ -213,9 +213,9 @@ def _comparison(page, headers, code, getRatioValue, pageLength):
                 seqMatcher.set_seq1(repr(seq1))
                 seqMatcher.set_seq2(repr(seq2))
 
-            if key in kb.cache.comparison:
-                ratio = kb.cache.comparison[key]
-            else:
+            ratio = kb.cache.comparison.get(key) if key else None
+
+            if ratio is None:
                 try:
                     try:
                         ratio = seqMatcher.quick_ratio() if not kb.heavilyDynamic else seqMatcher.ratio()

lib/request/dns.py

@@ -8,13 +8,19 @@
 from __future__ import print_function
 
 import binascii
+import collections
 import os
 import re
 import socket
 import struct
 import threading
 import time
 
+try:
+    from lib.core.settings import MAX_DNS_REQUESTS
+except ImportError:
+    MAX_DNS_REQUESTS = 1000     # fallback so this module stays runnable standalone
+
 class DNSQuery(object):
     """
     >>> DNSQuery(b'|K\\x01 \\x00\\x01\\x00\\x00\\x00\\x00\\x00\\x01\\x03www\\x06google\\x03com\\x00\\x00\\x01\\x00\\x01\\x00\\x00)\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x0c\\x00\\n\\x00\\x08O4|Np!\\x1d\\xb3')._query == b"www.google.com."
@@ -74,7 +80,7 @@ class DNSServer(object):
 
     def __init__(self):
         self._check_localhost()
-        self._requests = []
+        self._requests = collections.deque(maxlen=MAX_DNS_REQUESTS)
         self._lock = threading.Lock()
 
         try:
@@ -140,12 +146,28 @@ def _():
                 self._initialized = True
 
                 while True:
-                    data, addr = self._socket.recvfrom(1024)
-                    _ = DNSQuery(data)
-                    self._socket.sendto(_.response("127.0.0.1"), addr)
-
-                    with self._lock:
-                        self._requests.append(_._query)
+                    try:
+                        data, addr = self._socket.recvfrom(1024)
+                    except KeyboardInterrupt:
+                        raise
+                    except Exception:
+                        break       # socket closed/broken - stop serving (e.g. program exit)
+
+                    # Note: a single malformed packet or a transient send error must NOT kill the
+                    # server thread (otherwise all subsequent DNS exfiltration is silently lost).
+                    # The query is recorded BEFORE responding, so the exfiltrated data is captured
+                    # even if crafting/sending the (fake) resolution response fails.
+                    try:
+                        _ = DNSQuery(data)
+
+                        with self._lock:
+                            self._requests.append(_._query)
+
+                        self._socket.sendto(_.response("127.0.0.1"), addr)
+                    except KeyboardInterrupt:
+                        raise
+                    except Exception:
+                        pass
 
             except KeyboardInterrupt:
                 raise

lib/techniques/dns/use.py

@@ -84,7 +84,10 @@ def dnsUse(payload, expression):
                 _ = conf.dnsServer.pop(prefix, suffix)
 
                 if _:
-                    _ = extractRegexResult(r"%s\.(?P<result>.+)\.%s" % (prefix, suffix), _, re.I)
+                    # Note: non-greedy so a '--dns-domain' label that happens to match the random
+                    # suffix can't make the match run past the real boundary (the boundary alphabet
+                    # excludes hex characters, so it can never under-match into the hex payload)
+                    _ = extractRegexResult(r"%s\.(?P<result>.+?)\.%s" % (prefix, suffix), _, re.I)
                     _ = decodeDbmsHexValue(_)
                     output = (output or "") + _
                     offset += len(_)