diff --git a/data/txt/sha256sums.txt b/data/txt/sha256sums.txt index d92179036a..5a87ad6842 100644 --- a/data/txt/sha256sums.txt +++ b/data/txt/sha256sums.txt @@ -166,34 +166,34 @@ df768bcb9838dc6c46dab9b4a877056cb4742bd6cfaaf438c4a3712c5cc0d264 extra/shutils/ 96463b969312bd4fd29452b5fc739f33e5a73f81fdc1ef80ac27debbe9926e42 lib/controller/controller.py d69e84f1648cdb907f5d2dd454f03874a4613752b07867510145d51d84b3c56f lib/controller/handler.py 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/controller/__init__.py -9da83429449d78797c18bb79ff425aa1eddf5b26b9987d25d042eb0998053675 lib/core/agent.py -905e49d6e030a60f7767c71e0726e0def57c50542210afd9be1cdec122d2d1ce lib/core/bigarray.py -cd22e671c7c96ca8e0e23e1578780e7390dbb50055dabf7bd44f933318c2a9b0 lib/core/common.py +1276ff64ad145157d8c65ce08f3066b6db041d12f7d1eee590c06123c700b18d lib/core/agent.py +c51c33501cc905586a9aaac93b06f2ac6f71628d032a7dc39fd0ef05d7ee3856 lib/core/bigarray.py +5a8dcfc6c43927e4a132d34abf5d75193eaeb3feb0cb58d0ff5bdc059c876ba9 lib/core/common.py 8f1272487e1adfcc8c755a2f56f0c6d21eac5e685a73a9a159482f9dc9142bc5 lib/core/compat.py 742bce10b97034966021ec60c7ac294db4af4fe7893613d63172a02c29f009f8 lib/core/convert.py c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6 lib/core/data.py 6c8d40d6bbab4a60d09eb03324a3352d85df1a741c62044e73701e92172d1d38 lib/core/datatype.py 70fb2528e580b22564899595b0dff6b1bc257c6a99d2022ce3996a3d04e68e4e lib/core/decorators.py 147823c37596bd6a56d677697781f34b8d1d1671d5a2518fbc9468d623c6d07d lib/core/defaults.py -2f44a1bfe6f18aafe64147b99e69aa93cf438c0e7befe59f4e2aee9065c8b7b6 lib/core/dicts.py -12155385c1c4f763c1e8fcb92165015b913620ae1fec1e8de303e4fe841e5a69 lib/core/dump.py +7ce2c09ebcd63d57f7b6751f70f536e2a562230d51181eb24f5024bb6f3d74cc lib/core/dicts.py +a3125c682e891f67255b89d2db891cbaae241f36dd277a272ae6db943111a157 lib/core/dump.py 6b6514202c6ca2d29069176bccf10492927d83e6ede06c9f4b4fcc6164e61856 lib/core/enums.py 5387168e5dfedd94ae22af7bb255f27d6baaca50b24179c6b98f4f325f5cc7b4 lib/core/exception.py 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/core/__init__.py 914a13ee21fd610a6153a37cbe50830fcbd1324c7ebc1e7fc206d5e598b0f7ad lib/core/log.py b5da34bba9ce71ede23349698988939501f5df07be151856007b9b8425a228db lib/core/optiondict.py -c1a9edb894033f1cef0a15a05cca196f816df3465444134af171870dedbe1538 lib/core/option.py +4e7f2ad3d2866093aa195616a0e93de1687406edc0b9038fbfa76bf1c9c174b2 lib/core/option.py ccc4a717e887652b1fcce073d9409d9c59a3b28548c703a9e453d15845f90cd7 lib/core/patch.py 49c0fa7e3814dfda610d665ee02b12df299b28bc0b6773815b4395514ddf8dec lib/core/profiling.py 03db48f02c3d07a047ddb8fe33a757b6238867352d8ddda2a83e4fec09a98d04 lib/core/readlineng.py 9bf174058f15d14e24e94f9aaf42df045119d3617c6c54bd2f3af79b462f331d lib/core/replication.py 0b8c38a01bb01f843d94a6c5f2075ee47520d0c4aa799cecea9c3e2c5a4a23a6 lib/core/revision.py 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c lib/core/session.py -aefc6278c2eee19a2411d19afe85ee78a30214750903b2321d04b2a6a2881b59 lib/core/settings.py -cd5a66deee8963ba8e7e9af3dd36eb5e8127d4d68698811c29e789655f507f82 lib/core/shell.py -bcb5d8090d5e3e0ef2a586ba09ba80eef0c6d51feb0f611ed25299fbb254f725 lib/core/subprocessng.py -70ea3768f1b3062b22d20644df41c86238157ec80dd43da40545c620714273c6 lib/core/target.py -d213562601682fd72603a22f35e5af4e3f41e23bfb143e1584a4fa212a232635 lib/core/testing.py +8a424d4e91d0d5b57502d2699e95acbb12e1378330fbbb4ffa75b507141aec36 lib/core/settings.py +c7804223319e18eb0b8e2cbf0a8b6896d1cefb7b0b1a2e9f1cf826a8a3b56750 lib/core/shell.py +a2e98a94b231432736d6b304fc75525c8b5fdb4768c418387c5b4c1a610dad64 lib/core/subprocessng.py +19f1e3c5e3ba703d28d510cd7a9ab8284d5fbe9df5ce7e77c86e5931571364b7 lib/core/target.py +c1392cda2f202fa3c628f74533c8d9379d1cf7e754ac165e39021bbc2bbc4a22 lib/core/testing.py 95656c44bab1771f4808030dd6a17eae5b129cb1234443f00b19695c7b712b86 lib/core/threads.py b9aacb840310173202f79c2ba125b0243003ee6b44c92eca50424f2bdfc83c02 lib/core/unescaper.py 53e396902cb2546eaa09e77073fcba8be8827ee9ce055cfc899e81b0e6ad4d6d lib/core/update.py @@ -214,7 +214,7 @@ bc61bc944b81a7670884f82231033a6ac703324b34b071c9834886a92e249d0e lib/request/ch d4bb0869b03602a0c8f9e0e0fd217753f14ddadf848fc9f3c65a74d03feb9958 lib/request/comparison.py b9e2db44d265909792f6cc821ff910727b14aa2d5063c74b0f2ea6d40c4f3d9d lib/request/connect.py 8e06682280fce062eef6174351bfebcb6040e19976acff9dc7b3699779783498 lib/request/direct.py -05198477dbdeb6c405059eb21cbbcf9cb6804cc54a0f2a1d11741bfc6cbb7ca2 lib/request/dns.py +a6b37b436838caeb197fea858d0a39fadbff4736256e741b5fcec1f28fcf1ce0 lib/request/dns.py 92c81cc31ff4a396723242058fb2152c9e9745f8412d01ea74480b048a53af6c lib/request/httpshandler.py 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/request/__init__.py 7a0ac2522213e756348fd871a7af74cc963bdc82f9d7ade57be5de42b5bf7cab lib/request/inject.py @@ -587,14 +587,14 @@ a48c411fea864e6bcd6a1c7e1a35094b8cda8d15088fd9e7b0270542ae20daa9 tests/test_com 9c0a0cd0b2d52a53f75c98c60f87a022354b7c3dc4baaf3fe1e272a0af5b7f0a tests/test_dialectdbms.py e40a49cfa73c45b3c3c6d1d1d00738861e270cb7a07b28f5a5356f9c7c800cf2 tests/test_dialect.py 993a2d4d87c4fbaf261663b069629acc95ee4405aa0c42cf5a8f39649fdb0fff tests/test_dicts.py -a38f3257aa218fa706ddb903c181715b2286619c46aea0097b7d365d18c410c5 tests/test_dns_engine.py +7f12466974394312dad3d98651ef8a50d1585bee0f8cd25da0b77b08c2047e46 tests/test_dns_engine.py 703faac01f38224ba85bd0fc398d939ea034f1d7fd641cdc15da4f77ec049443 tests/test_dns_server.py 9cd5841349bc4db818658d12184929a96f7f279eff1f53ad18a54dbefbd6b276 tests/test_dump_jsonl.py 2bbe4b01f79992cfa8884651fc0a28dbd0e3abb0cbea9eb7eadf1f98ca3c3420 tests/test_encoding.py bb6991260a994fcbe79e05febaa34affd5631d02299fbc626820addd5f6ea4f4 tests/test_error_engine.py 8105de9978fe286a29f6b635a58db1e9998d86e8dded54d7efdfb9d52a121094 tests/test_hashdb.py c04e8358fb6df45f69f2f26435c971acde280535bf304e84d30cf2681158c6a7 tests/test_hash.py -205e84827461101a78b2cffaa3de49795a1214e92276fc7fd40f3456657062b9 tests/test_identifiers_output.py +d539d0ae758b5bb91e314ab82ab4fe03d6fb2f8b377d16aefa6d7d1d77a7d5a9 tests/test_identifiers_output.py 5372270b7ed82b62f273c2e9bd1f7ecd8605371e66cd0ad70663762cb08d42f1 tests/test_inference_engine.py caa06fed7323b2bb6d0f2443ce343de94f75bf8ad012c055d5e07741d908ebad tests/test_misc.py 57fa9713a3186020be8bcc3f06399e92bf9ce82ec6d3413c76babe19606bb698 tests/test_openapi_drift.py diff --git a/lib/core/agent.py b/lib/core/agent.py index bc0d1ed018..686eb43bb5 100644 --- a/lib/core/agent.py +++ b/lib/core/agent.py @@ -221,7 +221,7 @@ def payload(self, place=None, parameter=None, value=None, newValue=None, where=N elif BOUNDED_INJECTION_MARKER in paramDict[parameter]: if base64Encoding: retVal = paramString.replace("%s%s" % (_origValue, BOUNDED_INJECTION_MARKER), _newValue) - match = re.search(r"(%s)=([^&]*)" % re.sub(r" \(.+", "", parameter), retVal) + match = re.search(r"(%s)=([^&]*)" % re.escape(re.sub(r" \(.+", "", parameter)), retVal) if match: retVal = retVal.replace(match.group(0), "%s=%s" % (match.group(1), encodeBase64(match.group(2), binary=False, encoding=conf.encoding or UNICODE_ENCODING))) else: @@ -677,6 +677,49 @@ def preprocessField(self, table, field): pass return retVal + @staticmethod + def _collapseFieldDelimiterSpace(query): + """ + Collapses ", " into "," to normalize the column-list delimiter, but ONLY outside + single/double quoted string literals, so a comma-space inside a literal (e.g. in a + WHERE clause: name='John, Jr') is preserved verbatim. The quote/escape handling + mirrors splitFields()/zeroDepthSearch(). + + >>> Agent._collapseFieldDelimiterSpace("SELECT a, b FROM t") + 'SELECT a,b FROM t' + >>> Agent._collapseFieldDelimiterSpace("SELECT a, b FROM t WHERE name='John, Jr'") + "SELECT a,b FROM t WHERE name='John, Jr'" + """ + + retVal = [] + quote = None + index = 0 + length = len(query) + + while index < length: + char = query[index] + if quote: + retVal.append(char) + if char == quote: + if index + 1 < length and query[index + 1] == quote: # escaped quote (e.g. '') + retVal.append(query[index + 1]) + index += 2 + continue + else: + quote = None + elif char in ('"', "'"): + quote = char + retVal.append(char) + elif char == ',' and index + 1 < length and query[index + 1] == ' ': + retVal.append(',') # keep the delimiter, drop the single trailing space + index += 2 + continue + else: + retVal.append(char) + index += 1 + + return "".join(retVal) + def concatQuery(self, query, unpack=True): """ Take in input a query string and return its processed nulled, @@ -705,7 +748,7 @@ def concatQuery(self, query, unpack=True): if unpack: concatenatedQuery = "" - query = query.replace(", ", ',') + query = self._collapseFieldDelimiterSpace(query) fieldsSelectFrom, fieldsSelect, fieldsNoSelect, fieldsSelectTop, fieldsSelectCase, _, fieldsToCastStr, fieldsExists = self.getFields(query) castedFields = self.nullCastConcatFields(fieldsToCastStr) concatenatedQuery = query.replace(fieldsToCastStr, castedFields, 1) @@ -979,7 +1022,9 @@ def limitCondition(self, expression, dump=False): stopLimit = limitRegExp.group(int(limitGroupStop)) elif limitRegExp2: startLimit = 0 - stopLimit = limitRegExp2.group(int(limitGroupStart)) + # Note: query2 (LIMIT without OFFSET) always has exactly one group (the + # count); using limitGroupStart here would IndexError for H2 (groupstart=2) + stopLimit = limitRegExp2.group(1) limitCond = int(stopLimit) > 1 elif Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE): @@ -1281,7 +1326,10 @@ def whereQuery(self, query): if Backend.isDbms(DBMS.ORACLE) and re.search(r"qq ORDER BY \w+\)", query, re.I) is not None: prefix, suffix = re.sub(r"(?i)(qq)( ORDER BY \w+\))", r"\g<1> WHERE %s\g<2>" % conf.dumpWhere, query), "" else: - match = re.search(r" (LIMIT|ORDER).+", query, re.I) + # Note: require a genuine trailing clause (ORDER BY / LIMIT word-bounded), so a + # column/identifier merely starting with "order"/"limit" (e.g. order_id) is not + # mistaken for the suffix and the WHERE is not spliced into the wrong place + match = re.search(r" (ORDER\s+BY\b|LIMIT\b).+", query, re.I) if match: suffix = match.group(0) prefix = query[:-len(suffix)] diff --git a/lib/core/bigarray.py b/lib/core/bigarray.py index fa2976282f..1606bc69a4 100644 --- a/lib/core/bigarray.py +++ b/lib/core/bigarray.py @@ -226,9 +226,19 @@ def _checkcache(self, index): self.cache = None if (self.cache and self.cache.index != index and self.cache.dirty): + old_filename = self.chunks[self.cache.index] filename = self._dump(self.cache.data) self.chunks[self.cache.index] = filename + # Note: remove the now-superseded chunk file (mirrors __getstate__); otherwise every + # cross-chunk dirty flush orphans one temp file on disk and in self.filenames + if isinstance(old_filename, STRING_TYPES): + try: + self._os_remove(old_filename) + self.filenames.discard(old_filename) + except OSError: + pass + if not (self.cache and self.cache.index == index): try: with open(self.chunks[index], "rb") as f: diff --git a/lib/core/common.py b/lib/core/common.py index 938d9432b1..5b04c9589f 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -50,6 +50,7 @@ from lib.core.compat import cmp from lib.core.compat import codecs_open from lib.core.compat import LooseVersion +from lib.core.compat import RecursionError from lib.core.compat import round from lib.core.compat import xrange from lib.core.convert import base64pickle @@ -1459,11 +1460,6 @@ def jsonMinimize(content): True """ - try: - data = json.loads(content) - except (ValueError, TypeError): - return None - lines = [] def _walk(obj, path): @@ -1477,7 +1473,14 @@ def _walk(obj, path): else: lines.append("%s=%s" % (path, obj)) # scalar values kept (boolean detection flips values) - _walk(data, "") + # Note: both json.loads() and the _walk() recursion can hit RecursionError (RuntimeError on + # Python 2) on JSON nested past the interpreter limit; treat that as "not usable" and return + # None so callers fall back to text comparison, rather than crashing the comparison thread + try: + data = json.loads(content) + _walk(data, "") + except (ValueError, TypeError, RecursionError): + return None return "\n".join(sorted(lines)) @@ -1892,7 +1895,9 @@ def expandAsteriskForColumns(expression): the SQL query string (expression) """ - match = re.search(r"(?i)\ASELECT(\s+TOP\s+[\d]+)?\s+\*\s+FROM\s+(([`'\"][^`'\"]+[`'\"]|[\w.]+)+)(\s|\Z)", expression) + # Note: the table-reference group consumes one char / quoted-chunk per repetition ([\w.] not + # [\w.]+) to avoid catastrophic backtracking on a 'SELECT * FROM (' input + match = re.search(r"(?i)\ASELECT(\s+TOP\s+[\d]+)?\s+\*\s+FROM\s+(([`'\"][^`'\"]+[`'\"]|[\w.])+)(\s|\Z)", expression) if match: infoMsg = "you did not provide the fields in your query. " @@ -2957,6 +2962,7 @@ def findLocalPort(ports): retVal = None for port in ports: + s = None try: try: s = socket._orig_socket(socket.AF_INET, socket.SOCK_STREAM) @@ -2968,10 +2974,11 @@ def findLocalPort(ports): except socket.error: pass finally: - try: - s.close() - except socket.error: - pass + if s is not None: + try: + s.close() + except socket.error: + pass return retVal @@ -4233,7 +4240,12 @@ def _(value): # Note: naive approach retVal = content.replace(payload, REFLECTED_VALUE_MARKER) - retVal = retVal.replace(re.sub(r"\A\w+", "", payload), REFLECTED_VALUE_MARKER) + + # Note: guard against an empty needle (payload composed solely of word chars), as + # str.replace("", X) would insert X between every character and explode the page + _stripped = re.sub(r"\A\w+", "", payload) + if _stripped: + retVal = retVal.replace(_stripped, REFLECTED_VALUE_MARKER) if len(parts) > REFLECTED_MAX_REGEX_PARTS: # preventing CPU hogs regex = _("%s%s%s" % (REFLECTED_REPLACEMENT_REGEX.join(parts[:REFLECTED_MAX_REGEX_PARTS // 2]), REFLECTED_REPLACEMENT_REGEX, REFLECTED_REPLACEMENT_REGEX.join(parts[-REFLECTED_MAX_REGEX_PARTS // 2:]))) @@ -4552,14 +4564,18 @@ def safeCSValue(value): 'foobar' >>> safeCSValue('foo\\rbar') '"foo\\rbar"' + >>> safeCSValue('foo"bar') == '"foo""bar"' + True """ retVal = value + # Note: always RFC-4180 escape a value that contains the delimiter, a quote or a newline; an + # earlier "skip if it already begins and ends with a quote" heuristic corrupted cells whose + # content legitimately starts and ends with '"' (e.g. '"a","b"' or a lone '"') if retVal and isinstance(retVal, six.string_types): - if not (retVal[0] == retVal[-1] == '"'): - if any(_ in retVal for _ in (conf.get("csvDel", defaults.csvDel), '"', '\n', '\r')): - retVal = '"%s"' % retVal.replace('"', '""') + if any(_ in retVal for _ in (conf.get("csvDel", defaults.csvDel), '"', '\n', '\r')): + retVal = '"%s"' % retVal.replace('"', '""') return retVal @@ -4591,8 +4607,6 @@ def randomizeParameterValue(value): retVal = value - retVal = re.sub(r"%[0-9a-fA-F]{2}", "", retVal) - def _replace_upper(match): original = match.group() while True: @@ -4614,9 +4628,15 @@ def _replace_digit(match): if candidate != original: return candidate - retVal = re.sub(r"[A-Z]+", _replace_upper, retVal) - retVal = re.sub(r"[a-z]+", _replace_lower, retVal) - retVal = re.sub(r"[0-9]+", _replace_digit, retVal) + def _randomize(segment): + segment = re.sub(r"[A-Z]+", _replace_upper, segment) + segment = re.sub(r"[a-z]+", _replace_lower, segment) + segment = re.sub(r"[0-9]+", _replace_digit, segment) + return segment + + # Note: keep %XX percent-encoded bytes verbatim and randomize only the surrounding characters; + # deleting (or randomizing) the %XX would change the value's decoded content and byte length + retVal = "".join(part if re.match(r"\A%[0-9a-fA-F]{2}\Z", part) else _randomize(part) for part in re.split(r"(%[0-9a-fA-F]{2})", retVal)) if re.match(r"\A[^@]+@.+\.[a-z]+\Z", value): parts = retVal.split('.') @@ -4838,8 +4858,8 @@ def geturl(self): data = "" - for name, value in re.findall(r"['\"]?(\w+)['\"]?\s*:\s*(['\"][^'\"]+)?", match.group(2)): - data += "%s=%s%s" % (name, value, DEFAULT_GET_POST_DELIMITER) + for name, value in re.findall(r"['\"]?(\w+)['\"]?\s*:\s*['\"]?([^'\",}]*)['\"]?", match.group(2)): + data += "%s=%s%s" % (name, value.strip(), DEFAULT_GET_POST_DELIMITER) data = data.rstrip(DEFAULT_GET_POST_DELIMITER) retVal.add((url, HTTPMETHOD.POST, data, conf.cookie, None)) @@ -4904,6 +4924,10 @@ def getHostHeader(url): >>> getHostHeader('http://www.target.com/vuln.php?id=1') 'www.target.com' + >>> getHostHeader('http://[::1]:8080/vuln.php?id=1') + '[::1]:8080' + >>> getHostHeader('http://[::1]/vuln.php?id=1') + '[::1]' """ retVal = url @@ -4911,10 +4935,11 @@ def getHostHeader(url): if url: retVal = _urllib.parse.urlparse(url).netloc - if re.search(r"http(s)?://\[.+\]", url, re.I): - retVal = extractRegexResult(r"http(s)?://\[(?P.+)\]", url) - elif any(retVal.endswith(':%d' % _) for _ in (80, 443)): - retVal = retVal.split(':')[0] + # Note: netloc keeps the IPv6 brackets (and any port), so only the default ports are + # stripped here - mirroring the hostname/IPv4 branch and preserving non-default ports + # (e.g. '[::1]:8080') as required by RFC 7230 + if any(retVal.endswith(':%d' % _) for _ in (80, 443)): + retVal = retVal[:retVal.rfind(':')] if retVal and retVal.count(':') > 1 and not any(_ in retVal for _ in ('[', ']')): retVal = "[%s]" % retVal @@ -5010,7 +5035,14 @@ def incrementCounter(technique): Increments query counter for a given technique """ - kb.counters[technique] = getCounter(technique) + 1 + # Note: the read-modify-write must be atomic since worker threads increment concurrently; + # guard with the shared 'count' lock when available (it is absent in isolated/doctest use) + lock = kb.locks.count if kb.get("locks") else None + if lock is not None: + with lock: + kb.counters[technique] = getCounter(technique) + 1 + else: + kb.counters[technique] = getCounter(technique) + 1 def getCounter(technique): """ @@ -5541,8 +5573,10 @@ def _parseBurpLog(content): key, value = line.split(":", 1) value = value.strip().replace("\r", "").replace("\n", "") - # Note: overriding values with --headers '...' - match = re.search(r"(?i)\b(%s): ([^\n]*)" % re.escape(key), conf.headers or "") + # Note: overriding values with --headers '...'; the lookbehind prevents the key + # from matching the hyphen-suffix tail of a longer header name (e.g. 'Host' + # matching inside 'X-Forwarded-Host'), which would corrupt the outgoing header + match = re.search(r"(?i)(?' + # text would corrupt the INTEGER/REAL-typed columns inferred above + if len(info["values"]) <= i or info["values"][i] is None or info["values"][i] == " ": # NULL + values.append(None) + else: + values.append(getUnicode(info["values"][i])) maxlength = int(info["length"]) blank = " " * (maxlength - getConsoleLength(value)) @@ -708,8 +716,8 @@ def dbTableValues(self, tableValues): elif conf.dumpFormat in (DUMP_FORMAT.CSV, DUMP_FORMAT.HTML, DUMP_FORMAT.JSONL): if conf.dumpFormat == DUMP_FORMAT.HTML: dataToDumpFile(dumpFP, "\n\n\n\n") - elif conf.dumpFormat == DUMP_FORMAT.CSV: - dataToDumpFile(dumpFP, "\n") + # Note: each CSV row already ends with '\n' (above); no extra close-newline, otherwise + # the file ends with a blank line and a later --start/--stop append injects an empty record dumpFP.close() msg = "table '%s.%s' dumped to %s file '%s'" % (db, table, conf.dumpFormat, dumpFileName) diff --git a/lib/core/option.py b/lib/core/option.py index f1a6882936..6644cf08e8 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -438,19 +438,27 @@ def __next__(self): return self.next() def next(self): - try: - line = next(conf.stdinPipe) - except (IOError, OSError, TypeError, UnicodeDecodeError): - line = None - - if line: - match = re.search(r"\b(https?://[^\s'\"]+|[\w.]+\.\w{2,3}[/\w+]*\?[^\s'\"]+)", line, re.I) - if match: - return (match.group(0), conf.method, conf.data, conf.cookie, None) - elif self.__rest: - return self.__rest.pop() - - raise StopIteration() + while True: + try: + line = next(conf.stdinPipe) + except (IOError, OSError, TypeError, UnicodeDecodeError): + line = None + except StopIteration: + line = None + + if line: + match = re.search(r"\b(https?://[^\s'\"]+|[\w.]+\.\w{2,3}[/\w+]*\?[^\s'\"]+)", line, re.I) + if match: + return (match.group(0), conf.method, conf.data, conf.cookie, None) + # Note: a non-empty line that is not a target (blank line, comment, + # non-parameterized URL) must be skipped, not treated as end-of-input + continue + + # end-of-input (or read error): drain any queued targets, then stop + if self.__rest: + return self.__rest.pop() + + raise StopIteration() def add(self, elem): self.__rest.add(elem) @@ -1402,7 +1410,9 @@ def _setHTTPAuthentication(): conf.httpHeaders.append((HTTP_HEADER.AUTHORIZATION, "Bearer %s" % conf.authCred.strip())) return elif authType == AUTH_TYPE.NTLM: - regExp = "^(.*\\\\.*):(.*?)$" + # Note: the DOMAIN\username part is colon-free, so the password group takes the full + # remainder (a greedy first group would otherwise swallow colons inside the password) + regExp = "^([^:]*\\\\[^:]*):(.*)$" errMsg = "HTTP NTLM authentication credentials value must " errMsg += "be in format 'DOMAIN\\username:password'" elif authType == AUTH_TYPE.PKI: @@ -1460,14 +1470,14 @@ def _setHTTPExtraHeaders(): if not headerValue.strip(): continue - if headerValue.count(':') >= 1: + if headerValue.startswith('@'): + checkFile(headerValue[1:]) + kb.headersFile = headerValue[1:] + elif headerValue.count(':') >= 1: header, value = (_.lstrip() for _ in headerValue.split(":", 1)) if header and value: conf.httpHeaders.append((header, value)) - elif headerValue.startswith('@'): - checkFile(headerValue[1:]) - kb.headersFile = headerValue[1:] else: errMsg = "invalid header value: %s. Valid header format is 'name:value'" % repr(headerValue).lstrip('u') raise SqlmapSyntaxException(errMsg) @@ -2520,9 +2530,11 @@ def _setProxyList(): return conf.proxyList = [] - for match in re.finditer(r"(?i)((http[^:]*|socks[^:]*)://)?([\w\-.]+):(\d+)", readCachedFileContent(conf.proxyFile)): - _, type_, address, port = match.groups() - conf.proxyList.append("%s://%s:%s" % (type_ or "http", address, port)) + # Note: preserve an explicit scheme and any 'user:pass@' credentials (entries use the same format + # as --proxy); otherwise a SOCKS proxy is silently downgraded to HTTP and proxy auth is dropped + for match in re.finditer(r"(?i)((http[^:\s]*|socks[^:\s]*)://)?(?:([^:@\s/]+:[^@\s/]*)@)?([\w\-.]+):(\d+)", readCachedFileContent(conf.proxyFile)): + _, type_, cred, address, port = match.groups() + conf.proxyList.append("%s://%s%s:%s" % (type_ or "http", ("%s@" % cred) if cred else "", address, port)) def _setTorProxySettings(): if not conf.tor: @@ -2845,7 +2857,7 @@ def _basicOptionValidation(): raise SqlmapSyntaxException(errMsg) if conf.csrfToken and conf.threads > 1: - errMsg = "option '--csrf-url' is incompatible with option '--threads'" + errMsg = "option '--csrf-token' is incompatible with option '--threads'" raise SqlmapSyntaxException(errMsg) if conf.requestFile and conf.url and conf.url != DUMMY_URL: diff --git a/lib/core/settings.py b/lib/core/settings.py index b34db55c2b..d97547397e 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from thirdparty import six # sqlmap version (...) -VERSION = "1.10.6.143" +VERSION = "1.10.6.147" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -369,7 +369,7 @@ DBMS_DIRECTORY_DICT = dict((getattr(DBMS, _), getattr(DBMS_DIRECTORY_NAME, _)) for _ in dir(DBMS) if not _.startswith("_")) -SUPPORTED_DBMS = set(MSSQL_ALIASES + MYSQL_ALIASES + PGSQL_ALIASES + ORACLE_ALIASES + SQLITE_ALIASES + ACCESS_ALIASES + FIREBIRD_ALIASES + MAXDB_ALIASES + SYBASE_ALIASES + DB2_ALIASES + HSQLDB_ALIASES + H2_ALIASES + INFORMIX_ALIASES + MONETDB_ALIASES + DERBY_ALIASES + VERTICA_ALIASES + MCKOI_ALIASES + PRESTO_ALIASES + ALTIBASE_ALIASES + MIMERSQL_ALIASES + CLICKHOUSE_ALIASES + CRATEDB_ALIASES + CUBRID_ALIASES + CACHE_ALIASES + EXTREMEDB_ALIASES + RAIMA_ALIASES + VIRTUOSO_ALIASES + SNOWFLAKE_ALIASES + SPANNER_ALIASES) +SUPPORTED_DBMS = set(MSSQL_ALIASES + MYSQL_ALIASES + PGSQL_ALIASES + ORACLE_ALIASES + SQLITE_ALIASES + ACCESS_ALIASES + FIREBIRD_ALIASES + MAXDB_ALIASES + SYBASE_ALIASES + DB2_ALIASES + HSQLDB_ALIASES + H2_ALIASES + INFORMIX_ALIASES + MONETDB_ALIASES + DERBY_ALIASES + VERTICA_ALIASES + MCKOI_ALIASES + PRESTO_ALIASES + ALTIBASE_ALIASES + MIMERSQL_ALIASES + CLICKHOUSE_ALIASES + CRATEDB_ALIASES + CUBRID_ALIASES + CACHE_ALIASES + EXTREMEDB_ALIASES + FRONTBASE_ALIASES + RAIMA_ALIASES + VIRTUOSO_ALIASES + SNOWFLAKE_ALIASES + SPANNER_ALIASES) SUPPORTED_OS = ("linux", "windows") DBMS_ALIASES = ((DBMS.MSSQL, MSSQL_ALIASES), (DBMS.MYSQL, MYSQL_ALIASES), (DBMS.PGSQL, PGSQL_ALIASES), (DBMS.ORACLE, ORACLE_ALIASES), (DBMS.SQLITE, SQLITE_ALIASES), (DBMS.ACCESS, ACCESS_ALIASES), (DBMS.FIREBIRD, FIREBIRD_ALIASES), (DBMS.MAXDB, MAXDB_ALIASES), (DBMS.SYBASE, SYBASE_ALIASES), (DBMS.DB2, DB2_ALIASES), (DBMS.HSQLDB, HSQLDB_ALIASES), (DBMS.H2, H2_ALIASES), (DBMS.INFORMIX, INFORMIX_ALIASES), (DBMS.MONETDB, MONETDB_ALIASES), (DBMS.DERBY, DERBY_ALIASES), (DBMS.VERTICA, VERTICA_ALIASES), (DBMS.MCKOI, MCKOI_ALIASES), (DBMS.PRESTO, PRESTO_ALIASES), (DBMS.ALTIBASE, ALTIBASE_ALIASES), (DBMS.MIMERSQL, MIMERSQL_ALIASES), (DBMS.CLICKHOUSE, CLICKHOUSE_ALIASES), (DBMS.CRATEDB, CRATEDB_ALIASES), (DBMS.CUBRID, CUBRID_ALIASES), (DBMS.CACHE, CACHE_ALIASES), (DBMS.EXTREMEDB, EXTREMEDB_ALIASES), (DBMS.FRONTBASE, FRONTBASE_ALIASES), (DBMS.RAIMA, RAIMA_ALIASES), (DBMS.VIRTUOSO, VIRTUOSO_ALIASES), (DBMS.SNOWFLAKE, SNOWFLAKE_ALIASES), (DBMS.SPANNER, SPANNER_ALIASES)) @@ -1048,6 +1048,11 @@ globals()[_] = int(value) except ValueError: pass + elif isinstance(original, float): + try: + globals()[_] = float(value) + except ValueError: + pass elif isinstance(original, (list, tuple)): globals()[_] = [__.strip() for __ in value.split(',')] else: diff --git a/lib/core/shell.py b/lib/core/shell.py index b4ae92ab3e..8fde145640 100644 --- a/lib/core/shell.py +++ b/lib/core/shell.py @@ -40,6 +40,9 @@ def global_matches(self, text): except: readline._readline = None +_atexitRegistered = False +_activeCompletion = None + def readlineAvailable(): """ Check if the readline is available. By default @@ -148,4 +151,13 @@ def autoCompletion(completion=None, os=None, commands=None): readline.parse_and_bind("tab: complete") loadHistory(completion) - atexit.register(saveHistory, completion) + + # Note: readline keeps a single global in-memory history; loadHistory() above swaps it to the + # currently active shell. Registering a fresh atexit handler per call would make every shell + # write that one global buffer to its own path at exit, clobbering the others. Instead register + # a single handler that saves only the shell active at exit. + global _atexitRegistered, _activeCompletion + _activeCompletion = completion + if not _atexitRegistered: + atexit.register(lambda: saveHistory(_activeCompletion)) + _atexitRegistered = True diff --git a/lib/core/subprocessng.py b/lib/core/subprocessng.py index 97bac9bb26..a8c867a5dd 100644 --- a/lib/core/subprocessng.py +++ b/lib/core/subprocessng.py @@ -199,4 +199,9 @@ def send_all(p, data): sent = p.send(data) if not isinstance(sent, int): break + if sent == 0: + # Note: POSIX send() returns 0 when the child's stdin pipe is not currently writable; + # back off briefly instead of busy-spinning at 100% CPU until the child drains it + time.sleep(0.01) + continue data = buffer(data[sent:]) diff --git a/lib/core/target.py b/lib/core/target.py index 74d9d7adbb..b6666807fd 100644 --- a/lib/core/target.py +++ b/lib/core/target.py @@ -412,7 +412,7 @@ def process(match, repl): raise SqlmapGenericException(errMsg) if conf.csrfToken: - if not any(re.search(conf.csrfToken, ' '.join(_), re.I) for _ in (conf.paramDict.get(PLACE.GET, {}), conf.paramDict.get(PLACE.POST, {}), conf.paramDict.get(PLACE.COOKIE, {}))) and not re.search(r"\b%s\b" % conf.csrfToken, conf.data or "") and conf.csrfToken not in set(_[0].lower() for _ in conf.httpHeaders) and conf.csrfToken not in conf.paramDict.get(PLACE.COOKIE, {}) and not all(re.search(conf.csrfToken, _, re.I) for _ in conf.paramDict.get(PLACE.URI, {}).values()): + if not any(re.search(conf.csrfToken, ' '.join(_), re.I) for _ in (conf.paramDict.get(PLACE.GET, {}), conf.paramDict.get(PLACE.POST, {}), conf.paramDict.get(PLACE.COOKIE, {}))) and not re.search(r"\b%s\b" % conf.csrfToken, conf.data or "") and conf.csrfToken not in set(_[0].lower() for _ in conf.httpHeaders) and conf.csrfToken not in conf.paramDict.get(PLACE.COOKIE, {}) and not any(re.search(conf.csrfToken, _, re.I) for _ in conf.paramDict.get(PLACE.URI, {}).values()): errMsg = "anti-CSRF token parameter '%s' not " % conf.csrfToken._original errMsg += "found in provided GET, POST, Cookie or header values" raise SqlmapGenericException(errMsg) @@ -473,7 +473,9 @@ def _resumeHashDBValues(): kb.brute.columns = hashDBRetrieve(HASHDB_KEYS.KB_BRUTE_COLUMNS, True) or kb.brute.columns kb.chars = hashDBRetrieve(HASHDB_KEYS.KB_CHARS, True) or kb.chars kb.dynamicMarkings = hashDBRetrieve(HASHDB_KEYS.KB_DYNAMIC_MARKINGS, True) or kb.dynamicMarkings - kb.xpCmdshellAvailable = hashDBRetrieve(HASHDB_KEYS.KB_XP_CMDSHELL_AVAILABLE) or kb.xpCmdshellAvailable + # Note: the value is stored as text ("True"/"False"); coerce back to bool, otherwise a resumed + # "False" is a truthy string and would wrongly mark xp_cmdshell as available + kb.xpCmdshellAvailable = (hashDBRetrieve(HASHDB_KEYS.KB_XP_CMDSHELL_AVAILABLE) == str(True)) or kb.xpCmdshellAvailable kb.errorChunkLength = hashDBRetrieve(HASHDB_KEYS.KB_ERROR_CHUNK_LENGTH) if isNumPosStrValue(kb.errorChunkLength): @@ -619,7 +621,8 @@ def _createFilesDir(): if not any((conf.fileRead, conf.commonFiles)): return - conf.filePath = paths.SQLMAP_FILES_PATH % conf.hostname + # Note: normalize the hostname consistently with conf.outputPath / conf.dumpPath (see _createDumpDir) + conf.filePath = paths.SQLMAP_FILES_PATH % normalizeUnicode(getUnicode(conf.hostname)) if not os.path.isdir(conf.filePath): try: @@ -641,7 +644,9 @@ def _createDumpDir(): if not conf.dumpTable and not conf.dumpAll and not conf.search: return - conf.dumpPath = safeStringFormat(paths.SQLMAP_DUMP_PATH, conf.hostname) + # Note: normalize the hostname the same way _createTargetDirs() builds conf.outputPath, so a + # non-ASCII (IDN) target keeps its dump under the same per-host tree as the session/log/target.txt + conf.dumpPath = safeStringFormat(paths.SQLMAP_DUMP_PATH, normalizeUnicode(getUnicode(conf.hostname))) if not os.path.isdir(conf.dumpPath): try: diff --git a/lib/core/testing.py b/lib/core/testing.py index 0969c0eab9..a1773789c4 100644 --- a/lib/core/testing.py +++ b/lib/core/testing.py @@ -11,6 +11,7 @@ import os import random import re +import shutil import socket import sqlite3 import subprocess @@ -212,6 +213,7 @@ def _thread(): if "" in cmd: handle, tmp = tempfile.mkstemp() os.close(handle) + cleanups.append(tmp) cmd = cmd.replace("", tmp) os.environ["SQLMAP_UNSAFE_EVAL"] = '1' @@ -237,6 +239,11 @@ def _thread(): except: pass + try: + shutil.rmtree(tmpdir) + except: + pass + return retVal def apiTest(): diff --git a/lib/request/dns.py b/lib/request/dns.py index ffd389a4da..d51c795821 100644 --- a/lib/request/dns.py +++ b/lib/request/dns.py @@ -9,6 +9,7 @@ import binascii import collections +import errno import os import re import socket @@ -34,18 +35,36 @@ def __init__(self, raw): self._query = b"" try: + if len(raw) < 13: + return + type_ = (ord(raw[2:3]) >> 3) & 15 # Opcode bits if type_ == 0: # Standard query i = 12 - j = ord(raw[i:i + 1]) + labels = [] + + while True: + if i >= len(raw): + return - while j != 0: - self._query += raw[i + 1:i + j + 1] + b'.' - i = i + j + 1 j = ord(raw[i:i + 1]) - except TypeError: - pass + + if j == 0: + break + + i += 1 + + if i + j > len(raw): + return + + labels.append(raw[i:i + j]) + i += j + + if labels: + self._query = b".".join(labels) + b'.' + except (TypeError, ValueError, IndexError): + self._query = b"" def response(self, resolution): """ @@ -55,10 +74,15 @@ def response(self, resolution): retVal = b"" if self._query: + end = self._raw[12:].find(b"\x00") + + if end < 0 or len(self._raw) < 12 + end + 5: + return retVal + retVal += self._raw[:2] # Transaction ID retVal += b"\x85\x80" # Flags (Standard query response, No error) retVal += self._raw[4:6] + self._raw[4:6] + b"\x00\x00\x00\x00" # Questions and Answers Counts - retVal += self._raw[12:(12 + self._raw[12:].find(b"\x00") + 5)] # Original Domain Name Query + retVal += self._raw[12:(12 + end + 5)] # Original Domain Name Query retVal += b"\xc0\x0c" # Pointer to domain name retVal += b"\x00\x01" # Type A retVal += b"\x00\x01" # Class IN @@ -141,15 +165,31 @@ def run(self): """ def _(): + def _is_udp_connreset(ex): + return getattr(ex, "winerror", None) == 10054 or getattr(ex, "errno", None) in (errno.ECONNRESET, 10054) + try: self._running = True self._initialized = True + try: + if hasattr(socket, "SIO_UDP_CONNRESET") and hasattr(self._socket, "ioctl"): + # Windows reports ICMP "port unreachable" for UDP as WSAECONNRESET on + # recvfrom(). DNS clients in tests and in the wild can disappear before + # reading our fake response; that must not kill the server thread. + self._socket.ioctl(socket.SIO_UDP_CONNRESET, False) + except Exception: + pass + while True: try: data, addr = self._socket.recvfrom(1024) except KeyboardInterrupt: raise + except socket.error as ex: + if _is_udp_connreset(ex): + continue + break # socket closed/broken - stop serving (e.g. program exit) except Exception: break # socket closed/broken - stop serving (e.g. program exit) @@ -160,10 +200,16 @@ def _(): try: _ = DNSQuery(data) + if not _._query: + continue + with self._lock: self._requests.append(_._query) - self._socket.sendto(_.response("127.0.0.1"), addr) + response = _.response("127.0.0.1") + + if response: + self._socket.sendto(response, addr) except KeyboardInterrupt: raise except Exception: diff --git a/tests/test_dns_engine.py b/tests/test_dns_engine.py index efb2ac8819..bce8bff6a8 100644 --- a/tests/test_dns_engine.py +++ b/tests/test_dns_engine.py @@ -52,8 +52,6 @@ import lib.techniques.dns.use as dnsmod import lib.techniques.dns.test as dnstestmod -DNS_PORT = 5355 - def _build_query(name, tid=b"\x12\x34"): pkt = tid + b"\x01\x00" + b"\x00\x01" + b"\x00\x00" + b"\x00\x00" + b"\x00\x00" for label in name.split("."): @@ -63,15 +61,23 @@ def _build_query(name, tid=b"\x12\x34"): class _HighPortDNSServer(DNSServer): # same logic as the real server (parse/pop/run), just bound high so no root is needed - def __init__(self, port): + def __init__(self, port=0): self._requests = [] self._lock = threading.Lock() self._socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self._socket.bind(("127.0.0.1", port)) + self.port = self._socket.getsockname()[1] self._running = False self._initialized = False + def close(self): + self._running = False + try: + self._socket.close() + except socket.error: + pass + _CONF = {"dnsDomain": "exfil.test", "hexConvert": False, "api": False, "verbose": 0, "forceDns": False} _KB = {"dnsTest": True, "dnsMode": False, "bruteMode": False, "safeCharEncode": False} @@ -81,11 +87,18 @@ class _DnsCase(unittest.TestCase): @classmethod def setUpClass(cls): - cls.server = _HighPortDNSServer(DNS_PORT) + cls.server = _HighPortDNSServer() cls.server.run() while not cls.server._initialized: time.sleep(0.02) + @classmethod + def tearDownClass(cls): + server = getattr(cls, "server", None) + if server is not None: + server.close() + cls.server = None + def setUp(self): self._saved_conf = {k: conf.get(k) for k in _CONF} self._saved_kb = {k: kb.get(k) for k in _KB} @@ -156,7 +169,7 @@ def oracle(payload=None, *args, **kwargs): host = "%s.%s.%s.%s" % (prefix, binascii.hexlify(chunk).decode(), suffix, conf.dnsDomain) c = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) c.settimeout(3) - c.sendto(_build_query(host), ("127.0.0.1", DNS_PORT)) + c.sendto(_build_query(host), ("127.0.0.1", self.server.port)) try: c.recvfrom(512) finally: diff --git a/tests/test_identifiers_output.py b/tests/test_identifiers_output.py index 24ee9d6fd1..dfa27ab27a 100644 --- a/tests/test_identifiers_output.py +++ b/tests/test_identifiers_output.py @@ -60,15 +60,22 @@ class TestSafeCSValue(unittest.TestCase): ("foo,bar", '"foo,bar"'), # contains delimiter -> quoted ('he"y', '"he""y"'), # contains quote -> doubled + wrapped ("a\nb", '"a\nb"'), # contains newline -> quoted + ('"a","b"', '"""a"",""b"""'), # value that begins+ends with a quote must STILL be escaped + ('"', '""""'), # lone quote -> doubled + wrapped ] def test_table(self): for inp, expected in self.CASES: self.assertEqual(safeCSValue(inp), expected, msg="safeCSValue(%r)" % inp) - def test_idempotent_on_already_quoted(self): - once = safeCSValue("a,b") - self.assertEqual(safeCSValue(once), once) # already starts+ends with quote -> unchanged + def test_csv_roundtrip(self): + # the real invariant: a dumped cell must come back as exactly ONE field with its original + # content (a value that begins+ends with '"' must not be emitted verbatim - that splits it) + import csv + for value in ("foobar", "foo,bar", 'he"y', '"a","b"', '"', 'a"b"c'): + line = safeCSValue(value) + fields = next(csv.reader([line])) # csv.reader accepts any iterable of text lines (py2+py3) + self.assertEqual(fields, [value], msg="round-trip failed for %r -> %r" % (value, line)) # (DUMP_REPLACEMENTS markers are covered in test_dicts.py - not duplicated here)