From 1e03b23ccb743ce69700de48943b04bdd6c5a346 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 30 Apr 2019 13:20:31 +0200 Subject: [PATCH] Update (drei) --- lib/core/agent.py | 5 +- lib/core/common.py | 128 +++++++++++++++++------------------- lib/core/convert.py | 39 ++++++----- lib/core/decorators.py | 3 +- lib/core/settings.py | 2 +- lib/core/testing.py | 2 + lib/parse/cmdline.py | 2 +- lib/utils/xrange.py | 14 ++-- sqlmap.py | 3 + tamper/luanginx.py | 2 +- thirdparty/fcrypt/fcrypt.py | 2 +- 11 files changed, 104 insertions(+), 98 deletions(-) diff --git a/lib/core/agent.py b/lib/core/agent.py index de9dc2b38..e1dd364c4 100644 --- a/lib/core/agent.py +++ b/lib/core/agent.py @@ -315,7 +315,10 @@ class Agent(object): ("[HASH_REPLACE]", kb.chars.hash_), ("[GENERIC_SQL_COMMENT]", GENERIC_SQL_COMMENT) ) - payload = reduce(lambda x, y: x.replace(y[0], y[1]), replacements, payload) + + for value in re.findall(r"\[[A-Z_]+\]", payload): + if value in replacements: + payload = payload.replace(value, replacements[value]) for _ in set(re.findall(r"(?i)\[RANDNUM(?:\d+)?\]", payload)): payload = payload.replace(_, str(randomInt())) diff --git a/lib/core/common.py b/lib/core/common.py index 28f8294ab..bcce20c3c 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -950,16 +950,11 @@ def dataToStdout(data, forceOutput=False, bold=False, content_type=None, status= if multiThreadMode: logging._acquireLock() - if isinstance(data, six.text_type): - message = stdoutencode(data) - else: - message = data - try: if conf.get("api"): - sys.stdout.write(clearColors(message), status, content_type) + sys.stdout.write(stdoutencode(clearColors(data)), status, content_type) else: - sys.stdout.write(setColor(message, bold=bold)) + sys.stdout.write(stdoutencode(setColor(data, bold=bold))) sys.stdout.flush() except IOError: @@ -1069,7 +1064,7 @@ def readInput(message, default=None, checkBatch=True, boolean=False): elif default: options = getUnicode(default, UNICODE_ENCODING) else: - options = unicode() + options = six.text_type() dataToStdout("%s%s\n" % (message, options), forceOutput=not kb.wizardMode, bold=True) @@ -1113,9 +1108,8 @@ def randomRange(start=0, stop=1000, seed=None): """ Returns random integer value in given range - >>> random.seed(0) - >>> randomRange(1, 500) - 423 + >>> randomRange(1, 500, seed=0) + 9 """ if seed is not None: @@ -1131,9 +1125,8 @@ def randomInt(length=4, seed=None): """ Returns random integer value with provided number of digits - >>> random.seed(0) - >>> randomInt(6) - 874254 + >>> randomInt(6, seed=0) + 181911 """ if seed is not None: @@ -1149,9 +1142,8 @@ def randomStr(length=4, lowercase=False, alphabet=None, seed=None): """ Returns random string value with provided number of characters - >>> random.seed(0) - >>> randomStr(6) - 'RNvnAv' + >>> randomStr(6, seed=0) + 'aUfWgj' """ if seed is not None: @@ -1174,8 +1166,8 @@ def sanitizeStr(value): """ Sanitizes string value in respect to newline and line-feed characters - >>> sanitizeStr('foo\\n\\rbar') - u'foo bar' + >>> sanitizeStr('foo\\n\\rbar') == 'foo bar' + True """ return getUnicode(value).replace("\n", " ").replace("\r", "") @@ -2096,8 +2088,8 @@ def shellExec(cmd): """ Executes arbitrary shell command - >>> shellExec('echo 1').strip() - '1' + >>> shellExec('echo 1').strip() == b'1' + True """ try: @@ -2420,12 +2412,10 @@ def getUnicode(value, encoding=None, noneToNull=False): """ Return the unicode representation of the supplied value: - >>> getUnicode(u'test') - u'test' - >>> getUnicode('test') - u'test' - >>> getUnicode(1) - u'1' + >>> getUnicode('test') == u'test' + True + >>> getUnicode(1) == u'1' + True """ if noneToNull and value is None: @@ -2436,11 +2426,11 @@ def getUnicode(value, encoding=None, noneToNull=False): elif isinstance(value, six.binary_type): # Heuristics (if encoding not explicitly specified) candidates = filterNone((encoding, kb.get("pageEncoding") if kb.get("originalPage") else None, conf.get("encoding"), UNICODE_ENCODING, sys.getfilesystemencoding())) - if all(_ in value for _ in ('<', '>')): + if all(_ in value for _ in (b'<', b'>')): pass - elif any(_ in value for _ in (":\\", '/', '.')) and '\n' not in value: + elif any(_ in value for _ in (b":\\", b'/', b'.')) and b'\n' not in value: candidates = filterNone((encoding, sys.getfilesystemencoding(), kb.get("pageEncoding") if kb.get("originalPage") else None, UNICODE_ENCODING, conf.get("encoding"))) - elif conf.get("encoding") and '\n' not in value: + elif conf.get("encoding") and b'\n' not in value: candidates = filterNone((encoding, conf.get("encoding"), kb.get("pageEncoding") if kb.get("originalPage") else None, sys.getfilesystemencoding(), UNICODE_ENCODING)) for candidate in candidates: @@ -2708,6 +2698,8 @@ def urldecode(value, encoding=None, unsafe="%%&=;+%s" % CUSTOM_INJECTION_MARK_CH >>> urldecode('AND%201%3E%282%2B3%29%23', convall=True) u'AND 1>(2+3)#' + >>> urldecode('AND%201%3E%282%2B3%29%23', convall=False) + u'AND 1>(2%2B3)#' """ result = value @@ -2722,17 +2714,19 @@ def urldecode(value, encoding=None, unsafe="%%&=;+%s" % CUSTOM_INJECTION_MARK_CH if convall: result = _urllib.parse.unquote_plus(value) if spaceplus else _urllib.parse.unquote(value) else: + result = value + charset = set(string.printable) - set(unsafe) + def _(match): - charset = reduce(lambda x, y: x.replace(y, ""), unsafe, string.printable) char = chr(ord(match.group(1).decode("hex"))) return char if char in charset else match.group(0) - result = value + if spaceplus: result = result.replace('+', ' ') # plus sign has a special meaning in URL encoded data (hence the usage of _urllib.parse.unquote_plus in convall case) + result = re.sub(r"%([0-9a-fA-F]{2})", _, result) - if isinstance(result, str): - result = unicode(result, encoding or UNICODE_ENCODING, "replace") + result = getUnicode(result, encoding or UNICODE_ENCODING) return result @@ -2893,8 +2887,8 @@ def extractTextTagContent(page): """ Returns list containing content from "textual" tags - >>> extractTextTagContent(u'Title
foobar
Link') - [u'Title', u'foobar'] + >>> extractTextTagContent('Title
foobar
Link') + ['Title', 'foobar'] """ page = page or "" @@ -2911,8 +2905,8 @@ def trimAlphaNum(value): """ Trims alpha numeric characters from start and ending of a given value - >>> trimAlphaNum(u'AND 1>(2+3)-- foobar') - u' 1>(2+3)-- ' + >>> trimAlphaNum('AND 1>(2+3)-- foobar') + ' 1>(2+3)-- ' """ while value and value[-1].isalnum(): @@ -3043,8 +3037,8 @@ def filterStringValue(value, charRegex, replacement=""): Returns string value consisting only of chars satisfying supplied regular expression (note: it has to be in form [...]) - >>> filterStringValue(u'wzydeadbeef0123#', r'[0-9a-f]') - u'deadbeef0123' + >>> filterStringValue('wzydeadbeef0123#', r'[0-9a-f]') + 'deadbeef0123' """ retVal = value @@ -3058,8 +3052,8 @@ def filterControlChars(value, replacement=' '): """ Returns string value with control chars being supstituted with replacement character - >>> filterControlChars(u'AND 1>(2+3)\\n--') - u'AND 1>(2+3) --' + >>> filterControlChars('AND 1>(2+3)\\n--') + 'AND 1>(2+3) --' """ return filterStringValue(value, PRINTABLE_CHAR_REGEX, replacement) @@ -3281,8 +3275,8 @@ def arrayizeValue(value): """ Makes a list out of value if it is not already a list or tuple itself - >>> arrayizeValue(u'1') - [u'1'] + >>> arrayizeValue('1') + ['1'] """ if not isListLike(value): @@ -3294,8 +3288,8 @@ def unArrayizeValue(value): """ Makes a value out of iterable if it is a list or tuple itself - >>> unArrayizeValue([u'1']) - u'1' + >>> unArrayizeValue(['1']) + '1' """ if isListLike(value): @@ -3313,8 +3307,8 @@ def flattenValue(value): """ Returns an iterator representing flat representation of a given value - >>> [_ for _ in flattenValue([[u'1'], [[u'2'], u'3']])] - [u'1', u'2', u'3'] + >>> [_ for _ in flattenValue([['1'], [['2'], '3']])] + ['1', '2', '3'] """ for i in iter(value): @@ -3330,7 +3324,7 @@ def isListLike(value): >>> isListLike([1, 2, 3]) True - >>> isListLike(u'2') + >>> isListLike('2') False """ @@ -3373,7 +3367,7 @@ def filterListValue(value, regex): """ if isinstance(value, list) and regex: - retVal = filter(lambda _: re.search(regex, _, re.I), value) + retVal = [_ for _ in value if re.search(regex, _, re.I)] else: retVal = value @@ -3416,10 +3410,10 @@ def decodeIntToUnicode(value): """ Decodes inferenced integer value to an unicode character - >>> decodeIntToUnicode(35) - u'#' - >>> decodeIntToUnicode(64) - u'@' + >>> decodeIntToUnicode(35) == '#' + True + >>> decodeIntToUnicode(64) == '@' + True """ retVal = value @@ -3818,8 +3812,8 @@ def normalizeUnicode(value): # Reference: http://www.peterbe.com/plog/unicode-to-ascii - >>> normalizeUnicode(u'\u0161u\u0107uraj') - 'sucuraj' + >>> normalizeUnicode(u'\u0161u\u0107uraj') == b'sucuraj' + True """ return unicodedata.normalize("NFKD", value).encode("ascii", "ignore") if isinstance(value, six.text_type) else value @@ -4017,10 +4011,10 @@ def safeCSValue(value): # Reference: http://tools.ietf.org/html/rfc4180 - >>> safeCSValue(u'foo, bar') - u'"foo, bar"' - >>> safeCSValue(u'foobar') - u'foobar' + >>> safeCSValue('foo, bar') + '"foo, bar"' + >>> safeCSValue('foobar') + 'foobar' """ retVal = value @@ -4043,7 +4037,7 @@ def filterPairValues(values): retVal = [] if not isNoneValue(values) and hasattr(values, '__iter__'): - retVal = filter(lambda x: isinstance(x, (tuple, list, set)) and len(x) == 2, values) + retVal = [value for value in values if isinstance(value, (tuple, list, set)) and len(value) == 2] return retVal @@ -4469,10 +4463,10 @@ def decodeHexValue(value, raw=False): """ Returns value decoded from DBMS specific hexadecimal representation - >>> decodeHexValue('3132332031') - u'123 1' - >>> decodeHexValue(['0x31', '0x32']) - [u'1', u'2'] + >>> decodeHexValue('3132332031') == u'123 1' + True + >>> decodeHexValue(['0x31', '0x32']) == [u'1', u'2'] + True """ retVal = value @@ -4930,8 +4924,8 @@ def getSafeExString(ex, encoding=None): Safe way how to get the proper exception represtation as a string (Note: errors to be avoided: 1) "%s" % Exception(u'\u0161') and 2) "%s" % str(Exception(u'\u0161')) - >>> getSafeExString(SqlmapBaseException('foobar')) - u'foobar' + >>> getSafeExString(SqlmapBaseException('foobar')) == 'foobar' + True """ retVal = None diff --git a/lib/core/convert.py b/lib/core/convert.py index fe8148d16..ae47315b5 100644 --- a/lib/core/convert.py +++ b/lib/core/convert.py @@ -163,8 +163,10 @@ def htmlunescape(value): retVal = value if value and isinstance(value, six.string_types): - codes = (("<", '<'), (">", '>'), (""", '"'), (" ", ' '), ("&", '&'), ("'", "'")) - retVal = reduce(lambda x, y: x.replace(y[0], y[1]), codes, retVal) + replacements = (("<", '<'), (">", '>'), (""", '"'), (" ", ' '), ("&", '&'), ("'", "'")) + for code, value in replacements: + retVal = retVal.replace(code, value) + try: retVal = re.sub(r"&#x([^ ;]+);", lambda match: unichr(int(match.group(1), 16)), retVal) except ValueError: @@ -177,25 +179,26 @@ def singleTimeWarnMessage(message): # Cross-referenced function sys.stdout.flush() def stdoutencode(data): - retVal = None + retVal = data - try: - retVal = unicodeencode(data or "", sys.stdout.encoding) + if six.PY2: + try: + retVal = unicodeencode(data or "", sys.stdout.encoding) - # Reference: http://bugs.python.org/issue1602 - if IS_WIN: - if '?' in retVal and '?' not in retVal: - warnMsg = "cannot properly display Unicode characters " - warnMsg += "inside Windows OS command prompt " - warnMsg += "(http://bugs.python.org/issue1602). All " - warnMsg += "unhandled occurrences will result in " - warnMsg += "replacement with '?' character. Please, find " - warnMsg += "proper character representation inside " - warnMsg += "corresponding output files. " - singleTimeWarnMessage(warnMsg) + # Reference: http://bugs.python.org/issue1602 + if IS_WIN: + if '?' in retVal and '?' not in retVal: + warnMsg = "cannot properly display Unicode characters " + warnMsg += "inside Windows OS command prompt " + warnMsg += "(http://bugs.python.org/issue1602). All " + warnMsg += "unhandled occurrences will result in " + warnMsg += "replacement with '?' character. Please, find " + warnMsg += "proper character representation inside " + warnMsg += "corresponding output files. " + singleTimeWarnMessage(warnMsg) - except: - retVal = unicodeencode(data or "") + except: + retVal = unicodeencode(data or "") return retVal diff --git a/lib/core/decorators.py b/lib/core/decorators.py index 581d640d5..b98f4f1d3 100644 --- a/lib/core/decorators.py +++ b/lib/core/decorators.py @@ -10,6 +10,7 @@ import hashlib import threading from lib.core.settings import MAX_CACHE_ITEMS +from lib.core.settings import UNICODE_ENCODING from lib.core.datatype import LRUDict from lib.core.threads import getCurrentThreadData @@ -24,7 +25,7 @@ def cachedmethod(f, cache=LRUDict(capacity=MAX_CACHE_ITEMS)): @functools.wraps(f) def _(*args, **kwargs): - key = int(hashlib.md5("|".join(str(_) for _ in (f, args, kwargs))).hexdigest(), 16) & 0x7fffffffffffffff + key = int(hashlib.md5("|".join(str(_) for _ in (f, args, kwargs)).encode(UNICODE_ENCODING)).hexdigest(), 16) & 0x7fffffffffffffff try: with _lock: diff --git a/lib/core/settings.py b/lib/core/settings.py index 05673cfc2..114cd7db4 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -17,7 +17,7 @@ from lib.core.enums import DBMS_DIRECTORY_NAME from lib.core.enums import OS # sqlmap version (...) -VERSION = "1.3.4.47" +VERSION = "1.3.4.48" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/core/testing.py b/lib/core/testing.py index bf41e2ce4..071089386 100644 --- a/lib/core/testing.py +++ b/lib/core/testing.py @@ -73,6 +73,8 @@ def vulnTest(): ("--technique=T --fresh-queries --sql-query='SELECT 1234'", (": '1234'",)), ): output = shellExec("python %s -u http://%s:%d/?id=1 --batch %s" % (os.path.join(os.path.dirname(__file__), "..", "..", "sqlmap.py"), address, port, options)) + output = getUnicode(output) + if not all(check in output for check in checks): retVal = False diff --git a/lib/parse/cmdline.py b/lib/parse/cmdline.py index b20b95aa7..8405c1265 100644 --- a/lib/parse/cmdline.py +++ b/lib/parse/cmdline.py @@ -762,7 +762,7 @@ def cmdLineParser(argv=None): return retVal parser.formatter._format_option_strings = parser.formatter.format_option_strings - parser.formatter.format_option_strings = type(parser.formatter.format_option_strings)(_, parser, type(parser)) + parser.formatter.format_option_strings = type(parser.formatter.format_option_strings)(_, parser) # Dirty hack for making a short option '-hh' option = parser.get_option("--hh") diff --git a/lib/utils/xrange.py b/lib/utils/xrange.py index 086e46c3d..75e435605 100644 --- a/lib/utils/xrange.py +++ b/lib/utils/xrange.py @@ -12,19 +12,19 @@ class xrange(object): Advanced (re)implementation of xrange (supports slice/copy/etc.) Reference: http://code.activestate.com/recipes/521885-a-pythonic-implementation-of-xrange/ - >>> list(xrange(1, 9)) == range(1, 9) + >>> list(xrange(1, 9)) == list(range(1, 9)) True - >>> list(xrange(8, 0, -16)) == range(8, 0, -16) + >>> list(xrange(8, 0, -16)) == list(range(8, 0, -16)) True - >>> list(xrange(0, 8, 16)) == range(0, 8, 16) + >>> list(xrange(0, 8, 16)) == list(range(0, 8, 16)) True - >>> list(xrange(0, 4, 5)) == range(0, 4, 5) + >>> list(xrange(0, 4, 5)) == list(range(0, 4, 5)) True - >>> list(xrange(4, 0, 3)) == range(4, 0, 3) + >>> list(xrange(4, 0, 3)) == list(range(4, 0, 3)) True - >>> list(xrange(0, -3)) == range(0, -3) + >>> list(xrange(0, -3)) == list(range(0, -3)) True - >>> list(xrange(0, 7, 2)) == range(0, 7, 2) + >>> list(xrange(0, 7, 2)) == list(range(0, 7, 2)) True >>> foobar = xrange(1, 10) >>> 7 in foobar diff --git a/sqlmap.py b/sqlmap.py index dbefe65a0..2bdce3260 100755 --- a/sqlmap.py +++ b/sqlmap.py @@ -408,6 +408,9 @@ if __name__ == "__main__": main() except KeyboardInterrupt: pass + except: + if int(os.environ.get("SQLMAP_DREI", 0)): + traceback.print_exc() finally: # Reference: http://stackoverflow.com/questions/1635080/terminate-a-multi-thread-python-program if threading.activeCount() > 1: diff --git a/tamper/luanginx.py b/tamper/luanginx.py index fa1c6853f..f3c97b866 100644 --- a/tamper/luanginx.py +++ b/tamper/luanginx.py @@ -32,6 +32,6 @@ def tamper(payload, **kwargs): hints = kwargs.get("hints", {}) delimiter = kwargs.get("delimiter", DEFAULT_GET_POST_DELIMITER) - hints[HINT.PREPEND] = delimiter.join("%s=" % "".join(random.sample(string.letters + string.digits, 2)) for _ in xrange(500)) + hints[HINT.PREPEND] = delimiter.join("%s=" % "".join(random.sample(string.ascii_letters + string.digits, 2)) for _ in xrange(500)) return payload diff --git a/thirdparty/fcrypt/fcrypt.py b/thirdparty/fcrypt/fcrypt.py index ef601adb8..eb49d023b 100644 --- a/thirdparty/fcrypt/fcrypt.py +++ b/thirdparty/fcrypt/fcrypt.py @@ -573,7 +573,7 @@ In practice, you would read the password using something like the getpass module, and generate the salt randomly: >>> import random, string - >>> saltchars = string.letters + string.digits + './' + >>> saltchars = string.ascii_letters + string.digits + './' >>> salt = random.choice(saltchars) + random.choice(saltchars) Note that other ASCII characters are accepted in the salt, but the