From d465007dfe87cc96a9f160f6d26676167a36540c Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Thu, 2 May 2019 00:45:44 +0200 Subject: [PATCH] More drei updates --- extra/safe2bin/safe2bin.py | 6 ++++-- lib/controller/handler.py | 2 +- lib/core/common.py | 31 ++++++++++++++++++--------- lib/core/settings.py | 2 +- lib/core/testing.py | 2 +- lib/parse/cmdline.py | 7 +++--- lib/parse/headers.py | 4 ++-- lib/request/basic.py | 16 ++++++++------ lib/request/connect.py | 11 +++++----- lib/takeover/abstraction.py | 3 ++- lib/utils/api.py | 3 ++- plugins/generic/custom.py | 3 ++- tamper/hex2char.py | 5 +++-- thirdparty/multipart/multipartpost.py | 4 ++-- 14 files changed, 60 insertions(+), 39 deletions(-) diff --git a/extra/safe2bin/safe2bin.py b/extra/safe2bin/safe2bin.py index eb0f360cd..9efc81bda 100644 --- a/extra/safe2bin/safe2bin.py +++ b/extra/safe2bin/safe2bin.py @@ -21,8 +21,10 @@ from optparse import OptionParser if sys.version_info >= (3, 0): xrange = range text_type = str + string_types = (str,) else: text_type = unicode + string_types = (basestring,) # Regex used for recognition of hex encoded characters HEX_ENCODED_CHAR_REGEX = r"(?P\\x[0-9A-Fa-f]{2})" @@ -54,7 +56,7 @@ def safecharencode(value): retVal = value - if isinstance(value, basestring): + if isinstance(value, string_types): if any(_ not in SAFE_CHARS for _ in value): retVal = retVal.replace(HEX_ENCODED_PREFIX, HEX_ENCODED_PREFIX_MARKER) retVal = retVal.replace('\\', SLASH_MARKER) @@ -78,7 +80,7 @@ def safechardecode(value, binary=False): """ retVal = value - if isinstance(value, basestring): + if isinstance(value, string_types): retVal = retVal.replace('\\\\', SLASH_MARKER) while True: diff --git a/lib/controller/handler.py b/lib/controller/handler.py index 15f1cca8c..a502c053d 100644 --- a/lib/controller/handler.py +++ b/lib/controller/handler.py @@ -75,7 +75,7 @@ def setHandler(): (DBMS.INFORMIX, INFORMIX_ALIASES, InformixMap, InformixConn), ] - _ = max(_ if (conf.get("dbms") or Backend.getIdentifiedDbms() or kb.heuristicExtendedDbms or "").lower() in _[1] else None for _ in items) + _ = max(_ if (conf.get("dbms") or Backend.getIdentifiedDbms() or kb.heuristicExtendedDbms or "").lower() in _[1] else "" for _ in items) or None if _: items.remove(_) items.insert(0, _) diff --git a/lib/core/common.py b/lib/core/common.py index 8d4738677..49b0df0a8 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -177,6 +177,7 @@ from thirdparty.magic import magic from thirdparty.odict import OrderedDict from thirdparty.six.moves import configparser as _configparser from thirdparty.six.moves import http_client as _http_client +from thirdparty.six.moves import input as _input from thirdparty.six.moves import urllib as _urllib from thirdparty.termcolor.termcolor import colored @@ -942,8 +943,6 @@ def dataToStdout(data, forceOutput=False, bold=False, content_type=None, status= Writes text to the stdout (console) stream """ - message = "" - if not kb.get("threadException"): if forceOutput or not (getCurrentThreadData().disableStdOut or kb.get("wizardMode")): multiThreadMode = isMultiThreadMode() @@ -1082,7 +1081,7 @@ def readInput(message, default=None, checkBatch=True, boolean=False): dataToStdout("%s" % message, forceOutput=not kb.wizardMode, bold=True) kb.prependFlag = False - retVal = raw_input().strip() or default + retVal = _input().strip() or default retVal = getUnicode(retVal, encoding=sys.stdin.encoding) if retVal else retVal except: try: @@ -2452,11 +2451,21 @@ def getUnicode(value, encoding=None, noneToNull=False): except UnicodeDecodeError: return six.text_type(str(value), errors="ignore") # encoding ignored for non-basestring instances +def decodeHex(value): + """ + Returns byte representation of provided hexadecimal value + + >>> decodeHex("313233") == b"123" + True + """ + + return bytes.fromhex(value) if hasattr(bytes, "fromhex") else value.decode("hex") + def getBytes(value, encoding=UNICODE_ENCODING, errors="strict"): """ Returns byte representation of provided Unicode value - >>> getBytes(getUnicode("foo\x01\x83\xffbar")) == "foo\x01\x83\xffbar" + >>> getBytes(getUnicode("foo\x01\x83\xffbar")) == b"foo\x01\x83\xffbar" True """ @@ -2468,11 +2477,10 @@ def getBytes(value, encoding=UNICODE_ENCODING, errors="strict"): value = value.replace(unichr(char), "%s%02x" % (SAFE_HEX_MARKER, char - 0xF0000)) retVal = value.encode(encoding, errors) - - retVal = re.sub(r"%s([0-9a-f]{2})" % SAFE_HEX_MARKER, lambda _: _.group(1).decode("hex"), retVal) + retVal = re.sub(r"%s([0-9a-f]{2})" % SAFE_HEX_MARKER, lambda _: decodeHex(_.group(1)), retVal) else: retVal = value.encode(encoding, errors) - retVal = re.sub(r"\\x([0-9a-f]{2})", lambda _: _.group(1).decode("hex"), retVal) + retVal = re.sub(b"\\\\x([0-9a-f]{2})", lambda _: decodeHex(_.group(1)), retVal) return retVal @@ -2876,6 +2884,9 @@ def extractRegexResult(regex, content, flags=0): retVal = None if regex and content and "?P" in regex: + if isinstance(content, six.binary_type) and isinstance(regex, six.text_type): + regex = getBytes(regex) + match = re.search(regex, content, flags) if match: @@ -3812,11 +3823,11 @@ def normalizeUnicode(value): # Reference: http://www.peterbe.com/plog/unicode-to-ascii - >>> normalizeUnicode(u'\u0161u\u0107uraj') == b'sucuraj' + >>> normalizeUnicode(u'\u0161u\u0107uraj') == u'sucuraj' True """ - return unicodedata.normalize("NFKD", value).encode("ascii", "ignore") if isinstance(value, six.text_type) else value + return getUnicode(unicodedata.normalize("NFKD", value).encode("ascii", "ignore")) if isinstance(value, six.text_type) else value def safeSQLIdentificatorNaming(name, isTable=False): """ @@ -4656,7 +4667,7 @@ def getRequestHeader(request, name): if request and request.headers and name: _ = name.upper() - retVal = max(value if _ == key.upper() else None for key, value in request.header_items()) + retVal = max(value if _ == key.upper() else "" for key, value in request.header_items()) or None return retVal diff --git a/lib/core/settings.py b/lib/core/settings.py index bf95652e9..943a5124c 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -17,7 +17,7 @@ from lib.core.enums import DBMS_DIRECTORY_NAME from lib.core.enums import OS # sqlmap version (...) -VERSION = "1.3.4.51" +VERSION = "1.3.5.0" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/core/testing.py b/lib/core/testing.py index 071089386..b30e4a2a0 100644 --- a/lib/core/testing.py +++ b/lib/core/testing.py @@ -72,7 +72,7 @@ def vulnTest(): ("--technique=B --hex --fresh-queries --threads=4 --sql-query='SELECT 987654321'", ("length of query output", ": '987654321'",)), ("--technique=T --fresh-queries --sql-query='SELECT 1234'", (": '1234'",)), ): - output = shellExec("python %s -u http://%s:%d/?id=1 --batch %s" % (os.path.join(os.path.dirname(__file__), "..", "..", "sqlmap.py"), address, port, options)) + output = shellExec("%s %s -u http://%s:%d/?id=1 --batch %s" % (sys.executable, os.path.join(os.path.dirname(__file__), "..", "..", "sqlmap.py"), address, port, options)) output = getUnicode(output) if not all(check in output for check in checks): diff --git a/lib/parse/cmdline.py b/lib/parse/cmdline.py index 51209a098..bb6e9ef84 100644 --- a/lib/parse/cmdline.py +++ b/lib/parse/cmdline.py @@ -40,6 +40,7 @@ from lib.core.shell import autoCompletion from lib.core.shell import clearHistory from lib.core.shell import loadHistory from lib.core.shell import saveHistory +from thirdparty.six.moves import input as _input def cmdLineParser(argv=None): """ @@ -54,7 +55,7 @@ def cmdLineParser(argv=None): # Reference: https://stackoverflow.com/a/4012683 (Note: previously used "...sys.getfilesystemencoding() or UNICODE_ENCODING") _ = getUnicode(os.path.basename(argv[0]), encoding=sys.stdin.encoding) - usage = "%s%s [options]" % ("python " if not IS_WIN else "", "\"%s\"" % _ if " " in _ else _) + usage = "%s%s [options]" % ("%s " % os.path.basename(sys.executable) if not IS_WIN else "", "\"%s\"" % _ if " " in _ else _) parser = OptionParser(usage=usage) try: @@ -809,7 +810,7 @@ def cmdLineParser(argv=None): command = None try: - command = raw_input("sqlmap-shell> ").strip() + command = _input("sqlmap-shell> ").strip() command = getUnicode(command, encoding=sys.stdin.encoding) except (KeyboardInterrupt, EOFError): print() @@ -930,7 +931,7 @@ def cmdLineParser(argv=None): # Protection against Windows dummy double clicking if IS_WIN: dataToStdout("\nPress Enter to continue...") - raw_input() + _input() raise debugMsg = "parsing command line" diff --git a/lib/parse/headers.py b/lib/parse/headers.py index 2eb81973f..b16dea2ae 100644 --- a/lib/parse/headers.py +++ b/lib/parse/headers.py @@ -5,13 +5,13 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/) See the file 'LICENSE' for copying permission """ -import itertools import os from lib.core.common import parseXmlFile from lib.core.data import kb from lib.core.data import paths from lib.parse.handler import FingerprintHandler +from thirdparty.six.moves import filter as _filter def headersParser(headers): """ @@ -30,7 +30,7 @@ def headersParser(headers): "x-powered-by": os.path.join(paths.SQLMAP_XML_BANNER_PATH, "x-powered-by.xml"), } - for header in itertools.ifilter(lambda _: _ in kb.headerPaths, headers): + for header in _filter(lambda _: _ in kb.headerPaths, headers): value = headers[header] xmlfile = kb.headerPaths[header] handler = FingerprintHandler(value, kb.headersFp) diff --git a/lib/request/basic.py b/lib/request/basic.py index cd2a66cbb..1191bd53e 100644 --- a/lib/request/basic.py +++ b/lib/request/basic.py @@ -14,6 +14,7 @@ import struct import zlib from lib.core.common import Backend +from lib.core.common import decodeHex from lib.core.common import extractErrorMessage from lib.core.common import extractRegexResult from lib.core.common import filterNone @@ -156,6 +157,9 @@ def checkCharEncoding(encoding, warn=True): 'utf8' """ + if isinstance(encoding, six.binary_type): + encoding = getUnicode(encoding) + if isListLike(encoding): encoding = unArrayizeValue(encoding) @@ -316,16 +320,16 @@ def decodePage(page, contentEncoding, contentType): # can't do for all responses because we need to support binary files too if isinstance(page, six.binary_type) and "text/" in contentType: # e.g. Ãëàâà - if "&#" in page: - page = re.sub(r"&#x([0-9a-f]{1,2});", lambda _: (_.group(1) if len(_.group(1)) == 2 else "0%s" % _.group(1)).decode("hex"), page) - page = re.sub(r"&#(\d{1,3});", lambda _: chr(int(_.group(1))) if int(_.group(1)) < 256 else _.group(0), page) + if b"&#" in page: + page = re.sub(b"&#x([0-9a-f]{1,2});", lambda _: decodeHex(_.group(1) if len(_.group(1)) == 2 else "0%s" % _.group(1)), page) + page = re.sub(b"&#(\d{1,3});", lambda _: chr(int(_.group(1))) if int(_.group(1)) < 256 else _.group(0), page) # e.g. %20%28%29 - if "%" in page: - page = re.sub(r"%([0-9a-fA-F]{2})", lambda _: _.group(1).decode("hex"), page) + if b"%" in page: + page = re.sub(b"%([0-9a-fA-F]{2})", lambda _: decodeHex(_.group(1)), page) # e.g. & - page = re.sub(r"&([^;]+);", lambda _: chr(htmlEntities[_.group(1)]) if htmlEntities.get(_.group(1), 256) < 256 else _.group(0), page) + page = re.sub(b"&([^;]+);", lambda _: chr(htmlEntities[_.group(1)]) if htmlEntities.get(_.group(1), 256) < 256 else _.group(0), page) kb.pageEncoding = kb.pageEncoding or checkCharEncoding(getHeuristicCharEncoding(page)) diff --git a/lib/request/connect.py b/lib/request/connect.py index fe87fd101..0e8a59218 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -193,7 +193,7 @@ class Connect(object): @staticmethod def _connReadProxy(conn): - retVal = "" + retVal = b"" if not kb.dnsMode and conn: headers = conn.info() @@ -413,13 +413,12 @@ class Connect(object): if auxHeaders: headers = forgeHeaders(auxHeaders, headers) - for key, value in headers.items(): + for key, value in list(headers.items()): del headers[key] for char in (r"\r", r"\n"): value = re.sub(r"(%s)([^ \t])" % char, r"\g<1>\t\g<2>", value) headers[getBytes(key)] = getBytes(value.strip("\r\n")) - url = getBytes(url) post = getBytes(post) if websocket_: @@ -797,7 +796,7 @@ class Connect(object): responseMsg += "[#%d] (%s %s):\r\n" % (threadData.lastRequestUID, code, status) if responseHeaders: - logHeaders = getUnicode("".join(responseHeaders.headers).strip()) + logHeaders = getUnicode("".join(responseHeaders.headers).strip() if six.PY2 else responseHeaders.__bytes__()) logHTTPTraffic(requestMsg, "%s%s\r\n\r\n%s" % (responseMsg, logHeaders, (page or "")[:MAX_CONNECTION_CHUNK_SIZE]), start, time.time()) @@ -851,7 +850,7 @@ class Connect(object): if conf.httpHeaders: headers = OrderedDict(conf.httpHeaders) - contentType = max(headers[_] if _.upper() == HTTP_HEADER.CONTENT_TYPE.upper() else None for _ in headers) + contentType = max(headers[_] if _.upper() == HTTP_HEADER.CONTENT_TYPE.upper() else "" for _ in headers) or None if (kb.postHint or conf.skipUrlEncode) and postUrlEncode: postUrlEncode = False @@ -1266,7 +1265,7 @@ class Connect(object): warnMsg += "10 or more)" logger.critical(warnMsg) - if conf.safeFreq > 0: + if (conf.safeFreq or 0) > 0: kb.queryCounter += 1 if kb.queryCounter % conf.safeFreq == 0: if conf.safeUrl: diff --git a/lib/takeover/abstraction.py b/lib/takeover/abstraction.py index feef80fba..7b5bc2f54 100644 --- a/lib/takeover/abstraction.py +++ b/lib/takeover/abstraction.py @@ -28,6 +28,7 @@ from lib.request import inject from lib.takeover.udf import UDF from lib.takeover.web import Web from lib.takeover.xp_cmdshell import XP_cmdshell +from thirdparty.six.moves import input as _input class Abstraction(Web, UDF, XP_cmdshell): """ @@ -139,7 +140,7 @@ class Abstraction(Web, UDF, XP_cmdshell): command = None try: - command = raw_input("os-shell> ") + command = _input("os-shell> ") command = getUnicode(command, encoding=sys.stdin.encoding) except KeyboardInterrupt: print() diff --git a/lib/utils/api.py b/lib/utils/api.py index e42e18fd4..712f9e1f2 100644 --- a/lib/utils/api.py +++ b/lib/utils/api.py @@ -57,6 +57,7 @@ from thirdparty.bottle.bottle import response from thirdparty.bottle.bottle import run from thirdparty.bottle.bottle import server_names from thirdparty.six.moves import http_client as _http_client +from thirdparty.six.moves import input as _input from thirdparty.six.moves import urllib as _urllib # Global data storage @@ -762,7 +763,7 @@ def client(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, username=Non while True: try: - command = raw_input("api%s> " % (" (%s)" % taskid if taskid else "")).strip() + command = _input("api%s> " % (" (%s)" % taskid if taskid else "")).strip() command = re.sub(r"\A(\w+)", lambda match: match.group(1).lower(), command) except (EOFError, KeyboardInterrupt): print() diff --git a/plugins/generic/custom.py b/plugins/generic/custom.py index 5c10d0241..2c6ed4ea4 100644 --- a/plugins/generic/custom.py +++ b/plugins/generic/custom.py @@ -24,6 +24,7 @@ from lib.core.settings import NULL from lib.core.settings import PARAMETER_SPLITTING_REGEX from lib.core.shell import autoCompletion from lib.request import inject +from thirdparty.six.moves import input as _input class Custom: """ @@ -88,7 +89,7 @@ class Custom: query = None try: - query = raw_input("sql-shell> ") + query = _input("sql-shell> ") query = getUnicode(query, encoding=sys.stdin.encoding) query = query.strip("; ") except KeyboardInterrupt: diff --git a/tamper/hex2char.py b/tamper/hex2char.py index 562c910e6..37d9fd3c8 100644 --- a/tamper/hex2char.py +++ b/tamper/hex2char.py @@ -7,6 +7,7 @@ See the file 'LICENSE' for copying permission import re +from lib.core.common import decodeHex from lib.core.enums import PRIORITY __priority__ = PRIORITY.NORMAL @@ -36,9 +37,9 @@ def tamper(payload, **kwargs): if payload: for match in re.finditer(r"\b0x([0-9a-f]+)\b", retVal): if len(match.group(1)) > 2: - result = "CONCAT(%s)" % ','.join("CHAR(%d)" % ord(_) for _ in match.group(1).decode("hex")) + result = "CONCAT(%s)" % ','.join("CHAR(%d)" % ord(_) for _ in decodeHex(match.group(1))) else: - result = "CHAR(%d)" % ord(match.group(1).decode("hex")) + result = "CHAR(%d)" % ord(decodeHex(match.group(1))) retVal = retVal.replace(match.group(0), result) return retVal diff --git a/thirdparty/multipart/multipartpost.py b/thirdparty/multipart/multipartpost.py index 4355638af..52f75fa47 100644 --- a/thirdparty/multipart/multipartpost.py +++ b/thirdparty/multipart/multipartpost.py @@ -43,7 +43,7 @@ class MultipartPostHandler(_urllib.request.BaseHandler): handler_order = _urllib.request.HTTPHandler.handler_order - 10 # needs to run first def http_request(self, request): - data = request.get_data() + data = request.data if isinstance(data, dict): v_files = [] @@ -68,7 +68,7 @@ class MultipartPostHandler(_urllib.request.BaseHandler): # print "Replacing %s with %s" % (request.get_header("content-type"), "multipart/form-data") request.add_unredirected_header("Content-Type", contenttype) - request.add_data(data) + request.data = data return request def multipart_encode(vars, files, boundary=None, buf=None):