From 8cd257c893ea0474bb8b66c81a24989c4279fe9d Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 4 Mar 2019 15:24:12 +0100 Subject: [PATCH] Implementation for #3505 --- lib/core/enums.py | 1 + lib/core/option.py | 77 ++++++++++++++++++++++++++++++++++++++++++ lib/core/settings.py | 2 +- lib/parse/cmdline.py | 3 ++ lib/request/connect.py | 8 +++++ txt/checksum.md5 | 10 +++--- 6 files changed, 95 insertions(+), 6 deletions(-) diff --git a/lib/core/enums.py b/lib/core/enums.py index 0faa9404e..c572d10d6 100644 --- a/lib/core/enums.py +++ b/lib/core/enums.py @@ -377,6 +377,7 @@ class MKSTEMP_PREFIX: COOKIE_JAR = "sqlmapcookiejar-" BIG_ARRAY = "sqlmapbigarray-" SPECIFIC_RESPONSE = "sqlmapresponse-" + PREPROCESS = "sqlmappreprocess-" class TIMEOUT_STATE: NORMAL = 0 diff --git a/lib/core/option.py b/lib/core/option.py index 11570f129..8c11befa0 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -76,6 +76,7 @@ from lib.core.enums import CUSTOM_LOGGING from lib.core.enums import DUMP_FORMAT from lib.core.enums import HTTP_HEADER from lib.core.enums import HTTPMETHOD +from lib.core.enums import MKSTEMP_PREFIX from lib.core.enums import MOBILES from lib.core.enums import OPTION_TYPE from lib.core.enums import PAYLOAD @@ -825,6 +826,80 @@ def _setTamperingFunctions(): for _, function in priorities: kb.tamperFunctions.append(function) +def _setPreprocessFunctions(): + """ + Loads preprocess functions from given script(s) + """ + + if conf.preprocess: + for script in re.split(PARAMETER_SPLITTING_REGEX, conf.preprocess): + found = False + + script = script.strip().encode(sys.getfilesystemencoding() or UNICODE_ENCODING) + + try: + if not script: + continue + + if not os.path.exists(script): + errMsg = "preprocess script '%s' does not exist" % script + raise SqlmapFilePathException(errMsg) + + elif not script.endswith(".py"): + errMsg = "preprocess script '%s' should have an extension '.py'" % script + raise SqlmapSyntaxException(errMsg) + except UnicodeDecodeError: + errMsg = "invalid character provided in option '--preprocess'" + raise SqlmapSyntaxException(errMsg) + + dirname, filename = os.path.split(script) + dirname = os.path.abspath(dirname) + + infoMsg = "loading preprocess module '%s'" % filename[:-3] + logger.info(infoMsg) + + if not os.path.exists(os.path.join(dirname, "__init__.py")): + errMsg = "make sure that there is an empty file '__init__.py' " + errMsg += "inside of preprocess scripts directory '%s'" % dirname + raise SqlmapGenericException(errMsg) + + if dirname not in sys.path: + sys.path.insert(0, dirname) + + try: + module = __import__(filename[:-3].encode(sys.getfilesystemencoding() or UNICODE_ENCODING)) + except Exception as ex: + raise SqlmapSyntaxException("cannot import preprocess module '%s' (%s)" % (filename[:-3], getSafeExString(ex))) + + for name, function in inspect.getmembers(module, inspect.isfunction): + if name == "preprocess" and inspect.getargspec(function).args and all(_ in inspect.getargspec(function).args for _ in ("page", "headers", "code")): + found = True + + kb.preprocessFunctions.append(function) + function.func_name = module.__name__ + + break + + if not found: + errMsg = "missing function 'preprocess(page, headers=None, code=None)' " + errMsg += "in preprocess script '%s'" % script + raise SqlmapGenericException(errMsg) + else: + try: + _, _, _ = function("", {}, None) + except: + handle, filename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.PREPROCESS, suffix=".py") + os.close(handle) + + open(filename, "w+b").write("#!/usr/bin/env\n\ndef preprocess(page, headers=None, code=None):\n return page, headers, code\n") + open(os.path.join(os.path.dirname(filename), "__init__.py"), "w+b").write("pass") + + errMsg = "function 'preprocess(page, headers=None, code=None)' " + errMsg += "in preprocess script '%s' " % script + errMsg += "should return a tuple '(page, headers, code)' " + errMsg += "(Note: find template script at '%s')" % filename + raise SqlmapGenericException(errMsg) + def _setWafFunctions(): """ Loads WAF/IPS detecting functions from script(s) @@ -1937,6 +2012,7 @@ def _setKnowledgeBaseAttributes(flushAll=True): kb.headerPaths = {} kb.keywords = set(getFileItems(paths.SQL_KEYWORDS)) kb.passwordMgr = None + kb.preprocessFunctions = [] kb.skipVulnHost = None kb.tamperFunctions = [] kb.targets = oset() @@ -2549,6 +2625,7 @@ def init(): _setMultipleTargets() _listTamperingFunctions() _setTamperingFunctions() + _setPreprocessFunctions() _setWafFunctions() _setTrafficOutputFP() _setupHTTPCollector() diff --git a/lib/core/settings.py b/lib/core/settings.py index 656119f4b..e3144e47f 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -19,7 +19,7 @@ from lib.core.enums import DBMS_DIRECTORY_NAME from lib.core.enums import OS # sqlmap version (...) -VERSION = "1.3.3.1" +VERSION = "1.3.3.2" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/parse/cmdline.py b/lib/parse/cmdline.py index 388634e0e..7cec15c14 100644 --- a/lib/parse/cmdline.py +++ b/lib/parse/cmdline.py @@ -595,6 +595,9 @@ def cmdLineParser(argv=None): general.add_option("--parse-errors", dest="parseErrors", action="store_true", help="Parse and display DBMS error messages from responses") + general.add_option("--preprocess", dest="preprocess", + help="Use given script(s) for preprocessing of response data") + general.add_option("--repair", dest="repair", action="store_true", help="Redump entries having unknown character marker (%s)" % INFERENCE_UNKNOWN_CHAR) diff --git a/lib/request/connect.py b/lib/request/connect.py index 9f9751fa5..f147fe8c2 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -746,6 +746,14 @@ class Connect(object): page = getUnicode(page) socket.setdefaulttimeout(conf.timeout) + for function in kb.preprocessFunctions: + try: + page, responseHeaders, code = function(page, responseHeaders, code) + except Exception as ex: + errMsg = "error occurred while running preprocess " + errMsg += "function '%s' ('%s')" % (function.func_name, getSafeExString(ex)) + raise SqlmapGenericException(errMsg) + processResponse(page, responseHeaders, status) if conn and getattr(conn, "redurl", None): diff --git a/txt/checksum.md5 b/txt/checksum.md5 index 1e47ae13c..d988eb908 100644 --- a/txt/checksum.md5 +++ b/txt/checksum.md5 @@ -38,19 +38,19 @@ abcb1121eb56d3401839d14e8ed06b6e lib/core/data.py 5f4680b769ae07f22157bd832c97cf8f lib/core/defaults.py 9dfc69ba47209a4ceca494dde9ee8183 lib/core/dicts.py 4ba141124699fd7a763dea82f17fe523 lib/core/dump.py -0a49eaf3f940382464ee08c03c9891a8 lib/core/enums.py +1226fed38d1175aee8907e31ddf0cab2 lib/core/enums.py 84ef8f32e4582fcc294dc14e1997131d lib/core/exception.py fb6be55d21a70765e35549af2484f762 lib/core/__init__.py 18c896b157b03af716542e5fe9233ef9 lib/core/log.py 151136142a14bee82cb02a9ca64c741d lib/core/optiondict.py -7f9d7b65f2278e5d233008a8bdd22c87 lib/core/option.py +5d21cede75bd8043a0b9f2605047ea07 lib/core/option.py fe370021c6bc99daf44b2bfc0d1effb3 lib/core/patch.py 4b12aa67fbf6c973d12e54cf9cb54ea0 lib/core/profiling.py d5ef43fe3cdd6c2602d7db45651f9ceb lib/core/readlineng.py 7d8a22c582ad201f65b73225e4456170 lib/core/replication.py 3179d34f371e0295dd4604568fb30bcd lib/core/revision.py d6269c55789f78cf707e09a0f5b45443 lib/core/session.py -9dbce20566a1964f650b8986885ae370 lib/core/settings.py +177d5fddb467b206530dacbc8618928d lib/core/settings.py 4483b4a5b601d8f1c4281071dff21ecc lib/core/shell.py 10fd19b0716ed261e6d04f311f6f527c lib/core/subprocessng.py 43772ea73e9e3d446f782af591cb4eda lib/core/target.py @@ -61,7 +61,7 @@ d6269c55789f78cf707e09a0f5b45443 lib/core/session.py 5b3f08208be0579356f78ce5805d37b2 lib/core/wordlist.py fb6be55d21a70765e35549af2484f762 lib/__init__.py 4881480d0c1778053908904e04570dc3 lib/parse/banner.py -b23a0940d21347975a783c63fe671974 lib/parse/cmdline.py +fafa321d2bbfc60410a131f68d5203ea lib/parse/cmdline.py 06ccbccb63255c8f1c35950a4c8a6f6b lib/parse/configfile.py d34df646508c2dceb25205e1316673d1 lib/parse/handler.py 43deb2400e269e602e916efaec7c0903 lib/parse/headers.py @@ -72,7 +72,7 @@ adcecd2d6a8667b22872a563eb83eac0 lib/parse/payloads.py e4ea70bcd461f5176867dcd89d372386 lib/request/basicauthhandler.py b23163d485e0dbc038cbf1ba80be11da lib/request/basic.py fc25d951217077fe655ed2a3a81552ae lib/request/comparison.py -2b58b3ed5f3aff7025e02bb1427bc637 lib/request/connect.py +3925fef5710ac4e96b85c808df1c2f6a lib/request/connect.py 43005bd6a78e9cf0f3ed2283a1cb122e lib/request/direct.py 2b7509ba38a667c61cefff036ec4ca6f lib/request/dns.py ceac6b3bf1f726f8ff43c6814e9d7281 lib/request/httpshandler.py