Implementation for #3505

lib/core/enums.py

@@ -377,6 +377,7 @@ class MKSTEMP_PREFIX:
     COOKIE_JAR = "sqlmapcookiejar-"
     BIG_ARRAY = "sqlmapbigarray-"
     SPECIFIC_RESPONSE = "sqlmapresponse-"
+    PREPROCESS = "sqlmappreprocess-"
 
 class TIMEOUT_STATE:
     NORMAL = 0

lib/core/option.py

@@ -76,6 +76,7 @@ from lib.core.enums import CUSTOM_LOGGING
 from lib.core.enums import DUMP_FORMAT
 from lib.core.enums import HTTP_HEADER
 from lib.core.enums import HTTPMETHOD
+from lib.core.enums import MKSTEMP_PREFIX
 from lib.core.enums import MOBILES
 from lib.core.enums import OPTION_TYPE
 from lib.core.enums import PAYLOAD
@@ -825,6 +826,80 @@ def _setTamperingFunctions():
             for _, function in priorities:
                 kb.tamperFunctions.append(function)
 
+def _setPreprocessFunctions():
+    """
+    Loads preprocess functions from given script(s)
+    """
+
+    if conf.preprocess:
+        for script in re.split(PARAMETER_SPLITTING_REGEX, conf.preprocess):
+            found = False
+
+            script = script.strip().encode(sys.getfilesystemencoding() or UNICODE_ENCODING)
+
+            try:
+                if not script:
+                    continue
+
+                if not os.path.exists(script):
+                    errMsg = "preprocess script '%s' does not exist" % script
+                    raise SqlmapFilePathException(errMsg)
+
+                elif not script.endswith(".py"):
+                    errMsg = "preprocess script '%s' should have an extension '.py'" % script
+                    raise SqlmapSyntaxException(errMsg)
+            except UnicodeDecodeError:
+                errMsg = "invalid character provided in option '--preprocess'"
+                raise SqlmapSyntaxException(errMsg)
+
+            dirname, filename = os.path.split(script)
+            dirname = os.path.abspath(dirname)
+
+            infoMsg = "loading preprocess module '%s'" % filename[:-3]
+            logger.info(infoMsg)
+
+            if not os.path.exists(os.path.join(dirname, "__init__.py")):
+                errMsg = "make sure that there is an empty file '__init__.py' "
+                errMsg += "inside of preprocess scripts directory '%s'" % dirname
+                raise SqlmapGenericException(errMsg)
+
+            if dirname not in sys.path:
+                sys.path.insert(0, dirname)
+
+            try:
+                module = __import__(filename[:-3].encode(sys.getfilesystemencoding() or UNICODE_ENCODING))
+            except Exception as ex:
+                raise SqlmapSyntaxException("cannot import preprocess module '%s' (%s)" % (filename[:-3], getSafeExString(ex)))
+
+            for name, function in inspect.getmembers(module, inspect.isfunction):
+                if name == "preprocess" and inspect.getargspec(function).args and all(_ in inspect.getargspec(function).args for _ in ("page", "headers", "code")):
+                    found = True
+
+                    kb.preprocessFunctions.append(function)
+                    function.func_name = module.__name__
+
+                    break
+
+            if not found:
+                errMsg = "missing function 'preprocess(page, headers=None, code=None)' "
+                errMsg += "in preprocess script '%s'" % script
+                raise SqlmapGenericException(errMsg)
+            else:
+                try:
+                    _, _, _ = function("", {}, None)
+                except:
+                    handle, filename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.PREPROCESS, suffix=".py")
+                    os.close(handle)
+
+                    open(filename, "w+b").write("#!/usr/bin/env\n\ndef preprocess(page, headers=None, code=None):\n    return page, headers, code\n")
+                    open(os.path.join(os.path.dirname(filename), "__init__.py"), "w+b").write("pass")
+
+                    errMsg = "function 'preprocess(page, headers=None, code=None)' "
+                    errMsg += "in preprocess script '%s' " % script
+                    errMsg += "should return a tuple '(page, headers, code)' "
+                    errMsg += "(Note: find template script at '%s')" % filename
+                    raise SqlmapGenericException(errMsg)
+
 def _setWafFunctions():
     """
     Loads WAF/IPS detecting functions from script(s)
@@ -1937,6 +2012,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
         kb.headerPaths = {}
         kb.keywords = set(getFileItems(paths.SQL_KEYWORDS))
         kb.passwordMgr = None
+        kb.preprocessFunctions = []
         kb.skipVulnHost = None
         kb.tamperFunctions = []
         kb.targets = oset()
@@ -2549,6 +2625,7 @@ def init():
     _setMultipleTargets()
     _listTamperingFunctions()
     _setTamperingFunctions()
+    _setPreprocessFunctions()
     _setWafFunctions()
     _setTrafficOutputFP()
     _setupHTTPCollector()

lib/core/settings.py

@@ -19,7 +19,7 @@ from lib.core.enums import DBMS_DIRECTORY_NAME
 from lib.core.enums import OS
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.3.3.1"
+VERSION = "1.3.3.2"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/parse/cmdline.py

@@ -595,6 +595,9 @@ def cmdLineParser(argv=None):
         general.add_option("--parse-errors", dest="parseErrors", action="store_true",
                            help="Parse and display DBMS error messages from responses")
 
+        general.add_option("--preprocess", dest="preprocess",
+                             help="Use given script(s) for preprocessing of response data")
+
         general.add_option("--repair", dest="repair", action="store_true",
                            help="Redump entries having unknown character marker (%s)" % INFERENCE_UNKNOWN_CHAR)
 

lib/request/connect.py

@@ -746,6 +746,14 @@ class Connect(object):
                     page = getUnicode(page)
             socket.setdefaulttimeout(conf.timeout)
 
+        for function in kb.preprocessFunctions:
+            try:
+                page, responseHeaders, code = function(page, responseHeaders, code)
+            except Exception as ex:
+                errMsg = "error occurred while running preprocess "
+                errMsg += "function '%s' ('%s')" % (function.func_name, getSafeExString(ex))
+                raise SqlmapGenericException(errMsg)
+
         processResponse(page, responseHeaders, status)
 
         if conn and getattr(conn, "redurl", None):

txt/checksum.md5

@@ -38,19 +38,19 @@ abcb1121eb56d3401839d14e8ed06b6e  lib/core/data.py
 5f4680b769ae07f22157bd832c97cf8f  lib/core/defaults.py
 9dfc69ba47209a4ceca494dde9ee8183  lib/core/dicts.py
 4ba141124699fd7a763dea82f17fe523  lib/core/dump.py
-0a49eaf3f940382464ee08c03c9891a8  lib/core/enums.py
+1226fed38d1175aee8907e31ddf0cab2  lib/core/enums.py
 84ef8f32e4582fcc294dc14e1997131d  lib/core/exception.py
 fb6be55d21a70765e35549af2484f762  lib/core/__init__.py
 18c896b157b03af716542e5fe9233ef9  lib/core/log.py
 151136142a14bee82cb02a9ca64c741d  lib/core/optiondict.py
-7f9d7b65f2278e5d233008a8bdd22c87  lib/core/option.py
+5d21cede75bd8043a0b9f2605047ea07  lib/core/option.py
 fe370021c6bc99daf44b2bfc0d1effb3  lib/core/patch.py
 4b12aa67fbf6c973d12e54cf9cb54ea0  lib/core/profiling.py
 d5ef43fe3cdd6c2602d7db45651f9ceb  lib/core/readlineng.py
 7d8a22c582ad201f65b73225e4456170  lib/core/replication.py
 3179d34f371e0295dd4604568fb30bcd  lib/core/revision.py
 d6269c55789f78cf707e09a0f5b45443  lib/core/session.py
-9dbce20566a1964f650b8986885ae370  lib/core/settings.py
+177d5fddb467b206530dacbc8618928d  lib/core/settings.py
 4483b4a5b601d8f1c4281071dff21ecc  lib/core/shell.py
 10fd19b0716ed261e6d04f311f6f527c  lib/core/subprocessng.py
 43772ea73e9e3d446f782af591cb4eda  lib/core/target.py
@@ -61,7 +61,7 @@ d6269c55789f78cf707e09a0f5b45443  lib/core/session.py
 5b3f08208be0579356f78ce5805d37b2  lib/core/wordlist.py
 fb6be55d21a70765e35549af2484f762  lib/__init__.py
 4881480d0c1778053908904e04570dc3  lib/parse/banner.py
-b23a0940d21347975a783c63fe671974  lib/parse/cmdline.py
+fafa321d2bbfc60410a131f68d5203ea  lib/parse/cmdline.py
 06ccbccb63255c8f1c35950a4c8a6f6b  lib/parse/configfile.py
 d34df646508c2dceb25205e1316673d1  lib/parse/handler.py
 43deb2400e269e602e916efaec7c0903  lib/parse/headers.py
@@ -72,7 +72,7 @@ adcecd2d6a8667b22872a563eb83eac0  lib/parse/payloads.py
 e4ea70bcd461f5176867dcd89d372386  lib/request/basicauthhandler.py
 b23163d485e0dbc038cbf1ba80be11da  lib/request/basic.py
 fc25d951217077fe655ed2a3a81552ae  lib/request/comparison.py
-2b58b3ed5f3aff7025e02bb1427bc637  lib/request/connect.py
+3925fef5710ac4e96b85c808df1c2f6a  lib/request/connect.py
 43005bd6a78e9cf0f3ed2283a1cb122e  lib/request/direct.py
 2b7509ba38a667c61cefff036ec4ca6f  lib/request/dns.py
 ceac6b3bf1f726f8ff43c6814e9d7281  lib/request/httpshandler.py