Revamp of #3536

lib/core/common.py

@@ -98,7 +98,7 @@ from lib.core.exception import SqlmapUserQuitException
 from lib.core.exception import SqlmapValueException
 from lib.core.log import LOGGER_HANDLER
 from lib.core.optiondict import optDict
-from lib.core.settings import BANNER, CHUNKED_KEYWORDS
+from lib.core.settings import BANNER
 from lib.core.settings import BOLD_PATTERNS
 from lib.core.settings import BOUNDED_INJECTION_MARKER
 from lib.core.settings import BRUTE_DOC_ROOT_PREFIXES
@@ -126,6 +126,7 @@ from lib.core.settings import GITHUB_REPORT_OAUTH_TOKEN
 from lib.core.settings import GOOGLE_ANALYTICS_COOKIE_PREFIX
 from lib.core.settings import HASHDB_MILESTONE_VALUE
 from lib.core.settings import HOST_ALIASES
+from lib.core.settings import HTTP_CHUNKED_SPLIT_KEYWORDS
 from lib.core.settings import IGNORE_SAVE_OPTIONS
 from lib.core.settings import INFERENCE_UNKNOWN_CHAR
 from lib.core.settings import INVALID_UNICODE_CHAR_FORMAT
@@ -4896,49 +4897,35 @@ def firstNotNone(*args):
 
     return retVal
 
-def generateChunkDdata(data):
+def chunkSplitPostData(data):
     """
-    Convert post data to chunked format data. If the keyword is in a block, the keyword will be cut.
+    Convert POST data to chunked transfer-encoded data (Note: splitting done by SQL keywords)
-
-    >>> generateChunkDdata('select 1,2,3,4 from admin')
-    4;AZdYz
-    sele
-    2;fJS4D
-    ct
-    5;qbCOT
-    1,2,
-    7;KItpi
-    3,4 fro
-    2;pFu1R
-    m 
-    5;uRoYZ
-    admin
-    0
-
-
     """
-    dl = len(data)
+
-    ret = ""
+    length = len(data)
-    keywords = CHUNKED_KEYWORDS
+    retVal = ""
     index = 0
-    while index < dl:
-        chunk_size = random.randint(1, 9)
-        if index + chunk_size >= dl:
-            chunk_size = dl - index
-        salt = ''.join(random.sample(string.ascii_letters + string.digits, 5))
-        while 1:
-            tmp_chunk = data[index:index + chunk_size]
-            tmp_bool = True
-            for k in keywords:
-                if k in tmp_chunk:
-                    chunk_size -= 1
-                    tmp_bool = False
-                    break
-            if tmp_bool:
-                break
-        index += chunk_size
-        ret += "%s;%s\r\n" % (hex(chunk_size)[2:], salt)
-        ret += "%s\r\n" % tmp_chunk
 
-    ret += "0\r\n\r\n"
+    while index < length:
-    return ret
+        chunkSize = randomInt(1)
+
+        if index + chunkSize >= length:
+            chunkSize = length - index
+
+        salt = randomStr(5, alphabet=string.ascii_letters + string.digits)
+
+        while chunkSize:
+            candidate = data[index:index + chunkSize]
+
+            if re.search(r"\b%s\b" % '|'.join(HTTP_CHUNKED_SPLIT_KEYWORDS), candidate, re.I):
+                chunkSize -= 1
+            else:
+                break
+
+        index += chunkSize
+        retVal += "%x;%s\r\n" % (chunkSize, salt)
+        retVal += "%s\r\n" % candidate
+
+    retVal += "0\r\n\r\n"
+
+    return retVal

lib/core/option.py

@@ -140,7 +140,7 @@ from lib.request.basic import checkCharEncoding
 from lib.request.connect import Connect as Request
 from lib.request.dns import DNSServer
 from lib.request.basicauthhandler import SmartHTTPBasicAuthHandler
-from lib.request.httphandler import HTTPHandler
+from lib.request.chunkedhandler import ChunkedHandler
 from lib.request.httpshandler import HTTPSHandler
 from lib.request.pkihandler import HTTPSPKIAuthHandler
 from lib.request.rangehandler import HTTPRangeHandler
@@ -158,7 +158,7 @@ from thirdparty.socks import socks
 from xml.etree.ElementTree import ElementTree
 
 authHandler = urllib2.BaseHandler()
-httpHandler = HTTPHandler()
+chunkedHandler = ChunkedHandler()
 httpsHandler = HTTPSHandler()
 keepAliveHandler = keepalive.HTTPHandler()
 proxyHandler = urllib2.ProxyHandler()
@@ -1109,7 +1109,7 @@ def _setHTTPHandlers():
     debugMsg = "creating HTTP requests opener object"
     logger.debug(debugMsg)
 
-    handlers = filter(None, [multipartPostHandler, proxyHandler if proxyHandler.proxies else None, authHandler, redirectHandler, rangeHandler, httpHandler, httpsHandler])
+    handlers = filter(None, [multipartPostHandler, proxyHandler if proxyHandler.proxies else None, authHandler, redirectHandler, rangeHandler, chunkedHandler if conf.chunked else None, httpsHandler])
 
     if not conf.dropSetCookie:
         if not conf.loadCookies:
@@ -2314,6 +2314,10 @@ def _setTorSocksProxySettings():
     socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5 if conf.torType == PROXY_TYPE.SOCKS5 else socks.PROXY_TYPE_SOCKS4, LOCALHOST, port)
     socks.wrapmodule(urllib2)
 
+def _setHttpChunked():
+    if conf.chunked and conf.data:
+        httplib.HTTPConnection._set_content_length = lambda self, a, b: None
+
 def _checkWebSocket():
     if conf.url and (conf.url.startswith("ws:/") or conf.url.startswith("wss:/")):
         try:
@@ -2401,6 +2405,10 @@ def _basicOptionValidation():
         errMsg = "switch '--dump' is incompatible with switch '--search'"
         raise SqlmapSyntaxException(errMsg)
 
+    if conf.chunked and not any((conf.data, conf.requestFile)):
+        errMsg = "switch '--chunked' requires usage of option '--data' or '-r'"
+        raise SqlmapSyntaxException(errMsg)
+
     if conf.api and not conf.configFile:
         errMsg = "switch '--api' requires usage of option '-c'"
         raise SqlmapSyntaxException(errMsg)
@@ -2605,15 +2613,6 @@ def initOptions(inputOptions=AttribDict(), overrideOptions=False):
     _setKnowledgeBaseAttributes()
     _mergeOptions(inputOptions, overrideOptions)
 
-def _setHttpChunked():
-    conf.chunk = conf.chunk and conf.data
-    if conf.chunk:
-        def hook(self, a, b):
-            pass
-
-        httplib.HTTPConnection._set_content_length = hook
-
-
 def init():
     """
     Set attributes into both configuration and knowledge base singletons
@@ -2639,11 +2638,11 @@ def init():
     _listTamperingFunctions()
     _setTamperingFunctions()
     _setPreprocessFunctions()
-    _setHttpChunked()
     _setWafFunctions()
     _setTrafficOutputFP()
     _setupHTTPCollector()
     _resolveCrossReferences()
+    _setHttpChunked()
     _checkWebSocket()
 
     parseTargetDirect()

lib/core/optiondict.py

@@ -61,6 +61,7 @@ optDict = {
         "csrfToken": "string",
         "csrfUrl": "string",
         "forceSSL": "boolean",
+        "chunked": "boolean",
         "hpp": "boolean",
         "evalCode": "string",
     },

lib/core/settings.py

@@ -19,7 +19,7 @@ from lib.core.enums import DBMS_DIRECTORY_NAME
 from lib.core.enums import OS
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.3.3.31"
+VERSION = "1.3.3.32"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -794,8 +794,8 @@ KB_CHARS_BOUNDARY_CHAR = 'q'
 # Letters of lower frequency used in kb.chars
 KB_CHARS_LOW_FREQUENCY_ALPHABET = "zqxjkvbp"
 
-# Keywords that need to be cut in the chunked
+# SQL keywords used for splitting in HTTP Chunked encoding (switch --chunk)
-CHUNKED_KEYWORDS = ['select', 'update', 'insert', 'from', 'load_file', 'sysdatabases', 'msysaccessobjects', 'msysqueries', 'sysmodules', 'information_schema', 'union']
+HTTP_CHUNKED_SPLIT_KEYWORDS = ("SELECT", "UPDATE", "INSERT", "FROM", "LOAD_FILE", "UNION", "information_schema", "sysdatabases", "msysaccessobjects", "msysqueries", "sysmodules")
 
 # CSS style used in HTML dump format
 HTML_DUMP_CSS_STYLE = """<style>

lib/parse/cmdline.py

@@ -215,14 +215,15 @@ def cmdLineParser(argv=None):
         request.add_option("--force-ssl", dest="forceSSL", action="store_true",
                            help="Force usage of SSL/HTTPS")
 
+        request.add_option("--chunked", dest="chunked", action="store_true",
+                           help="Use HTTP Chunked transfer encoding method")
+
         request.add_option("--hpp", dest="hpp", action="store_true",
                            help="Use HTTP parameter pollution method")
 
         request.add_option("--eval", dest="evalCode",
                            help="Evaluate provided Python code before the request (e.g. \"import hashlib;id2=hashlib.md5(id).hexdigest()\")")
        
-        request.add_option("--chunk", dest="chunk", action="store_true", help="all requests will be added headers with 'Transfer-Encoding: Chunked' and sent by transcoding")
-
         # Optimization options
         optimization = OptionGroup(parser, "Optimization", "These options can be used to optimize the performance of sqlmap")
 

lib/request/chunkedhandler.py

@@ -6,16 +6,15 @@ See the file 'LICENSE' for copying permission
 """
 
 import urllib2
-import httplib
+
 from lib.core.data import conf
 
-
+class ChunkedHandler(urllib2.HTTPHandler):
-class HTTPHandler(urllib2.HTTPHandler):
     """
-    The hook http_requests function ensures that the chunk function is working properly.
+    Ensures that urllib2.HTTPHandler is working properly in case of Chunked Transfer-Encoding
     """
 
-    def _hook(self, request):
+    def _http_request(self, request):
         host = request.get_host()
         if not host:
             raise urllib2.URLError('no host given')
@@ -26,7 +25,7 @@ class HTTPHandler(urllib2.HTTPHandler):
                 request.add_unredirected_header(
                     'Content-type',
                     'application/x-www-form-urlencoded')
-            if not request.has_header('Content-length') and not conf.chunk:
+            if not request.has_header('Content-length') and not conf.chunked:
                 request.add_unredirected_header(
                     'Content-length', '%d' % len(data))
 
@@ -43,4 +42,4 @@ class HTTPHandler(urllib2.HTTPHandler):
                 request.add_unredirected_header(name, value)
         return request
 
-    http_request = _hook
+    http_request = _http_request

lib/request/connect.py

@@ -31,6 +31,7 @@ from lib.core.agent import agent
 from lib.core.common import asciifyUrl
 from lib.core.common import calculateDeltaSeconds
 from lib.core.common import checkSameHost
+from lib.core.common import chunkSplitPostData
 from lib.core.common import clearConsoleLine
 from lib.core.common import dataToStdout
 from lib.core.common import escapeJsonValue
@@ -61,7 +62,6 @@ from lib.core.common import unicodeencode
 from lib.core.common import unsafeVariableNaming
 from lib.core.common import urldecode
 from lib.core.common import urlencode
-from lib.core.common import generateChunkDdata
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
@@ -272,13 +272,14 @@ class Connect(object):
         checking = kwargs.get("checking", False)
         skipRead = kwargs.get("skipRead", False)
         finalCode = kwargs.get("finalCode", False)
-        chunked = conf.chunk
+        chunked = kwargs.get("chunked", False) or conf.chunked
 
         if multipart:
             post = multipart
+
         if chunked:
             post = urllib.unquote(post)
-            post = generateChunkDdata(post)
+            post = chunkSplitPostData(post)
 
         websocket_ = url.lower().startswith("ws")
 
@@ -403,7 +404,7 @@ class Connect(object):
                 headers[HTTP_HEADER.CONNECTION] = "keep-alive"
             
             if chunked:
-                headers[HTTP_HEADER.TRANSFER_ENCODING] = "Chunked"
+                headers[HTTP_HEADER.TRANSFER_ENCODING] = "chunked"
 
             if auxHeaders:
                 headers = forgeHeaders(auxHeaders, headers)

sqlmap.conf

@@ -190,6 +190,10 @@ csrfUrl =
 # Valid: True or False
 forceSSL = False
 
+# Use HTTP Chunked transfer encoding method.
+# Valid: True or False
+chunked = False
+
 # Use HTTP parameter pollution.
 # Valid: True or False
 hpp = False

txt/checksum.md5

@@ -30,7 +30,7 @@ c1da277517c7ec4c23e953a51b51e203  lib/controller/handler.py
 fb6be55d21a70765e35549af2484f762  lib/controller/__init__.py
 ed7874be0d2d3802f3d20184f2b280d5  lib/core/agent.py
 a932126e7d80e545c5d44af178d0bc0c  lib/core/bigarray.py
-b096680d917729fd9658f9b75d44bb3b  lib/core/common.py
+2344f86e7eb59920645bea6d5f40f580  lib/core/common.py
 de8d27ae6241163ff9e97aa9e7c51a18  lib/core/convert.py
 abcb1121eb56d3401839d14e8ed06b6e  lib/core/data.py
 f89512ef3ebea85611c5dde6c891b657  lib/core/datatype.py
@@ -42,15 +42,15 @@ f89512ef3ebea85611c5dde6c891b657  lib/core/datatype.py
 84ef8f32e4582fcc294dc14e1997131d  lib/core/exception.py
 fb6be55d21a70765e35549af2484f762  lib/core/__init__.py
 18c896b157b03af716542e5fe9233ef9  lib/core/log.py
-947f41084e551ff3b7ef7dda2f25ef20  lib/core/optiondict.py
+2f474c3c7a56f0c3cf3371838e7e5fd4  lib/core/optiondict.py
-94679a06c134ca5c1db1e435e1cb9fb1  lib/core/option.py
+e9b27d7328a8da0b48a908f0112c7745  lib/core/option.py
 fe370021c6bc99daf44b2bfc0d1effb3  lib/core/patch.py
 4b12aa67fbf6c973d12e54cf9cb54ea0  lib/core/profiling.py
 d5ef43fe3cdd6c2602d7db45651f9ceb  lib/core/readlineng.py
 7d8a22c582ad201f65b73225e4456170  lib/core/replication.py
 3179d34f371e0295dd4604568fb30bcd  lib/core/revision.py
 d6269c55789f78cf707e09a0f5b45443  lib/core/session.py
-858db5e54ce928b2ae4ed7fe55c25c12  lib/core/settings.py
+d89a43c92f2995116fce16b0e00e8aff  lib/core/settings.py
 4483b4a5b601d8f1c4281071dff21ecc  lib/core/shell.py
 10fd19b0716ed261e6d04f311f6f527c  lib/core/subprocessng.py
 10d7e4f7ba2502cce5cf69223c52eddc  lib/core/target.py
@@ -61,7 +61,7 @@ d6269c55789f78cf707e09a0f5b45443  lib/core/session.py
 5b3f08208be0579356f78ce5805d37b2  lib/core/wordlist.py
 fb6be55d21a70765e35549af2484f762  lib/__init__.py
 4881480d0c1778053908904e04570dc3  lib/parse/banner.py
-fafa321d2bbfc60410a131f68d5203ea  lib/parse/cmdline.py
+79777f4f934f3b0a436fdd6600f7d3b8  lib/parse/cmdline.py
 06ccbccb63255c8f1c35950a4c8a6f6b  lib/parse/configfile.py
 d34df646508c2dceb25205e1316673d1  lib/parse/handler.py
 43deb2400e269e602e916efaec7c0903  lib/parse/headers.py
@@ -71,8 +71,9 @@ adcecd2d6a8667b22872a563eb83eac0  lib/parse/payloads.py
 993104046c7d97120613409ef7780c76  lib/parse/sitemap.py
 e4ea70bcd461f5176867dcd89d372386  lib/request/basicauthhandler.py
 bd4b654767eab19cd4dcd4520a68eed5  lib/request/basic.py
+caa52d249fbcf1705cd9208b84d93387  lib/request/chunkedhandler.py
 fc25d951217077fe655ed2a3a81552ae  lib/request/comparison.py
-e6792ea3cdbcbe17a7fa8cf856d2b956  lib/request/connect.py
+f44c5d2716317a0dfa024c98ce4865e9  lib/request/connect.py
 43005bd6a78e9cf0f3ed2283a1cb122e  lib/request/direct.py
 2b7509ba38a667c61cefff036ec4ca6f  lib/request/dns.py
 ceac6b3bf1f726f8ff43c6814e9d7281  lib/request/httpshandler.py