Speed optimization(s)

extra/safe2bin/safe2bin.py

@@ -23,7 +23,7 @@ HEX_ENCODED_CHAR_REGEX = r"(?P<result>\\x[0-9A-Fa-f]{2})"
 SAFE_ENCODE_SLASH_REPLACEMENTS = "\t\n\r\x0b\x0c"
 
 # Characters that don't need to be safe encoded
-SAFE_CHARS = "".join(filter(lambda x: x not in SAFE_ENCODE_SLASH_REPLACEMENTS, string.printable.replace('\\', '')))
+SAFE_CHARS = "".join(filter(lambda _: _ not in SAFE_ENCODE_SLASH_REPLACEMENTS, string.printable.replace('\\', '')))
 
 # Prefix used for hex encoded values
 HEX_ENCODED_PREFIX = r"\x"
@@ -47,7 +47,7 @@ def safecharencode(value):
     retVal = value
 
     if isinstance(value, basestring):
-        if any(_ not in SAFE_CHARS for _ in value):
+        if any([_ not in SAFE_CHARS for _ in value]):
             retVal = retVal.replace(HEX_ENCODED_PREFIX, HEX_ENCODED_PREFIX_MARKER)
             retVal = retVal.replace('\\', SLASH_MARKER)
 

lib/core/common.py

@@ -2227,10 +2227,6 @@ def getUnicode(value, encoding=None, noneToNull=False):
     if noneToNull and value is None:
         return NULL
 
-    if isListLike(value):
-        value = list(getUnicode(_, encoding, noneToNull) for _ in value)
-        return value
-
     if isinstance(value, unicode):
         return value
     elif isinstance(value, basestring):
@@ -2242,6 +2238,9 @@ def getUnicode(value, encoding=None, noneToNull=False):
                     return unicode(value, UNICODE_ENCODING)
                 except:
                     value = value[:ex.start] + "".join(INVALID_UNICODE_CHAR_FORMAT % ord(_) for _ in value[ex.start:ex.end]) + value[ex.end:]
+    elif isListLike(value):
+        value = list(getUnicode(_, encoding, noneToNull) for _ in value)
+        return value
     else:
         try:
             return unicode(value)
@@ -2559,6 +2558,7 @@ def logHTTPTraffic(requestLogMsg, responseLogMsg):
 def getPageTemplate(payload, place):  # Cross-linked function
     raise NotImplementedError
 
+@cachedmethod
 def getPublicTypeMembers(type_, onlyValues=False):
     """
     Useful for getting members from types (e.g. in enums)
@@ -2567,12 +2567,16 @@ def getPublicTypeMembers(type_, onlyValues=False):
     ['Linux', 'Windows']
     """
 
+    retVal = []
+
     for name, value in inspect.getmembers(type_):
         if not name.startswith('__'):
             if not onlyValues:
-                yield (name, value)
+                retVal.append((name, value))
             else:
-                yield value
+                retVal.append(value)
+
+    return retVal
 
 def enumValueToNameLookup(type_, value_):
     """
@@ -3581,6 +3585,7 @@ def randomizeParameterValue(value):
 
     return retVal
 
+@cachedmethod
 def asciifyUrl(url, forceQuote=False):
     """
     Attempts to make a unicode URL usuable with ``urllib/urllib2``.
@@ -4075,8 +4080,11 @@ def getRequestHeader(request, name):
     """
 
     retVal = None
+
     if request and name:
-        retVal = max(value if name.upper() == key.upper() else None for key, value in request.header_items())
+        _ = name.upper()
+        retVal = max([value if _ == key.upper() else None for key, value in request.header_items()])
+
     return retVal
 
 def isNumber(value):

lib/core/option.py

@@ -1014,12 +1014,12 @@ def _setDNSCache():
     """
 
     def _getaddrinfo(*args, **kwargs):
-        if args in kb.cache:
+        if args in kb.cache.addrinfo:
-            return kb.cache[args]
+            return kb.cache.addrinfo[args]
 
         else:
-            kb.cache[args] = socket._getaddrinfo(*args, **kwargs)
+            kb.cache.addrinfo[args] = socket._getaddrinfo(*args, **kwargs)
-            return kb.cache[args]
+            return kb.cache.addrinfo[args]
 
     if not hasattr(socket, "_getaddrinfo"):
         socket._getaddrinfo = socket.getaddrinfo
@@ -1841,7 +1841,10 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.bruteMode = False
 
     kb.cache = AttribDict()
+    kb.cache.addrinfo = {}
     kb.cache.content = {}
+    kb.cache.encoding = {}
+    kb.cache.parsedDbms = {}
     kb.cache.regex = {}
     kb.cache.stdev = {}
 

lib/core/settings.py

@@ -19,7 +19,7 @@ from lib.core.enums import OS
 from lib.core.revision import getRevisionNumber
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.0.9.10"
+VERSION = "1.0.9.11"
 REVISION = getRevisionNumber()
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}

lib/parse/html.py

@@ -59,6 +59,13 @@ def htmlParser(page):
 
     xmlfile = paths.ERRORS_XML
     handler = HTMLHandler(page)
+    key = hash(page)
+
+    if key in kb.cache.parsedDbms:
+        retVal = kb.cache.parsedDbms[key]
+        if retVal:
+            handler._markAsErrorPage()
+        return retVal
 
     parseXmlFile(xmlfile, handler)
 
@@ -68,6 +75,8 @@ def htmlParser(page):
     else:
         kb.lastParserStatus = None
 
+    kb.cache.parsedDbms[key] = handler.dbms
+
     # generic SQL warning/error messages
     if re.search(r"SQL (warning|error|syntax)", page, re.I):
         handler._markAsErrorPage()

lib/request/basic.py

@@ -26,6 +26,7 @@ from lib.core.common import singleTimeWarnMessage
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
+from lib.core.decorators import cachedmethod
 from lib.core.enums import DBMS
 from lib.core.enums import HTTP_HEADER
 from lib.core.enums import PLACE
@@ -136,6 +137,7 @@ def parseResponse(page, headers):
     if page:
         htmlParser(page)
 
+@cachedmethod
 def checkCharEncoding(encoding, warn=True):
     """
     Checks encoding name, repairs common misspellings and adjusts to
@@ -230,7 +232,10 @@ def getHeuristicCharEncoding(page):
     Returns page encoding charset detected by usage of heuristics
     Reference: http://chardet.feedparser.org/docs/
     """
-    retVal = detect(page)["encoding"]
+
+    key = hash(page)
+    retVal = kb.cache.encoding.get(key) or detect(page)["encoding"]
+    kb.cache.encoding[key] = retVal
 
     if retVal:
         infoMsg = "heuristics detected web page charset '%s'" % retVal

lib/request/connect.py

@@ -403,7 +403,7 @@ class Connect(object):
                 responseHeaders = _(ws.getheaders())
                 responseHeaders.headers = ["%s: %s\r\n" % (_[0].capitalize(), _[1]) for _ in responseHeaders.items()]
 
-                requestHeaders += "\n".join("%s: %s" % (getUnicode(key.capitalize() if isinstance(key, basestring) else key), getUnicode(value)) for (key, value) in responseHeaders.items())
+                requestHeaders += "\n".join(["%s: %s" % (getUnicode(key.capitalize() if isinstance(key, basestring) else key), getUnicode(value)) for (key, value) in responseHeaders.items()])
                 requestMsg += "\n%s" % requestHeaders
 
                 if post is not None:
@@ -422,7 +422,7 @@ class Connect(object):
                 else:
                     req = urllib2.Request(url, post, headers)
 
-                requestHeaders += "\n".join("%s: %s" % (getUnicode(key.capitalize() if isinstance(key, basestring) else key), getUnicode(value)) for (key, value) in req.header_items())
+                requestHeaders += "\n".join(["%s: %s" % (getUnicode(key.capitalize() if isinstance(key, basestring) else key), getUnicode(value)) for (key, value) in req.header_items()])
 
                 if not getRequestHeader(req, HTTP_HEADER.COOKIE) and conf.cj:
                     conf.cj._policy._now = conf.cj._now = int(time.time())
@@ -556,7 +556,7 @@ class Connect(object):
             responseMsg += "[#%d] (%d %s):\n" % (threadData.lastRequestUID, code, status)
 
             if responseHeaders:
-                logHeaders = "\n".join("%s: %s" % (getUnicode(key.capitalize() if isinstance(key, basestring) else key), getUnicode(value)) for (key, value) in responseHeaders.items())
+                logHeaders = "\n".join(["%s: %s" % (getUnicode(key.capitalize() if isinstance(key, basestring) else key), getUnicode(value)) for (key, value) in responseHeaders.items()])
 
             logHTTPTraffic(requestMsg, "%s%s\n\n%s" % (responseMsg, logHeaders, (page or "")[:MAX_CONNECTION_CHUNK_SIZE]))
 
@@ -691,7 +691,7 @@ class Connect(object):
             responseMsg += "[#%d] (%d %s):\n" % (threadData.lastRequestUID, code, status)
 
         if responseHeaders:
-            logHeaders = "\n".join("%s: %s" % (getUnicode(key.capitalize() if isinstance(key, basestring) else key), getUnicode(value)) for (key, value) in responseHeaders.items())
+            logHeaders = "\n".join(["%s: %s" % (getUnicode(key.capitalize() if isinstance(key, basestring) else key), getUnicode(value)) for (key, value) in responseHeaders.items()])
 
         if not skipLogTraffic:
             logHTTPTraffic(requestMsg, "%s%s\n\n%s" % (responseMsg, logHeaders, (page or "")[:MAX_CONNECTION_CHUNK_SIZE]))

txt/checksum.md5

@@ -10,7 +10,7 @@ acba8b5dc93db0fe6b2b04ff0138c33c  extra/icmpsh/icmpsh.exe_
 cc9c82cfffd8ee9b25ba3af6284f057e  extra/__init__.py
 2237d0568236c354b0436d2cd9434f97  extra/mssqlsig/update.py
 cc9c82cfffd8ee9b25ba3af6284f057e  extra/safe2bin/__init__.py
-cc5b67714d8a0b6b81d29a4f15634c16  extra/safe2bin/safe2bin.py
+a54bde99fd05fdb412cba5a8780f3e18  extra/safe2bin/safe2bin.py
 d229479d02d21b29f209143cb0547780  extra/shellcodeexec/linux/shellcodeexec.x32_
 2fe2f94eebc62f7614f0391a8a90104f  extra/shellcodeexec/linux/shellcodeexec.x64_
 c55b400b72acc43e0e59c87dd8bb8d75  extra/shellcodeexec/windows/shellcodeexec.x32.exe_
@@ -26,7 +26,7 @@ d1451b43f3ac80bfbea8657e288865f8  lib/controller/checks.py
 cc9c82cfffd8ee9b25ba3af6284f057e  lib/controller/__init__.py
 2689f320908964b2c88a3eb8265fd2dd  lib/core/agent.py
 eb0bd28b0bd9fbf67dcc3119116df377  lib/core/bigarray.py
-1dd298ac06c961037bb76a675bb4b322  lib/core/common.py
+d11993cd69f919216a9e4d54c77bb020  lib/core/common.py
 5680d0c446a3bed5c0f2a0402d031557  lib/core/convert.py
 e77cca1cb063016f71f6e6bdebf4ec73  lib/core/data.py
 1d042f0bc0557d3fd564ea5a46deb77e  lib/core/datatype.py
@@ -39,13 +39,13 @@ e4aec2b11c1ad6039d0c3dbbfbc5eb1a  lib/core/exception.py
 cc9c82cfffd8ee9b25ba3af6284f057e  lib/core/__init__.py
 91c514013daa796e2cdd940389354eac  lib/core/log.py
 b9779615206791e6ebbaa84947842b49  lib/core/optiondict.py
-cfb45d70fe381b85490374a8947437e4  lib/core/option.py
+ccd57542de8f3ae812a017c8c6da12b6  lib/core/option.py
 1e8948dddbd12def5c2af52530738059  lib/core/profiling.py
 e60456db5380840a586654344003d4e6  lib/core/readlineng.py
 5ef56abb8671c2ca6ceecb208258e360  lib/core/replication.py
 99a2b496b9d5b546b335653ca801153f  lib/core/revision.py
 7c15dd2777af4dac2c89cab6df17462e  lib/core/session.py
-022bb5857eb2db9df5ab982c056007ad  lib/core/settings.py
+f2b116a5b237ecd1678adaba584101f6  lib/core/settings.py
 7af83e4f18cab6dff5e67840eb65be80  lib/core/shell.py
 23657cd7d924e3c6d225719865855827  lib/core/subprocessng.py
 0bc2fae1dec18cdd11954b22358293f2  lib/core/target.py
@@ -61,14 +61,14 @@ daea32290b63c43f7d1c0e14c66d4826  lib/parse/cmdline.py
 8ec4d4f02634834701f8258726f2e511  lib/parse/configfile.py
 fe4e2152292587928edb94c9a4d311ff  lib/parse/handler.py
 8e6bfb13e5a34b2610f3ff23467a34cf  lib/parse/headers.py
-c8e14fbfc6616d8149b2603c97abec84  lib/parse/html.py
+cfa7b4c52915e1d7d59409ed49f0e623  lib/parse/html.py
 cc9c82cfffd8ee9b25ba3af6284f057e  lib/parse/__init__.py
 af6b8e1c6eb074b56bbd9cd80aebcd97  lib/parse/payloads.py
 b40a4c5d91770d347df36d3065b63798  lib/parse/sitemap.py
 9299f21804033f099681525bb9bf51c0  lib/request/basicauthhandler.py
-81bf50cdc078e640e1f382b7eeeb70d8  lib/request/basic.py
+ca9879ea9277810c5fce0f0d8d2f8e03  lib/request/basic.py
 97fb6323bfb5f941b27cbdb00f9078e1  lib/request/comparison.py
-8bc040159a145a1dfdf8a3fe76a0adbc  lib/request/connect.py
+20fe3d96ae45cf46d19504415cebd819  lib/request/connect.py
 49b4c583af68689de5f9acb162de2939  lib/request/direct.py
 1a46f7bb26b23ec0c0d9d9c95828241b  lib/request/dns.py
 70ceefe39980611494d4f99afb96f652  lib/request/httpshandler.py