switching to SQLite resume support (on error and union techniques this moment)

lib/core/option.py

@@ -1360,6 +1360,8 @@ def __setConfAttributes():
     conf.dbmsConnector = None
     conf.dbmsHandler = None
     conf.dumpPath = None
+    conf.hashDB = None
+    conf.hashDBFile = None
     conf.httpHeaders = []
     conf.hostname = None
     conf.multipleTargets = False

lib/core/target.py

@@ -40,6 +40,7 @@ from lib.core.settings import UNICODE_ENCODING
 from lib.core.settings import URI_INJECTABLE_REGEX
 from lib.core.settings import URI_INJECTION_MARK_CHAR
 from lib.core.settings import USER_AGENT_ALIASES
+from lib.utils.hashdb import HashDB
 from lib.core.xmldump import dumper as xmldumper
 from lib.request.connect import Connect as Request
 
@@ -174,6 +175,9 @@ def __setOutputResume():
     if not conf.sessionFile:
         conf.sessionFile = "%s%ssession" % (conf.outputPath, os.sep)
 
+    if not conf.hashDBFile:
+        conf.hashDBFile = "%s%shashdb" % (conf.outputPath, os.sep)
+
     logger.info("using '%s' as session file" % conf.sessionFile)
 
     if os.path.exists(conf.sessionFile):
@@ -223,6 +227,7 @@ def __setOutputResume():
         else:
             try:
                 os.remove(conf.sessionFile)
+                os.remove(conf.hashDBFile)
                 logger.info("flushing session file")
             except OSError, msg:
                 errMsg = "unable to flush the session file (%s)" % msg
@@ -230,6 +235,7 @@ def __setOutputResume():
 
     try:
         conf.sessionFP = codecs.open(conf.sessionFile, "a", UNICODE_ENCODING)
+        conf.hashDB = HashDB(conf.hashDBFile)
         dataToSessionFile("\n[%s]\n" % time.strftime("%X %x"))
     except IOError:
         errMsg = "unable to write on the session file specified"
@@ -338,12 +344,16 @@ def initTargetEnv():
         if conf.sessionFP:
             conf.sessionFP.close()
 
+        if conf.hashDB:
+            conf.hashDB.close()
+
         if conf.cj:
             conf.cj.clear()
 
         conf.paramDict = {}
         conf.parameters = {}
         conf.sessionFile = None
+        conf.hashDBFile = None
 
         __setKnowledgeBaseAttributes(False)
         __restoreCmdLineOptions()

lib/techniques/error/use.py

@@ -53,10 +53,12 @@ def __oneShotErrorUse(expression, field):
 
     threadData = getCurrentThreadData()
 
-    retVal = None
+    retVal = conf.hashDB.retrieve(expression) if not conf.freshQueries else None
+
     offset = 1
     chunk_length = None
 
+    if not retVal:
         while True:
             check = "%s(?P<result>.*?)%s" % (kb.misc.start, kb.misc.stop)
             trimcheck = "%s(?P<result>.*?)</" % (kb.misc.start)
@@ -129,7 +131,8 @@ def __oneShotErrorUse(expression, field):
 
         retVal = __errorReplaceChars(retVal)
 
-    dataToSessionFile("[%s][%s][%s][%s][%s]\n" % (conf.url, kb.injection.place, conf.parameters[kb.injection.place], expression, replaceNewlineTabs(retVal)))
+        #dataToSessionFile("[%s][%s][%s][%s][%s]\n" % (conf.url, kb.injection.place, conf.parameters[kb.injection.place], expression, replaceNewlineTabs(retVal)))
+        conf.hashDB.write(expression, retVal)
 
     return safecharencode(retVal) if kb.safeCharEncode else retVal
 

lib/techniques/union/use.py

@@ -50,6 +50,9 @@ reqCount = 0
 def __oneShotUnionUse(expression, unpack=True, limited=False):
     global reqCount
 
+    retVal = conf.hashDB.retrieve(expression) if not conf.freshQueries else None
+
+    if not retVal:
         check = "(?P<result>%s.*%s)" % (kb.misc.start, kb.misc.stop)
         trimcheck = "%s(?P<result>.*?)</" % (kb.misc.start)
 
@@ -74,14 +77,14 @@ def __oneShotUnionUse(expression, unpack=True, limited=False):
 
         # Parse the returned page to get the exact union-based
         # sql injection output
-    output = reduce(lambda x, y: x if x is not None else y, [ \
+        retVal = reduce(lambda x, y: x if x is not None else y, [ \
                 extractRegexResult(check, removeReflectiveValues(page, payload), re.DOTALL | re.IGNORECASE), \
                 extractRegexResult(check, removeReflectiveValues(listToStrValue(headers.headers \
                 if headers else None), payload, True), re.DOTALL | re.IGNORECASE)], \
                 None)
 
-    if output is not None:
+        if retVal is not None:
-        output = getUnicode(output, kb.pageEncoding)
+            retVal = getUnicode(retVal, kb.pageEncoding)
         else:
             trimmed = extractRegexResult(trimcheck, removeReflectiveValues(page, payload), re.DOTALL | re.IGNORECASE) \
                     or extractRegexResult(trimcheck, removeReflectiveValues(listToStrValue(headers.headers \
@@ -97,7 +100,9 @@ def __oneShotUnionUse(expression, unpack=True, limited=False):
                 warnMsg += "issues)"
                 singleTimeWarnMessage(warnMsg)
 
-    return output
+        conf.hashDB.write(expression, retVal)
+
+    return retVal
 
 def configUnion(char=None, columns=None):
     def __configUnionChar(char):

lib/utils/hashdb.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+
+"""
+$Id$
+
+Copyright (c) 2006-2011 sqlmap developers (http://www.sqlmap.org/)
+See the file 'doc/COPYING' for copying permission
+"""
+
+import hashlib
+import sqlite3
+
+from lib.core.settings import UNICODE_ENCODING
+
+class HashDB:
+    def __init__(self, filepath):
+        self.connection = sqlite3.connect(filepath)
+        self.cursor = self.connection.cursor()
+        self.cursor.execute("CREATE TABLE IF NOT EXISTS storage (id INTEGER PRIMARY KEY, value TEXT)")
+
+    def __del__(self):
+        self.close()
+
+    def close(self):
+        try:
+            self.endTransaction()
+            self.connection.close()
+        except:
+            pass
+
+    def hashKey(self, key):
+        key = key.encode(UNICODE_ENCODING) if isinstance(key, unicode) else repr(key)
+        retVal = int(hashlib.md5(key).hexdigest()[:8], 16)
+        return retVal
+
+    def beginTransaction(self):
+        """
+        Great speed improvement can be gained by using explicit transactions around multiple inserts.
+        Reference: http://stackoverflow.com/questions/4719836/python-and-sqlite3-adding-thousands-of-rows
+        """
+        self.cursor.execute('BEGIN TRANSACTION')
+
+    def endTransaction(self):
+        try:
+            self.cursor.execute('END TRANSACTION')
+        except sqlite3.OperationalError:
+            pass
+
+    def retrieve(self, key):
+        retVal = None
+        if key:
+            hash_ = self.hashKey(key)
+            for row in self.cursor.execute("SELECT value FROM storage WHERE id=?", (hash_,)):
+                retVal = row[0]
+        return retVal
+
+    def write(self, key, value):
+        if key:
+            hash_ = self.hashKey(key)
+            try:
+                self.cursor.execute("INSERT INTO storage VALUES (?, ?)", (hash_, value,))
+            except sqlite3.IntegrityError:
+                self.cursor.execute("UPDATE storage SET value=? WHERE id=?", (value, hash_,))