some more optimizations

_sqlmap.py

@@ -129,6 +129,9 @@ def main():
         kb.threadContinue = False
         kb.threadException = True
 
+        if conf.hashDB:
+            conf.hashDB.flush(True)
+
         # Reference: http://stackoverflow.com/questions/1635080/terminate-a-multi-thread-python-program
         if hasattr(conf, "threads") and conf.threads > 1:
             os._exit(0)

lib/core/settings.py

@@ -415,3 +415,6 @@ DEFAULT_GET_POST_DELIMITER = '&'
 
 # Default delimiter in cookie values
 DEFAULT_COOKIE_DELIMITER = ';'
+
+# Skip unforced HashDB flush requests below the threshold number of cached items
+HASHDB_FLUSH_THRESHOLD = 10

lib/core/threads.py

@@ -181,7 +181,7 @@ def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardExceptio
         kb.threadContinue = True
         kb.threadException = False
 
-        conf.hashDB.flush()
+        conf.hashDB.flush(True)
 
         if cleanupFunction:
             cleanupFunction()

lib/utils/hashdb.py

@@ -12,6 +12,7 @@ import sqlite3
 import threading
 
 from lib.core.data import conf
+from lib.core.settings import HASHDB_FLUSH_THRESHOLD
 from lib.core.settings import UNICODE_ENCODING
 from lib.core.threads import getCurrentThreadData
 from lib.core.threads import getCurrentThreadName
@@ -54,15 +55,17 @@ class HashDB(object):
         retVal = None
         if key:
             hash_ = HashDB.hashKey(key)
-            while True:
-                try:
-                    for row in self.cursor.execute("SELECT value FROM storage WHERE id=?", (hash_,)):
-                        retVal = row[0]
-                except sqlite3.OperationalError, ex:
-                    if not 'locked' in ex.message:
-                        raise
-                else:
-                    break
+            retVal = self._write_cache.get(hash_, None)
+            if not retVal:
+                while True:
+                    try:
+                        for row in self.cursor.execute("SELECT value FROM storage WHERE id=?", (hash_,)):
+                            retVal = row[0]
+                    except sqlite3.OperationalError, ex:
+                        if not 'locked' in ex.message:
+                            raise
+                    else:
+                        break
         return retVal
 
     def write(self, key, value):
@@ -75,10 +78,13 @@ class HashDB(object):
         if getCurrentThreadName() in ('0', 'MainThread'):
             self.flush()
 
-    def flush(self):
+    def flush(self, forced=False):
         if not self._write_cache:
             return
 
+        if not forced and len(self._write_cache) < HASHDB_FLUSH_THRESHOLD:
+            return
+
         self._cache_lock.acquire()
         items = self._write_cache.items()
         self._write_cache.clear()

plugins/generic/enumeration.py

@@ -1564,7 +1564,6 @@ class Enumeration:
 
             try:
                 kb.dumpMode = True
-                #conf.hashDB.beginTransaction()
 
                 if not safeSQLIdentificatorNaming(conf.db) in kb.data.cachedColumns \
                    or safeSQLIdentificatorNaming(tbl, True) not in \
@@ -1789,7 +1788,6 @@ class Enumeration:
 
             finally:
                 kb.dumpMode = False
-                #conf.hashDB.endTransaction()
 
     def dumpAll(self):
         if conf.db is not None and conf.tbl is None: