major improvement of HashDB speed in multi-threaded mode

lib/core/threads.py

@@ -65,6 +65,13 @@ def getCurrentThreadData():
 
     return ThreadData
 
+def getCurrentThreadName():
+    """
+    Returns current's thread name
+    """
+
+    return threading.current_thread().getName()
+
 def exceptionHandledFunction(threadFunction):
     try:
         threadFunction()

lib/utils/hashdb.py

@@ -9,14 +9,18 @@ See the file 'doc/COPYING' for copying permission
 
 import hashlib
 import sqlite3
+import threading
 
 from lib.core.data import conf
 from lib.core.settings import UNICODE_ENCODING
 from lib.core.threads import getCurrentThreadData
+from lib.core.threads import getCurrentThreadName
 
 class HashDB(object):
     def __init__(self, filepath):
         self.filepath = filepath
+        self._write_cache = {}
+        self._cache_lock = threading.Lock()
 
     def _get_cursor(self):
         threadData = getCurrentThreadData()
@@ -64,6 +68,21 @@ class HashDB(object):
     def write(self, key, value):
         if key:
             hash_ = HashDB.hashKey(key)
+            self._cache_lock.acquire()
+            self._write_cache[hash_] = value
+            self._cache_lock.release()
+
+        if getCurrentThreadName() in ('0', 'MainThread'):
+            self.flush()
+
+    def flush(self):
+        self._cache_lock.acquire()
+        items = self._write_cache.items()
+        self._write_cache.clear()
+        self._cache_lock.release()
+
+        self.beginTransaction()
+        for hash_, value in items:
             while True:
                 try:
                     try:
@@ -75,6 +94,7 @@ class HashDB(object):
                         raise
                 else:
                     break
+        self.endTransaction()
 
     def beginTransaction(self):
         self.cursor.execute('BEGIN TRANSACTION')

plugins/generic/enumeration.py

@@ -1564,6 +1564,7 @@ class Enumeration:
 
             try:
                 kb.dumpMode = True
+                #conf.hashDB.beginTransaction()
 
                 if not safeSQLIdentificatorNaming(conf.db) in kb.data.cachedColumns \
                    or safeSQLIdentificatorNaming(tbl, True) not in \
@@ -1788,6 +1789,7 @@ class Enumeration:
 
             finally:
                 kb.dumpMode = False
+                #conf.hashDB.endTransaction()
 
     def dumpAll(self):
         if conf.db is not None and conf.tbl is None: