some refactoring

lib/controller/controller.py

@@ -31,14 +31,13 @@ from lib.core.common import paramToDict
 from lib.core.common import parseTargetUrl
 from lib.core.common import randomStr
 from lib.core.common import readInput
-from lib.core.common import serializeObject
 from lib.core.common import showHttpErrorCodes
-from lib.core.common import unserializeObject
 from lib.core.convert import urlencode
 from lib.core.convert import urldecode
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
+from lib.core.enums import HASHDB_KEYS
 from lib.core.enums import HTTPHEADER
 from lib.core.enums import HTTPMETHOD
 from lib.core.enums import PAYLOAD
@@ -177,14 +176,12 @@ def __saveToSessionFile():
         setInjection(inj)
 
 def __saveToHashDB():
-    key = "kb.absFilePaths"
+    _ = conf.hashDB.retrieve(HASHDB_KEYS.KB_ABS_FILE_PATHS, True) or set()
-    value = unserializeObject(conf.hashDB.retrieve(key)) or set()
+    _.update(kb.absFilePaths)
-    value.update(kb.absFilePaths)
+    conf.hashDB.write(HASHDB_KEYS.KB_ABS_FILE_PATHS, _, True)
-    conf.hashDB.write(key, serializeObject(value))
 
-    key = "kb.chars"
+    if not conf.hashDB.retrieve(HASHDB_KEYS.KB_CHARS):
-    if not conf.hashDB.retrieve(key):
+        conf.hashDB.write(HASHDB_KEYS.KB_CHARS, kb.chars, True)
-        conf.hashDB.write(key, serializeObject(kb.chars))
 
 def __saveToResultsFile():
     if not conf.resultsFP:

lib/core/enums.py

@@ -121,6 +121,12 @@ class EXPECTED:
     BOOL = "bool"
     INT = "int"
 
+class HASHDB_KEYS:
+    KB_ABS_FILE_PATHS = "KB_ABS_FILE_PATHS"
+    KB_CHARS = "KB_CHARS"
+    KB_BRUTE_TABLES = "KB_BRUTE_TABLES"
+    KB_BRUTE_COLUMNS = "KB_BRUTE_COLUMNS"
+
 class REDIRECTION:
     FOLLOW = "1"
     ORIGINAL = "2"

lib/core/session.py

@@ -274,37 +274,6 @@ def resumeConfKb(expression, url, value):
     elif conf.freshQueries:
         pass
 
-    elif expression == "TABLE_EXISTS" and url == conf.url:
-        table = unSafeFormatString(value[:-1])
-        split = '..' if Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE) else '.'
-
-        if split in table:
-            db, table = table.split(split)
-        else:
-            db = "%s%s" % (Backend.getIdentifiedDbms(), METADB_SUFFIX)
-
-        infoMsg = "resuming brute forced table name "
-        infoMsg += "'%s' from session file" % table
-        logger.info(infoMsg)
-
-        kb.brute.tables.append((db, table))
-
-    elif expression == "COLUMN_EXISTS" and url == conf.url:
-        table, column = unSafeFormatString(value[:-1]).split('|')
-        colName, colType = column.split(' ')
-        split = '..' if Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE) else '.'
-
-        if split in table:
-            db, table = table.split(split)
-        else:
-            db = "%s%s" % (Backend.getIdentifiedDbms(), METADB_SUFFIX)
-
-        infoMsg = "resuming brute forced column name "
-        infoMsg += "'%s' for table '%s' from session file" % (colName, table)
-        logger.info(infoMsg)
-
-        kb.brute.columns.append((db, table, colName, colType))
-
     elif expression == "xp_cmdshell availability" and url == conf.url:
         kb.xpCmdshellAvailable = True if unSafeFormatString(value[:-1]).lower() == "true" else False
         infoMsg = "resuming xp_cmdshell availability"

lib/core/target.py

@@ -17,7 +17,6 @@ from lib.core.common import dataToSessionFile
 from lib.core.common import intersect
 from lib.core.common import paramToDict
 from lib.core.common import readInput
-from lib.core.common import unserializeObject
 from lib.core.convert import urldecode
 from lib.core.data import cmdLineOptions
 from lib.core.data import conf
@@ -25,6 +24,7 @@ from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.data import paths
 from lib.core.dump import dumper
+from lib.core.enums import HASHDB_KEYS
 from lib.core.enums import HTTPMETHOD
 from lib.core.enums import PLACE
 from lib.core.exception import sqlmapFilePathException
@@ -209,8 +209,10 @@ def __resumeHashDBValues():
     Resume stored data values from HashDB
     """
 
-    kb.absFilePaths = unserializeObject(conf.hashDB.retrieve("kb.absFilePaths")) or kb.absFilePaths
+    kb.absFilePaths = conf.hashDB.retrieve(HASHDB_KEYS.KB_ABS_FILE_PATHS, True) or kb.absFilePaths
-    kb.chars = unserializeObject(conf.hashDB.retrieve("kb.chars")) or kb.chars
+    kb.chars = conf.hashDB.retrieve(HASHDB_KEYS.KB_CHARS, True) or kb.chars
+    kb.brute.tables = conf.hashDB.retrieve(HASHDB_KEYS.KB_BRUTE_TABLES, True) or kb.brute.tables
+    kb.brute.columns = conf.hashDB.retrieve(HASHDB_KEYS.KB_BRUTE_COLUMNS, True) or kb.brute.columns
 
 def __setOutputResume():
     """

lib/techniques/brute/use.py

@@ -11,7 +11,6 @@ import threading
 import time
 
 from lib.core.common import clearConsoleLine
-from lib.core.common import dataToSessionFile
 from lib.core.common import dataToStdout
 from lib.core.common import filterListValue
 from lib.core.common import getFileItems
@@ -28,6 +27,7 @@ from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.enums import DBMS
+from lib.core.enums import HASHDB_KEYS
 from lib.core.exception import sqlmapDataException
 from lib.core.exception import sqlmapMissingMandatoryOptionException
 from lib.core.exception import sqlmapThreadException
@@ -103,10 +103,6 @@ def tableExists(tableFile, regex=None):
                 threadData.shared.outputs.append(table)
                 threadData.shared.unique.add(table.lower())
 
-                dataToSessionFile("[%s][%s][%s][TABLE_EXISTS][%s]\n" % (conf.url,\
-                  kb.injection.place, safeFormatString(conf.parameters[kb.injection.place]),\
-                  safeFormatString(fullTableName)))
-
                 if conf.verbose in (1, 2):
                     clearConsoleLine(True)
                     infoMsg = "[%s] [INFO] retrieved: %s\r\n" % (time.strftime("%X"), table)
@@ -139,6 +135,12 @@ def tableExists(tableFile, regex=None):
             else:
                 kb.data.cachedTables[conf.db].append(item)
 
+    for _ in map(lambda x: (conf.db, x), threadData.shared.outputs):
+        if _ not in kb.brute.tables:
+            kb.brute.tables.append(_)
+
+    conf.hashDB.write(HASHDB_KEYS.KB_BRUTE_TABLES, kb.brute.tables, True)
+
     return kb.data.cachedTables
 
 def columnExists(columnFile, regex=None):
@@ -230,10 +232,12 @@ def columnExists(columnFile, regex=None):
             else:
                 columns[column] = 'non-numeric'
 
-            dataToSessionFile("[%s][%s][%s][COLUMN_EXISTS][%s|%s %s]\n" % (conf.url, kb.injection.place,\
-              safeFormatString(conf.parameters[kb.injection.place]), safeFormatString(table),\
-              safeFormatString(column), safeFormatString(columns[column])))
-
         kb.data.cachedColumns[conf.db] = {conf.tbl: columns}
 
+        for _ in map(lambda x: (conf.db, conf.tbl, x[0], x[1]), columns.items()):
+            if _ not in kb.brute.columns:
+                kb.brute.columns.append(_)
+
+        conf.hashDB.write(HASHDB_KEYS.KB_BRUTE_COLUMNS, kb.brute.columns, True)
+
     return kb.data.cachedColumns

lib/techniques/error/use.py

@@ -15,7 +15,6 @@ from lib.core.agent import agent
 from lib.core.common import Backend
 from lib.core.common import BigArray
 from lib.core.common import calculateDeltaSeconds
-from lib.core.common import dataToSessionFile
 from lib.core.common import dataToStdout
 from lib.core.common import extractRegexResult
 from lib.core.common import getUnicode

lib/utils/hashdb.py

@@ -11,6 +11,8 @@ import hashlib
 import sqlite3
 import threading
 
+from lib.core.common import serializeObject
+from lib.core.common import unserializeObject
 from lib.core.data import conf
 from lib.core.settings import HASHDB_FLUSH_THRESHOLD
 from lib.core.settings import UNICODE_ENCODING
@@ -51,7 +53,7 @@ class HashDB(object):
         retVal = int(hashlib.md5(key).hexdigest()[:8], 16)
         return retVal
 
-    def retrieve(self, key):
+    def retrieve(self, key, unserialize=False):
         retVal = None
         if key:
             hash_ = HashDB.hashKey(key)
@@ -66,13 +68,13 @@ class HashDB(object):
                             raise
                     else:
                         break
-        return retVal
+        return retVal if not unserialize else unserializeObject(retVal)
 
-    def write(self, key, value):
+    def write(self, key, value, serialize=False):
         if key:
             hash_ = HashDB.hashKey(key)
             self._cache_lock.acquire()
-            self._write_cache[hash_] = value
+            self._write_cache[hash_] = value if not serialize else serializeObject(value)
             self._cache_lock.release()
 
         if getCurrentThreadName() in ('0', 'MainThread'):