speed optimization and bug fix (kb.absFilePaths were not stored previously; also, they are now extracted only in heuristic phase)

lib/controller/checks.py

@@ -26,6 +26,7 @@ from lib.core.common import getSortedInjectionTests
 from lib.core.common import getUnicode
 from lib.core.common import intersect
 from lib.core.common import listToStrValue
+from lib.core.common import parseFilePaths
 from lib.core.common import popValue
 from lib.core.common import pushValue
 from lib.core.common import randomInt
@@ -589,7 +590,9 @@ def heuristicCheckSqlInjection(place, parameter):
 
     payload = "%s%s%s" % (prefix, randomStr(length=10, alphabet=['"', '\'', ')', '(']), suffix)
     payload = agent.payload(place, parameter, newValue=payload)
-    Request.queryPage(payload, place, content=True, raise404=False)
+    page, _ = Request.queryPage(payload, place, content=True, raise404=False)
+
+    parseFilePaths(page)
 
     result = wasLastRequestDBMSError()
 

lib/controller/controller.py

@@ -31,7 +31,9 @@ from lib.core.common import paramToDict
 from lib.core.common import parseTargetUrl
 from lib.core.common import randomStr
 from lib.core.common import readInput
+from lib.core.common import serializeObject
 from lib.core.common import showHttpErrorCodes
+from lib.core.common import unserializeObject
 from lib.core.convert import urlencode
 from lib.core.convert import urldecode
 from lib.core.data import conf
@@ -172,6 +174,12 @@ def __saveToSessionFile():
 
         setInjection(inj)
 
+def __saveAbsFilePaths():
+    key = "kb.absFilePaths"
+    value = unserializeObject(conf.hashDB.retrieve(key)) or set()
+    value.update(kb.absFilePaths)
+    conf.hashDB.write(key, serializeObject(value))
+
 def __saveToResultsFile():
     if not conf.resultsFP:
         return
@@ -553,6 +561,7 @@ def start():
 
                 __saveToSessionFile()
                 __saveToResultsFile()
+                __saveAbsFilePaths()
                 __showInjections()
                 __selectInjection()
 

lib/core/common.py

@@ -1390,6 +1390,23 @@ def parseUnionPage(output, expression, partial=False, condition=None, sort=True)
 
     return data
 
+def parseFilePaths(page):
+    """
+    Detect (possible) absolute system paths inside the provided page content
+    """
+
+    if page:
+        for regex in ( r" in <b>(?P<result>.*?)</b> on line",  r"(?:>|\s)(?P<result>[A-Za-z]:[\\/][\w.\\/]*)", r"(?:>|\s)(?P<result>/\w[/\w.]+)" ):
+            for match in re.finditer(regex, page):
+                absFilePath = match.group("result").strip()
+                page = page.replace(absFilePath, "")
+
+                if isWindowsDriveLetterPath(absFilePath):
+                    absFilePath = posixToNtSlashes(absFilePath)
+
+                if absFilePath not in kb.absFilePaths:
+                    kb.absFilePaths.add(absFilePath)
+
 def getDelayQuery(andCond=False):
     query = None
 
@@ -3146,3 +3163,12 @@ def executeCode(code, variables=None):
     except Exception, ex:
         errMsg = "an error occured while evaluating provided code ('%s'). " % ex
         raise sqlmapGenericException, errMsg
+
+def serializeObject(object_):
+    return pickle.dumps(object_)
+
+def unserializeObject(value):
+    retVal = None
+    if value:
+        retVal = pickle.loads(value.encode(UNICODE_ENCODING)) # pickle has problems with Unicode
+    return retVal

lib/core/target.py

@@ -17,6 +17,7 @@ from lib.core.common import dataToSessionFile
 from lib.core.common import intersect
 from lib.core.common import paramToDict
 from lib.core.common import readInput
+from lib.core.common import unserializeObject
 from lib.core.convert import urldecode
 from lib.core.data import cmdLineOptions
 from lib.core.data import conf
@@ -177,6 +178,7 @@ def __setHashDB():
     """
     Check and set the HashDB SQLite file for query resume functionality.
     """
+
     if not conf.hashDBFile:
         conf.hashDBFile = "%s%shashdb" % (conf.outputPath, os.sep)
 
@@ -191,6 +193,13 @@ def __setHashDB():
 
     conf.hashDB = HashDB(conf.hashDBFile)
 
+def __resumeHashDBValues():
+    """
+    Resume stored data values from HashDB
+    """
+
+    kb.absFilePaths = unserializeObject(conf.hashDB.retrieve("kb.absFilePaths")) or kb.absFilePaths
+
 def __setOutputResume():
     """
     Check and set the output text file and the resume functionality.
@@ -383,4 +392,5 @@ def setupTargetEnv():
     __setRequestParams()
     __setOutputResume()
     __setHashDB()
+    __resumeHashDBValues()
     __setResultsFile()

lib/request/basic.py

@@ -18,7 +18,6 @@ import zlib
 from extra.chardet import detect
 from lib.core.common import extractErrorMessage
 from lib.core.common import extractRegexResult
-from lib.core.common import getCompiledRegex
 from lib.core.common import getUnicode
 from lib.core.common import isWindowsDriveLetterPath
 from lib.core.common import posixToNtSlashes
@@ -74,23 +73,6 @@ def parseResponse(page, headers):
     if page:
         htmlParser(page)
 
-        # Detect injectable page absolute system path
-        # NOTE: this regular expression works if the remote web
-        # application is written in PHP and debug/error messages are
-        # enabled
-        for regex in ( r" in <b>(?P<result>.*?)</b> on line",  r"(?:>|\s)(?P<result>[A-Za-z]:[\\/][\w.\\/]*)", r"(?:>|\s)(?P<result>/\w[/\w.]+)" ):
-            regObj = getCompiledRegex(regex)
-
-            for match in regObj.finditer(page):
-                absFilePath = match.group("result").strip()
-                page = page.replace(absFilePath, "")
-
-                if isWindowsDriveLetterPath(absFilePath):
-                    absFilePath = posixToNtSlashes(absFilePath)
-
-                if absFilePath not in kb.absFilePaths:
-                    kb.absFilePaths.add(absFilePath)
-
 def checkCharEncoding(encoding):
     if encoding:
         encoding = encoding.lower()

lib/techniques/union/test.py

@@ -44,7 +44,6 @@ from lib.core.settings import MIN_STATISTICAL_RANGE
 from lib.core.settings import MIN_UNION_RESPONSES
 from lib.core.settings import ORDER_BY_STEP
 from lib.core.unescaper import unescaper
-from lib.parse.html import htmlParser
 from lib.request.comparison import comparison
 from lib.request.connect import Connect as Request