refactoring, code clearing and removal of obsolete switch --longest-common

lib/controller/checks.py

@@ -48,9 +48,6 @@ from lib.core.exception import sqlmapGenericException
 from lib.core.exception import sqlmapNoneDataException
 from lib.core.exception import sqlmapUserQuitException
 from lib.core.session import setDynamicMarkings
-from lib.core.session import setString
-from lib.core.session import setRegexp
-from lib.core.session import setTextOnly
 from lib.core.settings import CONSTANT_RATIO
 from lib.core.settings import UPPER_RATIO_BOUND
 from lib.core.unescaper import unescaper
@@ -424,9 +421,14 @@ def checkSqlInjection(place, parameter, value):
                         injection.data[stype].where = where
                         injection.data[stype].vector = vector
                         injection.data[stype].comment = comment
-                        injection.data[stype].matchRatio = kb.matchRatio
                         injection.data[stype].templatePayload = templatePayload
 
+                        injection.data[stype].conf = advancedDict()
+                        injection.data[stype].conf.matchRatio = kb.matchRatio
+                        injection.data[stype].conf.textOnly = conf.textOnly
+                        injection.data[stype].conf.string = conf.string
+                        injection.data[stype].conf.regexp = conf.regexp
+
                         if hasattr(test, "details"):
                             for detailKey, detailValue in test.details.items():
                                 if detailKey == "dbms" and injection.dbms is None:
@@ -585,12 +587,6 @@ def checkDynamicContent(firstPage, secondPage):
         logger.debug(debugMsg)
         return
 
-    if conf.longestCommon:
-        debugMsg  = "dynamic content checking skipped "
-        debugMsg += "because longest common comparison used"
-        logger.debug(debugMsg)
-        return
-
     conf.seqMatcher.set_seq1(firstPage)
     conf.seqMatcher.set_seq2(secondPage)
 
@@ -608,7 +604,6 @@ def checkDynamicContent(firstPage, secondPage):
                 logger.warn(warnMsg)
 
                 conf.textOnly = True
-                setTextOnly()
                 return
 
             warnMsg = "target url is heavily dynamic"
@@ -677,7 +672,6 @@ def checkStability():
 
             if test:
                 conf.string = test
-                setString()
 
                 if kb.nullConnection:
                     debugMsg  = "turning off NULL connection "
@@ -695,7 +689,6 @@ def checkStability():
 
             if test:
                 conf.regex = test
-                setRegexp()
 
                 if kb.nullConnection:
                     debugMsg  = "turning off NULL connection "
@@ -709,7 +702,6 @@ def checkStability():
 
         elif test and test[0] in ("t", "T"):
             conf.textOnly = True
-            setTextOnly()
 
             if kb.nullConnection:
                 debugMsg  = "turning off NULL connection "
@@ -727,24 +719,13 @@ def checkString():
     if not conf.string:
         return True
 
-    condition = (
-                  kb.resumedQueries.has_key(conf.url) and
-                  kb.resumedQueries[conf.url].has_key("String") and
-                  kb.resumedQueries[conf.url]["String"][:-1] == conf.string
-                )
-
-    if condition:
-        return True
-
     infoMsg  = "testing if the provided string is within the "
     infoMsg += "target URL page content"
     logger.info(infoMsg)
 
     page, _ = Request.queryPage(content=True)
 
-    if conf.string in page:
+    if conf.string not in page:
-        setString()
-    else:
         warnMsg  = "you provided '%s' as the string to " % conf.string
         warnMsg += "match, but such a string is not within the target "
         warnMsg += "URL page content original request, sqlmap will "
@@ -757,24 +738,13 @@ def checkRegexp():
     if not conf.regexp:
         return True
 
-    condition = (
-                  kb.resumedQueries.has_key(conf.url) and
-                  kb.resumedQueries[conf.url].has_key("Regular expression") and
-                  kb.resumedQueries[conf.url]["Regular expression"][:-1] == conf.regexp
-                )
-
-    if condition:
-        return True
-
     infoMsg  = "testing if the provided regular expression matches within "
     infoMsg += "the target URL page content"
     logger.info(infoMsg)
 
     page, _ = Request.queryPage(content=True)
 
-    if re.search(conf.regexp, page, re.I | re.M):
+    if not re.search(conf.regexp, page, re.I | re.M):
-        setRegexp()
-    else:
         warnMsg  = "you provided '%s' as the regular expression to " % conf.regexp
         warnMsg += "match, but such a regular expression does not have any "
         warnMsg += "match within the target URL page content, sqlmap "

lib/core/common.py

@@ -1943,7 +1943,20 @@ def initTechnique(technique=None):
 
         if data:
             kb.pageTemplate, kb.errorIsNone = getPageTemplate(data.templatePayload, kb.injection.place)
-            kb.matchRatio = data.matchRatio
+
+            kb.matchRatio = data.conf.matchRatio
+            if data.conf.textOnly:
+                conf.textOnly = True
+                debugMsg = "restoring switch --text-only"
+                logger.debug(debugMsg)
+            if data.conf.string:
+                conf.string = data.conf.string
+                debugMsg = "restoring option --string '%s'" % data.conf.string
+                logger.debug(debugMsg)
+            if data.conf.regexp:
+                conf.regexp = data.conf.regexp
+                debugMsg = "restoring option --regexp '%s'" % data.conf.regexp
+                logger.debug(debugMsg)
         else:
             warnMsg = "there is no injection data available for technique "
             warnMsg += "'%s'" % enumValueToNameLookup(PAYLOAD.TECHNIQUE, technique)

lib/core/datatype.py

@@ -80,3 +80,4 @@ class injectionDict(advancedDict):
         self.dbms = None
         self.dbms_version = None
         self.os = None
+

lib/core/option.py

@@ -1045,7 +1045,7 @@ def __cleanupOptions():
     if conf.optimize:
         #conf.predictOutput = True
         conf.keepAlive = True
-        conf.nullConnection = not (conf.textOnly or conf.longestCommon)
+        conf.nullConnection = not conf.textOnly
         conf.threads = 4 if conf.threads < 2 else conf.threads
 
     if conf.realTest:
@@ -1320,10 +1320,6 @@ def __basicOptionValidation():
         errMsg = "switch --text-only is incompatible with switch --null-connection"
         raise sqlmapSyntaxException, errMsg
 
-    if conf.longestCommon and conf.nullConnection:
-        errMsg = "switch --longest-common is incompatible with switch --null-connection"
-        raise sqlmapSyntaxException, errMsg
-
     if conf.data and conf.nullConnection:
         errMsg = "switch --data is incompatible with switch --null-connection"
         raise sqlmapSyntaxException, errMsg

lib/core/optiondict.py

@@ -70,8 +70,7 @@ optDict = {
                                "eString":           "string",
                                "eRegexp":           "string",
                                "thold":             "float",
-                               "textOnly":          "boolean",
+                               "textOnly":          "boolean"
-                               "longestCommon":     "boolean"
                              },
 
             "Techniques":    {

lib/core/session.py

@@ -42,48 +42,6 @@ def unSafeFormatString(value):
         retVal = retVal.replace("__LEFT_SQUARE_BRACKET__", "[").replace("__RIGHT_SQUARE_BRACKET__", "]")
     return retVal
 
-def setTextOnly():
-    """
-    Save text only option to session file.
-    """
-
-    condition = (
-                  not kb.resumedQueries or ( kb.resumedQueries.has_key(conf.url) and
-                  not kb.resumedQueries[conf.url].has_key("Text only") )
-                )
-
-    if condition:
-        dataToSessionFile("[%s][None][None][Text only][True]\n" % conf.url)
-
-    kb.originalPage = getFilteredPageContent(kb.originalPage)
-    kb.pageTemplates.clear()
-
-def setString():
-    """
-    Save string to match in session file.
-    """
-
-    condition = (
-                  not kb.resumedQueries or ( kb.resumedQueries.has_key(conf.url) and
-                  not kb.resumedQueries[conf.url].has_key("String") )
-                )
-
-    if condition:
-        dataToSessionFile("[%s][None][None][String][%s]\n" % (conf.url, safeFormatString(conf.string)))
-
-def setRegexp():
-    """
-    Save regular expression to match in session file.
-    """
-
-    condition = (
-                  not kb.resumedQueries or ( kb.resumedQueries.has_key(conf.url) and
-                  not kb.resumedQueries[conf.url].has_key("Regular expression") )
-                )
-
-    if condition:
-        dataToSessionFile("[%s][None][None][Regular expression][%s]\n" % (conf.url, safeFormatString(conf.regexp)))
-
 def setInjection(inj):
     """
     Save information retrieved about injection place and parameter in the
@@ -202,23 +160,7 @@ def setRemoteTempPath():
         dataToSessionFile("[%s][%s][%s][Remote temp path][%s]\n" % (conf.url, kb.injection.place, safeFormatString(conf.parameters[kb.injection.place]), safeFormatString(conf.tmpPath)))
 
 def resumeConfKb(expression, url, value):
-    if expression == "Text only" and url == conf.url:
+    if expression == "String" and url == conf.url:
-        value = unSafeFormatString(value[:-1])
-
-        logMsg = "resuming text only option '%s' from session file" % value
-        logger.info(logMsg)
-
-        if value and not conf.textOnly:
-            message = "you did not turned on --text-only switch this time "
-            message += "which could potentially lead to different "
-            message += "and/or unstable results. "
-            message += "Do you want to turn it on? [Y/n] "
-            test = readInput(message, default="Y")
-
-            if not test or test[0] in ("y", "Y"):
-                conf.textOnly = value
-
-    elif expression == "String" and url == conf.url:
         string = unSafeFormatString(value[:-1])
 
         logMsg = "resuming string match '%s' from session file" % string

lib/core/target.py

@@ -30,7 +30,6 @@ from lib.core.exception import sqlmapSyntaxException
 from lib.core.option import __setDBMS
 from lib.core.option import __setKnowledgeBaseAttributes
 from lib.core.session import resumeConfKb
-from lib.core.session import setTextOnly
 from lib.core.xmldump import dumper as xmldumper
 from lib.request.connect import Connect as Request
 
@@ -265,13 +264,6 @@ def __createTargetDirs():
     __createFilesDir()
     __configureDumper()
 
-def __saveSwitches():
-    """
-    Store critical switches to the session file.
-    """
-    if conf.textOnly:
-        setTextOnly()
-
 def __restoreCmdLineOptions():
     """
     Restore command line options that could be possibly 
@@ -302,4 +294,3 @@ def setupTargetEnv():
     __createTargetDirs()
     __setRequestParams()
     __setOutputResume()
-    __saveSwitches()

lib/parse/cmdline.py

@@ -214,10 +214,6 @@ def cmdLineParser():
                              action="store_true", default=False,
                              help="Compare pages based only on their textual content")
 
-        detection.add_option("--longest-common", dest="longestCommon",
-                             action="store_true", default=False,
-                             help="Compare pages based on their longest common match")
-
 
         # Techniques options
         techniques = OptionGroup(parser, "Techniques", "These options can "

lib/request/comparison.py

@@ -11,6 +11,7 @@ import re
 
 from difflib import SequenceMatcher
 
+from lib.core.common import getFilteredPageContent
 from lib.core.common import removeDynamicContent
 from lib.core.common import wasLastRequestDBMSError
 from lib.core.common import wasLastRequestHTTPError
@@ -63,7 +64,7 @@ def comparison(page, headers=None, getSeqMatcher=False, pageLength=None):
             return None
 
         # Dynamic content lines to be excluded before comparison
-        if not kb.nullConnection and not conf.longestCommon:
+        if not kb.nullConnection:
             page = removeDynamicContent(page)
             conf.seqMatcher.set_seq1(removeDynamicContent(kb.pageTemplate))
 
@@ -73,12 +74,10 @@ def comparison(page, headers=None, getSeqMatcher=False, pageLength=None):
     if kb.locks.seqLock:
         kb.locks.seqLock.acquire()
 
-    if conf.longestCommon:
+    if conf.textOnly:
-        (firstPage, secondPage) = (conf.seqMatcher.a, page)
+        (conf.seqMatcher.a, page) = map(getFilteredPageContent, (conf.seqMatcher.a, page))
-        match = SequenceMatcher(None, firstPage, secondPage).find_longest_match(0, len(firstPage), 0, len(secondPage))
-        ratio = round(SequenceMatcher(None, firstPage[match[0]:match[0]+match[2]], secondPage[match[1]:match[1]+match[2]]).ratio(), 3)
 
-    elif not conf.eRegexp and not conf.eString and kb.nullConnection and pageLength:
+    if not conf.eRegexp and not conf.eString and kb.nullConnection and pageLength:
         ratio = 1. * pageLength / len(conf.seqMatcher.a)
 
         if ratio > 1.:

lib/request/connect.py

@@ -458,9 +458,6 @@ class Connect:
 
         threadData.lastQueryDuration = calculateDeltaSeconds(start)
 
-        if conf.textOnly:
-            page = getFilteredPageContent(page)
-
         if kb.testMode:
             kb.testQueryCount += 1
 

sqlmap.conf

@@ -236,10 +236,6 @@ thold =
 # Valid: True or False
 textOnly = False
 
-# Compare pages based on their longest common match
-# Valid: True or False
-longestCommon = False
-
 
 # These options can be used to test for specific SQL injection technique
 # or to use one of them to exploit the affected parameter(s) rather than