Added common pattern value support to bisection algorithm

lib/core/common.py

@@ -1252,22 +1252,36 @@ def goGoodSamaritan(part, prevValue, originalCharset):
     predictionSet = set()
     wildIndexes = []
     singleValue = None
+    commonPatternValue = None
+    countSingleValues = 0
 
     # If the header (e.g. Databases) we are looking for has common
     # outputs defined
     if part in kb.commonOutputs:
+        commonPartOutputs = kb.commonOutputs[part]
+        commonPatternValue = common_finder_only(prevValue, commonPartOutputs)
+
+        # If the longest common prefix is the same as previous value then
+        # do not consider it
+        if commonPatternValue and commonPatternValue == prevValue:
+            commonPatternValue = None
+
         # For each common output
-        for item in kb.commonOutputs[part]:
+        for item in commonPartOutputs:
             # Check if the common output (item) starts with prevValue
             # where prevValue is the enumerated character(s) so far
             if item.startswith(prevValue):
                 singleValue = item
+                countSingleValues += 1
 
                 if len(item) > len(prevValue):
                     char = item[len(prevValue)]
+                    predictionSet.add(char)
 
-                    if char not in predictionSet:
+        # Reset single value if there is more than one possible common
-                        predictionSet.add(char)
+        # output
+        if countSingleValues > 1:
+            singleValue = None
 
         commonCharset = []
         otherCharset = []
@@ -1282,12 +1296,9 @@ def goGoodSamaritan(part, prevValue, originalCharset):
 
         commonCharset.sort()
 
-        if len(commonCharset) > 1:
+        return singleValue, commonPatternValue, commonCharset, originalCharset
-            return None, commonCharset, otherCharset
-        else:
-            return singleValue, commonCharset, originalCharset
     else:
-        return None, None, originalCharset
+        return None, None, None, originalCharset
 
 def getCompiledRegex(regex, *args):
     """
@@ -1389,3 +1400,25 @@ class UnicodeRawConfigParser(RawConfigParser):
                         fp.write("%s = %s\n" % (key, getUnicode(value).replace('\n', '\n\t')))
 
             fp.write("\n")
+
+# http://boredzo.org/blog/archives/2007-01-06/longest-common-prefix-in-python-2
+def longest_common_prefix(*sequences):
+    if len(sequences) == 1:
+        return sequences[0]
+
+    sequences = [pair[1] for pair in sorted((len(fi), fi) for fi in sequences)]
+
+    if not sequences:
+        return None
+
+    for i, comparison_ch in enumerate(sequences[0]):
+        for fi in sequences[1:]:
+            ch = fi[i]
+
+            if ch != comparison_ch:
+                return fi[:i]
+
+    return sequences[0]
+
+def common_finder_only(initial, sequence):
+    return longest_common_prefix(*filter(lambda x: x.startswith(initial), sequence))

lib/techniques/blind/inference.py

@@ -38,6 +38,7 @@ from lib.core.convert import urlencode
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
+from lib.core.data import queries
 from lib.core.exception import sqlmapConnectionException
 from lib.core.exception import sqlmapValueException
 from lib.core.exception import sqlmapThreadException
@@ -144,7 +145,12 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
 
         return None
 
-    def getChar(idx, charTbl=asciiTbl, continuousOrder=True, expand=charsetType is None): # continuousOrder means that distance between each two neighbour's numerical values is exactly 1
+    def getChar(idx, charTbl=asciiTbl, continuousOrder=True, expand=charsetType is None):
+        """
+        continuousOrder means that distance between each two neighbour's
+        numerical values is exactly 1
+        """
+
         result = tryHint(idx)
 
         if result:
@@ -153,7 +159,8 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
         if not continuousOrder:
             originalTbl = list(charTbl)
         else:
-            shiftTable = [5, 4] # used for gradual expanding into unicode charspace
+            # Used for gradual expanding into unicode charspace
+            shiftTable = [5, 4]
 
         if len(charTbl) == 1:
             forgedPayload = safeStringFormat(payload.replace('%3E', '%3D'), (expressionUnescaped, idx, charTbl[0]))
@@ -192,7 +199,8 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
 
                 if type(charTbl) != xrange:
                     charTbl = charTbl[position:]
-                else: # xrange - extended virtual charset used for memory/space optimization
+                else:
+                    # xrange() - extended virtual charset used for memory/space optimization
                     charTbl = xrange(charTbl[position], charTbl[-1] + 1)
             else:
                 maxValue = posValue
@@ -206,9 +214,14 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                 if continuousOrder:
                     if maxValue == 1:
                         return None
-                    elif minValue == maxChar: # going beyond the original charset
+
-                        # if the original charTbl was [0,..,127] new one will be [128,..,128*16-1] or from 128 to 2047
+                    # Going beyond the original charset
-                        # and instead of making a HUGE list with all elements we use here xrange, which is a virtual list
+                    elif minValue == maxChar:
+                        # If the original charTbl was [0,..,127] new one
+                        # will be [128,..,128*16-1] or from 128 to 2047
+                        # and instead of making a HUGE list with all
+                        # elements we use here xrange, which is a virtual
+                        # list
                         if expand and shiftTable:
                             charTbl = xrange(maxChar + 1, (maxChar + 1) << shiftTable.pop())
                             maxChar = maxValue = charTbl[-1]
@@ -222,7 +235,10 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                     if minValue == maxChar or maxValue == minChar:
                         return None
 
-                    for retVal in (originalTbl[originalTbl.index(minValue)], originalTbl[originalTbl.index(minValue) + 1]): # if we are working with non-continuous set both minValue and character afterwards are possible candidates
+                    # If we are working with non-continuous elements, set
+                    # both minValue and character afterwards are possible
+                    # candidates
+                    for retVal in (originalTbl[originalTbl.index(minValue)], originalTbl[originalTbl.index(minValue) + 1]):
                         forgedPayload = safeStringFormat(payload.replace('%3E', '%3D'), (expressionUnescaped, idx, retVal))
                         queriesCount[0] += 1
                         result = Request.queryPage(urlencode(forgedPayload))
@@ -244,6 +260,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
         progress.update(index)
         progress.draw(eta)
 
+    # Go multi-threading (--threads > 1)
     if conf.threads > 1 and isinstance(length, int) and length > 1:
         value   = [ None ] * length
         index   = [ firstChar ]    # As list for python nested function scoping
@@ -386,6 +403,8 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
             dataToStdout(infoMsg)
 
         conf.seqLock = None
+
+    # No multi-threading (--threads = 1)
     else:
         index = firstChar
 
@@ -398,7 +417,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
             # the moment
             if conf.useCommonPrediction and len(finalValue) > 0 and kb.partRun is not None:
                 val = None
-                singleValue, commonCharset, otherCharset = goGoodSamaritan(kb.partRun, finalValue, asciiTbl)
+                singleValue, commonPatternValue, commonCharset, otherCharset = goGoodSamaritan(kb.partRun, finalValue, asciiTbl)
 
                 # If there is one single output in common-outputs, check
                 # it via equal against the query output
@@ -422,10 +441,26 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
 
                         break
 
+                # If there is a common pattern starting with finalValue,
+                # check it via equal against the substring-query output
+                if commonPatternValue is not None:
+                    # Substring-query containing equals commonPatternValue
+                    subquery = queries[kb.dbms].substring % (expressionUnescaped, 1, len(commonPatternValue))
+                    query = agent.prefixQuery(" %s" % safeStringFormat('AND (%s) = %s', (subquery, unescaper.unescape('\'%s\'' % commonPatternValue))))
+                    query = agent.postfixQuery(query)
+                    queriesCount[0] += 1
+                    result = Request.queryPage(urlencode(agent.payload(newValue=query)))
+
+                    # Did we have luck?
+                    if result:
+                        val = commonPatternValue[index-1:]
+                        index += len(val)-1
+
                 # Otherwise if there is no singleValue (single match from
-                # txt/common-outputs.txt) use the returned common
+                # txt/common-outputs.txt) and no commonPatternValue
-                # charset only to retrieve the query output
+                # (common pattern) use the returned common charset only
-                if commonCharset:
+                # to retrieve the query output
+                if not val and commonCharset:
                     val = getChar(index, commonCharset, False)
 
                 # If we had no luck with singleValue and common charset,