implementation of a new dynamic content removal engine

lib/controller/checks.py

@@ -147,65 +147,41 @@ def checkDynParam(place, parameter, value):
 
     return condition
 
-def checkDynamicContent(*pages):
+def checkDynamicContent(firstPage, secondPage):
     """
     This function checks if the provided pages have dynamic content. If they
-    are dynamic, their content differs at specific lines.
+    are dynamic, proper markings will be made.
     """
 
     infoMsg = "searching for dynamic content"
     logger.info(infoMsg)
 
-    for i in xrange(len(pages)):
+    blocks = SequenceMatcher(None, firstPage, secondPage).get_matching_blocks()
-        firstPage = pages[i]
+    kb.dynamicMarkings = []
-        linesFirst = preparePageForLineComparison(firstPage)
-        pageLinesNumber = len(linesFirst)
 
-        for j in xrange(i+1, len(pages)):
+    i = 0
-            secondPage = pages[j]
+    while i < len(blocks):
-            linesSecond = preparePageForLineComparison(secondPage)
+        block = blocks[i]
+        (_, _, length) = block
+        if length <= conf.minMatchBlock:
+            blocks.remove(block)
+        else:
+            i += 1
 
-            if pageLinesNumber == len(linesSecond):
+    if len(blocks) > 0:
-                for k in xrange(0, pageLinesNumber):
+        blocks.insert(0, None)
-                    if (linesFirst[k] != linesSecond[k]):
+        blocks.append(None)
-                        item = DynamicContentItem(k, pageLinesNumber, \
+        for i in xrange(len(blocks) - 1):
-                            linesFirst[k-1] if k > 0 else None, \
+            prefix = firstPage[blocks[i][0]:blocks[i][0] + blocks[i][2]] if blocks[i] else None
-                            linesFirst[k+1] if k < pageLinesNumber - 1 else None)
+            postfix = firstPage[blocks[i + 1][0]:blocks[i + 1][0] + blocks[i + 1][2]] if blocks[i + 1] else None
+            if prefix is None and blocks[i + 1][0] == 0:
+                continue
+            if postfix is None and (blocks[i][0] + blocks[i][2] >= len(firstPage)):
+                continue
+            kb.dynamicMarkings.append((prefix[-conf.dynMarkLength:] if prefix else None, postfix[:conf.dynMarkLength] if postfix else None))
 
-                        found = None
+    if len(kb.dynamicMarkings) > 0:
-
+        infoMsg = "dynamic content marked for removal (%d region%s)" % (len(kb.dynamicMarkings), 's' if len(kb.dynamicMarkings) > 1 else '')
-                        for other in kb.dynamicContent:
-                            found = True
-
-                            if other.pageTotal == item.pageTotal:
-                                if isinstance(other.lineNumber, int):
-                                    if other.lineNumber == item.lineNumber - 1:
-                                        other.lineNumber = [other.lineNumber, item.lineNumber]
-                                        other.lineContentAfter = item.lineContentAfter
-                                        break
-
-                                    elif other.lineNumber == item.lineNumber + 1:
-                                        other.lineNumber = [item.lineNumber, other.lineNumber]
-                                        other.lineContentBefore = item.lineContentBefore
-                                        break
-
-                                elif item.lineNumber - 1 == other.lineNumber[-1]:
-                                    other.lineNumber.append(item.lineNumber)
-                                    other.lineContentAfter = item.lineContentAfter
-                                    break
-
-                                elif item.lineNumber + 1 == other.lineNumber[0]:
-                                    other.lineNumber.insert(0, item.lineNumber)
-                                    other.lineContentBefore = item.lineContentBefore
-                                    break
-
-                            found = False
-
-                        if not found:
-                            kb.dynamicContent.append(item)
-
-    if kb.dynamicContent:
-        infoMsg = "found probably removable dynamic lines"
         logger.info(infoMsg)
 
 def checkStability():

lib/core/common.py

@@ -1079,8 +1079,6 @@ def safeStringFormat(formatStr, params):
                 if count < len(params):
                     retVal = retVal[:index] + getUnicode(params[count]) + retVal[index+2:]
                 else:
-                    import pdb
-                    pdb.set_trace()
                     raise sqlmapNoneDataException, "wrong number of parameters during string formatting"
                 count += 1
 

lib/core/option.py

@@ -1005,6 +1005,8 @@ def __setConfAttributes():
     conf.dbmsConnector    = None
     conf.dbmsHandler      = None
     conf.dumpPath         = None
+    conf.minMatchBlock    = 8
+    conf.dynMarkLength    = 32
     conf.httpHeaders      = []
     conf.hostname         = None
     conf.loggedToOut      = None
@@ -1058,7 +1060,7 @@ def __setKnowledgeBaseAttributes():
 
     kb.dep             = None
     kb.docRoot         = None
-    kb.dynamicContent  = []
+    kb.dynamicMarkings = []
     kb.errorTest       = None
     kb.headersCount    = 0
     kb.headersFp       = {}

lib/core/target.py

@@ -290,7 +290,8 @@ def initTargetEnv():
         kb.dbms            = None
         kb.dbmsDetected    = False
         kb.dbmsVersion     = [ "Unknown" ]
-        kb.errorTest       = None        
+        kb.dynamicMarkings = []
+        kb.errorTest       = None
         kb.htmlFp          = []
         kb.lastErrorPage   = None
         kb.injParameter    = None

lib/request/comparison.py

@@ -48,19 +48,14 @@ def comparison(page, headers=None, getSeqMatcher=False, pageLength=None):
         return re.search(conf.regexp, page, re.I | re.M) is not None
 
     # Dynamic content lines to be excluded before calculating page hash
-    if kb.dynamicContent:
+    for item in kb.dynamicMarkings:
-        lines = preparePageForLineComparison(page)
+        prefix, postfix = item
-        for item in kb.dynamicContent:
+        if prefix is None:
-            if len(lines) == item.pageTotal:
+            page = re.sub('(?s)^.+%s' % postfix, postfix, page)
-                before = item.lineNumber - 1 if isinstance(item.lineNumber, int) else item.lineNumber[0] - 1
+        elif postfix is None:
-                after = item.lineNumber + 1 if isinstance(item.lineNumber, int) else item.lineNumber[-1] + 1
+            page = re.sub('(?s)%s.+$' % prefix, prefix, page)
-                if (item.lineContentBefore and lines[before] != item.lineContentBefore) or (item.lineContentAfter and lines[after] != item.lineContentAfter):
+        else:
-                    continue
+            page = re.sub('(?s)%s.+%s' % (prefix, postfix), '%s%s' % (prefix, postfix), page)
-                if isinstance(item.lineNumber, int):
-                    page = page.replace(lines[item.lineNumber], '')
-                else:
-                    for i in item.lineNumber:
-                        page = page.replace(lines[i], '')
 
     if conf.seqLock:
         conf.seqLock.acquire()