Definite patch for MemoryError(s) (fixes #1991)

lib/controller/checks.py

@@ -72,6 +72,7 @@ from lib.core.settings import HEURISTIC_CHECK_ALPHABET
 from lib.core.settings import IDS_WAF_CHECK_PAYLOAD
 from lib.core.settings import IDS_WAF_CHECK_RATIO
 from lib.core.settings import IDS_WAF_CHECK_TIMEOUT
+from lib.core.settings import MAX_DIFFLIB_SEQUENCE_LENGTH
 from lib.core.settings import NON_SQLI_CHECK_PREFIX_SUFFIX_LENGTH
 from lib.core.settings import SUHOSIN_MAX_VALUE_LENGTH
 from lib.core.settings import SUPPORTED_DBMS
@@ -1058,12 +1059,22 @@ def checkDynamicContent(firstPage, secondPage):
         logger.critical(warnMsg)
         return
 
-    seqMatcher = getCurrentThreadData().seqMatcher
+    if firstPage and secondPage and any(len(_) > MAX_DIFFLIB_SEQUENCE_LENGTH for _ in (firstPage, secondPage)):
-    seqMatcher.set_seq1(firstPage)
+        ratio = None
-    seqMatcher.set_seq2(secondPage)
+    else:
+        try:
+            seqMatcher = getCurrentThreadData().seqMatcher
+            seqMatcher.set_seq1(firstPage)
+            seqMatcher.set_seq2(secondPage)
+            ratio = seqMatcher.quick_ratio()
+        except MemoryError:
+            ratio = None
+
+    if ratio is None:
+        kb.skipSeqMatcher = True
 
     # In case of an intolerable difference turn on dynamicity removal engine
-    if seqMatcher.quick_ratio() <= UPPER_RATIO_BOUND:
+    elif ratio <= UPPER_RATIO_BOUND:
         findDynamicContent(firstPage, secondPage)
 
         count = 0

lib/core/option.py

@@ -1944,6 +1944,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.safeCharEncode = False
     kb.safeReq = AttribDict()
     kb.singleLogFlags = set()
+    kb.skipSeqMatcher = False
     kb.reduceTests = None
     kb.tlsSNI = {}
     kb.stickyDBMS = False

lib/core/settings.py

@@ -19,7 +19,7 @@ from lib.core.enums import OS
 from lib.core.revision import getRevisionNumber
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.0.6.65"
+VERSION = "1.0.6.66"
 REVISION = getRevisionNumber()
 STABLE = VERSION.count('.') <= 2
 VERSION_STRING = "sqlmap/%s#%s" % (VERSION, "stable" if STABLE else "dev")
@@ -581,6 +581,9 @@ MAX_CONNECTION_CHUNK_SIZE = 10 * 1024 * 1024
 # Maximum response total page size (trimmed if larger)
 MAX_CONNECTION_TOTAL_SIZE = 100 * 1024 * 1024
 
+# For preventing MemoryError exceptions (caused when using large sequences in difflib.SequenceMatcher)
+MAX_DIFFLIB_SEQUENCE_LENGTH = 10 * 1024 * 1024
+
 # Maximum (multi-threaded) length of entry in bisection algorithm
 MAX_BISECTION_LENGTH = 50 * 1024 * 1024
 

lib/request/comparison.py

@@ -21,6 +21,7 @@ from lib.core.settings import DEFAULT_PAGE_ENCODING
 from lib.core.settings import DIFF_TOLERANCE
 from lib.core.settings import HTML_TITLE_REGEX
 from lib.core.settings import MIN_RATIO
+from lib.core.settings import MAX_DIFFLIB_SEQUENCE_LENGTH
 from lib.core.settings import MAX_RATIO
 from lib.core.settings import REFLECTED_VALUE_MARKER
 from lib.core.settings import LOWER_RATIO_BOUND
@@ -54,8 +55,6 @@ def _comparison(page, headers, code, getRatioValue, pageLength):
     if page is None and pageLength is None:
         return None
 
-    count = 0
-
     seqMatcher = threadData.seqMatcher
     seqMatcher.set_seq1(kb.pageTemplate)
 
@@ -110,59 +109,37 @@ def _comparison(page, headers, code, getRatioValue, pageLength):
         elif isinstance(seqMatcher.a, unicode) and isinstance(page, str):
             seqMatcher.a = seqMatcher.a.encode(kb.pageEncoding or DEFAULT_PAGE_ENCODING, 'ignore')
 
-        seq1, seq2 = None, None
+        if seqMatcher.a and page and seqMatcher.a == page:
-
+            ratio = 1
-        if conf.titles:
+        elif kb.skipSeqMatcher or seqMatcher.a and page and any(len(_) > MAX_DIFFLIB_SEQUENCE_LENGTH for _ in (seqMatcher.a, page)):
-            seq1 = extractRegexResult(HTML_TITLE_REGEX, seqMatcher.a)
+            ratio = 1.0 * len(seqMatcher.a) / len(page)
-            seq2 = extractRegexResult(HTML_TITLE_REGEX, page)
+            if ratio > 1:
+                ratio = 1. / ratio
         else:
-            seq1 = getFilteredPageContent(seqMatcher.a, True) if conf.textOnly else seqMatcher.a
+            seq1, seq2 = None, None
-            seq2 = getFilteredPageContent(page, True) if conf.textOnly else page
 
-        if seq1 is None or seq2 is None:
+            if conf.titles:
-            return None
+                seq1 = extractRegexResult(HTML_TITLE_REGEX, seqMatcher.a)
-
+                seq2 = extractRegexResult(HTML_TITLE_REGEX, page)
-        seq1 = seq1.replace(REFLECTED_VALUE_MARKER, "")
-        seq2 = seq2.replace(REFLECTED_VALUE_MARKER, "")
-
-        while count < min(len(seq1), len(seq2)):
-            if seq1[count] == seq2[count]:
-                count += 1
             else:
-                break
+                seq1 = getFilteredPageContent(seqMatcher.a, True) if conf.textOnly else seqMatcher.a
+                seq2 = getFilteredPageContent(page, True) if conf.textOnly else page
 
-        if count:
+            if seq1 is None or seq2 is None:
-            try:
+                return None
-                _seq1 = seq1[count:]
-                _seq2 = seq2[count:]
-            except MemoryError:
-                pass
-            else:
-                seq1 = _seq1
-                seq2 = _seq2
 
-        while True:
+            seq1 = seq1.replace(REFLECTED_VALUE_MARKER, "")
-            try:
+            seq2 = seq2.replace(REFLECTED_VALUE_MARKER, "")
-                seqMatcher.set_seq1(seq1)
-            except MemoryError:
-                seq1 = seq1[:len(seq1) / 1024]
-            else:
-                break
 
-        while True:
+            seqMatcher.set_seq1(seq1)
-            try:
+            seqMatcher.set_seq2(seq2)
-                seqMatcher.set_seq2(seq2)
-            except MemoryError:
-                seq2 = seq2[:len(seq2) / 1024]
-            else:
-                break
 
-        ratio = round(seqMatcher.quick_ratio(), 3)
+            ratio = round(seqMatcher.quick_ratio(), 3)
 
     # If the url is stable and we did not set yet the match ratio and the
     # current injected value changes the url page content
     if kb.matchRatio is None:
-        if (count or ratio >= LOWER_RATIO_BOUND) and ratio <= UPPER_RATIO_BOUND:
+        if ratio >= LOWER_RATIO_BOUND and ratio <= UPPER_RATIO_BOUND:
             kb.matchRatio = ratio
             logger.debug("setting match ratio for current parameter to %.3f" % kb.matchRatio)