diff --git a/lib/controller/checks.py b/lib/controller/checks.py index 20ad7f552..cfa5880b7 100644 --- a/lib/controller/checks.py +++ b/lib/controller/checks.py @@ -72,6 +72,7 @@ from lib.core.settings import HEURISTIC_CHECK_ALPHABET from lib.core.settings import IDS_WAF_CHECK_PAYLOAD from lib.core.settings import IDS_WAF_CHECK_RATIO from lib.core.settings import IDS_WAF_CHECK_TIMEOUT +from lib.core.settings import MAX_DIFFLIB_SEQUENCE_LENGTH from lib.core.settings import NON_SQLI_CHECK_PREFIX_SUFFIX_LENGTH from lib.core.settings import SUHOSIN_MAX_VALUE_LENGTH from lib.core.settings import SUPPORTED_DBMS @@ -1058,12 +1059,22 @@ def checkDynamicContent(firstPage, secondPage): logger.critical(warnMsg) return - seqMatcher = getCurrentThreadData().seqMatcher - seqMatcher.set_seq1(firstPage) - seqMatcher.set_seq2(secondPage) + if firstPage and secondPage and any(len(_) > MAX_DIFFLIB_SEQUENCE_LENGTH for _ in (firstPage, secondPage)): + ratio = None + else: + try: + seqMatcher = getCurrentThreadData().seqMatcher + seqMatcher.set_seq1(firstPage) + seqMatcher.set_seq2(secondPage) + ratio = seqMatcher.quick_ratio() + except MemoryError: + ratio = None + + if ratio is None: + kb.skipSeqMatcher = True # In case of an intolerable difference turn on dynamicity removal engine - if seqMatcher.quick_ratio() <= UPPER_RATIO_BOUND: + elif ratio <= UPPER_RATIO_BOUND: findDynamicContent(firstPage, secondPage) count = 0 diff --git a/lib/core/option.py b/lib/core/option.py index dda62c5de..88d9b216e 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -1944,6 +1944,7 @@ def _setKnowledgeBaseAttributes(flushAll=True): kb.safeCharEncode = False kb.safeReq = AttribDict() kb.singleLogFlags = set() + kb.skipSeqMatcher = False kb.reduceTests = None kb.tlsSNI = {} kb.stickyDBMS = False diff --git a/lib/core/settings.py b/lib/core/settings.py index 1b0c0adac..6ebf65770 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -19,7 +19,7 @@ from lib.core.enums import OS from lib.core.revision import getRevisionNumber # sqlmap version (...) -VERSION = "1.0.6.65" +VERSION = "1.0.6.66" REVISION = getRevisionNumber() STABLE = VERSION.count('.') <= 2 VERSION_STRING = "sqlmap/%s#%s" % (VERSION, "stable" if STABLE else "dev") @@ -581,6 +581,9 @@ MAX_CONNECTION_CHUNK_SIZE = 10 * 1024 * 1024 # Maximum response total page size (trimmed if larger) MAX_CONNECTION_TOTAL_SIZE = 100 * 1024 * 1024 +# For preventing MemoryError exceptions (caused when using large sequences in difflib.SequenceMatcher) +MAX_DIFFLIB_SEQUENCE_LENGTH = 10 * 1024 * 1024 + # Maximum (multi-threaded) length of entry in bisection algorithm MAX_BISECTION_LENGTH = 50 * 1024 * 1024 diff --git a/lib/request/comparison.py b/lib/request/comparison.py index 6b2a282ae..5ca7cd771 100644 --- a/lib/request/comparison.py +++ b/lib/request/comparison.py @@ -21,6 +21,7 @@ from lib.core.settings import DEFAULT_PAGE_ENCODING from lib.core.settings import DIFF_TOLERANCE from lib.core.settings import HTML_TITLE_REGEX from lib.core.settings import MIN_RATIO +from lib.core.settings import MAX_DIFFLIB_SEQUENCE_LENGTH from lib.core.settings import MAX_RATIO from lib.core.settings import REFLECTED_VALUE_MARKER from lib.core.settings import LOWER_RATIO_BOUND @@ -54,8 +55,6 @@ def _comparison(page, headers, code, getRatioValue, pageLength): if page is None and pageLength is None: return None - count = 0 - seqMatcher = threadData.seqMatcher seqMatcher.set_seq1(kb.pageTemplate) @@ -110,59 +109,37 @@ def _comparison(page, headers, code, getRatioValue, pageLength): elif isinstance(seqMatcher.a, unicode) and isinstance(page, str): seqMatcher.a = seqMatcher.a.encode(kb.pageEncoding or DEFAULT_PAGE_ENCODING, 'ignore') - seq1, seq2 = None, None - - if conf.titles: - seq1 = extractRegexResult(HTML_TITLE_REGEX, seqMatcher.a) - seq2 = extractRegexResult(HTML_TITLE_REGEX, page) + if seqMatcher.a and page and seqMatcher.a == page: + ratio = 1 + elif kb.skipSeqMatcher or seqMatcher.a and page and any(len(_) > MAX_DIFFLIB_SEQUENCE_LENGTH for _ in (seqMatcher.a, page)): + ratio = 1.0 * len(seqMatcher.a) / len(page) + if ratio > 1: + ratio = 1. / ratio else: - seq1 = getFilteredPageContent(seqMatcher.a, True) if conf.textOnly else seqMatcher.a - seq2 = getFilteredPageContent(page, True) if conf.textOnly else page + seq1, seq2 = None, None - if seq1 is None or seq2 is None: - return None - - seq1 = seq1.replace(REFLECTED_VALUE_MARKER, "") - seq2 = seq2.replace(REFLECTED_VALUE_MARKER, "") - - while count < min(len(seq1), len(seq2)): - if seq1[count] == seq2[count]: - count += 1 + if conf.titles: + seq1 = extractRegexResult(HTML_TITLE_REGEX, seqMatcher.a) + seq2 = extractRegexResult(HTML_TITLE_REGEX, page) else: - break + seq1 = getFilteredPageContent(seqMatcher.a, True) if conf.textOnly else seqMatcher.a + seq2 = getFilteredPageContent(page, True) if conf.textOnly else page - if count: - try: - _seq1 = seq1[count:] - _seq2 = seq2[count:] - except MemoryError: - pass - else: - seq1 = _seq1 - seq2 = _seq2 + if seq1 is None or seq2 is None: + return None - while True: - try: - seqMatcher.set_seq1(seq1) - except MemoryError: - seq1 = seq1[:len(seq1) / 1024] - else: - break + seq1 = seq1.replace(REFLECTED_VALUE_MARKER, "") + seq2 = seq2.replace(REFLECTED_VALUE_MARKER, "") - while True: - try: - seqMatcher.set_seq2(seq2) - except MemoryError: - seq2 = seq2[:len(seq2) / 1024] - else: - break + seqMatcher.set_seq1(seq1) + seqMatcher.set_seq2(seq2) - ratio = round(seqMatcher.quick_ratio(), 3) + ratio = round(seqMatcher.quick_ratio(), 3) # If the url is stable and we did not set yet the match ratio and the # current injected value changes the url page content if kb.matchRatio is None: - if (count or ratio >= LOWER_RATIO_BOUND) and ratio <= UPPER_RATIO_BOUND: + if ratio >= LOWER_RATIO_BOUND and ratio <= UPPER_RATIO_BOUND: kb.matchRatio = ratio logger.debug("setting match ratio for current parameter to %.3f" % kb.matchRatio)