From 2b64c107105743ea9b06e4b3464390da12ac8faa Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 18 Dec 2012 09:36:26 +0100 Subject: [PATCH] Patch for an Issue #304 --- lib/request/basic.py | 2 +- lib/request/comparison.py | 2 +- lib/request/connect.py | 16 ++++++++++++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/lib/request/basic.py b/lib/request/basic.py index 2ee247613..27921ce59 100644 --- a/lib/request/basic.py +++ b/lib/request/basic.py @@ -231,7 +231,7 @@ def decodePage(page, contentEncoding, contentType): kb.pageEncoding = conf.charset # can't do for all responses because we need to support binary files too - if contentType and not isinstance(page, unicode) and any(map(lambda x: x in contentType.lower(), ("text/txt", "text/raw", "text/html", "text/xml"))): + if contentType and not isinstance(page, unicode) and any(map(lambda _: _ in contentType.lower(), ("text/txt", "text/raw", "text/html", "text/xml"))): # e.g. Ãëàâà if "&#" in page: page = re.sub('&#(\d{1,3});', lambda _: chr(int(_.group(1))) if int(_.group(1)) < 256 else _.group(0), page) diff --git a/lib/request/comparison.py b/lib/request/comparison.py index a96e04909..4ac7d7b68 100644 --- a/lib/request/comparison.py +++ b/lib/request/comparison.py @@ -71,7 +71,7 @@ def _comparison(page, headers, code, getRatioValue, pageLength): return re.search(conf.regexp, rawResponse, re.I | re.M) is not None # HTTP code to match when the query is valid - if isinstance(code, int) and conf.code: + if conf.code: return conf.code == code if page: diff --git a/lib/request/connect.py b/lib/request/connect.py index 80cdbbaf3..c100b8481 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -142,7 +142,11 @@ class Connect(object): headers = conn.info() if headers and (headers.getheader(HTTPHEADER.CONTENT_ENCODING, "").lower() in ("gzip", "deflate")\ or "text" not in headers.getheader(HTTPHEADER.CONTENT_TYPE, "").lower()): - retVal = conn.read() + retVal = conn.read(MAX_CONNECTION_TOTAL_SIZE) + if len(retVal) == MAX_CONNECTION_TOTAL_SIZE: + warnMsg = "large compressed response detected. Disabling compression" + singleTimeWarnMessage(warnMsg) + kb.pageCompress = False else: while True: _ = conn.read(MAX_CONNECTION_CHUNK_SIZE) @@ -794,7 +798,15 @@ class Connect(object): pageLength = int(headers[HTTPHEADER.CONTENT_RANGE][headers[HTTPHEADER.CONTENT_RANGE].find('/') + 1:]) if not pageLength: - page, headers, code = Connect.getPage(url=uri, get=get, post=post, cookie=cookie, ua=ua, referer=referer, host=host, silent=silent, method=method, auxHeaders=auxHeaders, response=response, raise404=raise404, ignoreTimeout=timeBasedCompare) + try: + page, headers, code = Connect.getPage(url=uri, get=get, post=post, cookie=cookie, ua=ua, referer=referer, host=host, silent=silent, method=method, auxHeaders=auxHeaders, response=response, raise404=raise404, ignoreTimeout=timeBasedCompare) + except MemoryError: + page, headers, code = None, None, None + warnMsg = "site returned insanely large response" + if kb.testMode: + warnMsg += " in testing phase. This is a common " + warnMsg += "behavior in custom WAF/IDS/IPS solutions" + singleTimeWarnMessage(warnMsg) if conf.secondOrder: page, headers, code = Connect.getPage(url=conf.secondOrder, cookie=cookie, ua=ua, silent=silent, auxHeaders=auxHeaders, response=response, raise404=False, ignoreTimeout=timeBasedCompare, refreshing=True)