From f382443ddd48a8a7020b2a6ec28afe6d617a7340 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 12 Nov 2019 22:51:11 +0100 Subject: [PATCH] Minor patch for crawling --- lib/core/common.py | 2 +- lib/core/settings.py | 2 +- lib/request/basic.py | 7 ++++--- lib/request/connect.py | 4 ++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/core/common.py b/lib/core/common.py index bc1931461..f1a44af0e 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -2762,7 +2762,7 @@ def findMultipartPostBoundary(post): return retVal -def urldecode(value, encoding=None, unsafe="%%&=;+%s" % CUSTOM_INJECTION_MARK_CHAR, convall=False, spaceplus=True): +def urldecode(value, encoding=None, unsafe="%%?&=;+%s" % CUSTOM_INJECTION_MARK_CHAR, convall=False, spaceplus=True): """ URL decodes given value diff --git a/lib/core/settings.py b/lib/core/settings.py index e0610bff1..05a4dca52 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from lib.core.enums import OS from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.3.11.32" +VERSION = "1.3.11.33" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/request/basic.py b/lib/request/basic.py index 554c4ac0b..6cdb6e793 100644 --- a/lib/request/basic.py +++ b/lib/request/basic.py @@ -267,7 +267,7 @@ def getHeuristicCharEncoding(page): return retVal -def decodePage(page, contentEncoding, contentType): +def decodePage(page, contentEncoding, contentType, percentDecode=True): """ Decode compressed/charset HTTP response @@ -340,8 +340,9 @@ def decodePage(page, contentEncoding, contentType): page = re.sub(b"&#(\\d{1,3});", lambda _: six.int2byte(int(_.group(1))) if int(_.group(1)) < 256 else _.group(0), page) # e.g. %20%28%29 - if b"%" in page: - page = re.sub(b"%([0-9a-fA-F]{2})", lambda _: decodeHex(_.group(1)), page) + if percentDecode: + if b"%" in page: + page = re.sub(b"%([0-9a-fA-F]{2})", lambda _: decodeHex(_.group(1)), page) # e.g. & page = re.sub(b"&([^;]+);", lambda _: six.int2byte(HTML_ENTITIES[getText(_.group(1))]) if HTML_ENTITIES.get(getText(_.group(1)), 256) < 256 else _.group(0), page) diff --git a/lib/request/connect.py b/lib/request/connect.py index 3594e0d33..af2ddb299 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -550,7 +550,7 @@ class Connect(object): code = None responseHeaders = {} - page = decodePage(page, responseHeaders.get(HTTP_HEADER.CONTENT_ENCODING), responseHeaders.get(HTTP_HEADER.CONTENT_TYPE)) + page = decodePage(page, responseHeaders.get(HTTP_HEADER.CONTENT_ENCODING), responseHeaders.get(HTTP_HEADER.CONTENT_TYPE), percentDecode=not crawling) status = getUnicode(conn.msg) if conn and getattr(conn, "msg", None) else None kb.connErrorCounter = 0 @@ -628,7 +628,7 @@ class Connect(object): responseHeaders = ex.info() responseHeaders[URI_HTTP_HEADER] = ex.geturl() patchHeaders(responseHeaders) - page = decodePage(page, responseHeaders.get(HTTP_HEADER.CONTENT_ENCODING), responseHeaders.get(HTTP_HEADER.CONTENT_TYPE)) + page = decodePage(page, responseHeaders.get(HTTP_HEADER.CONTENT_ENCODING), responseHeaders.get(HTTP_HEADER.CONTENT_TYPE), percentDecode=not crawling) except socket.timeout: warnMsg = "connection timed out while trying " warnMsg += "to get error page information (%d)" % ex.code