From 8df4cc3983d499dac9fca7c677d764a8943f2a86 Mon Sep 17 00:00:00 2001 From: Louis-Philippe Huberdeau Date: Fri, 23 Jun 2017 09:44:33 -0400 Subject: [PATCH 1/5] Adding initial hook to receive the request/response pairs --- lib/core/common.py | 3 +++ lib/core/option.py | 8 ++++++++ lib/core/optiondict.py | 1 + lib/core/threads.py | 2 ++ lib/parse/cmdline.py | 4 ++++ lib/utils/collect.py | 32 ++++++++++++++++++++++++++++++++ 6 files changed, 50 insertions(+) create mode 100644 lib/utils/collect.py diff --git a/lib/core/common.py b/lib/core/common.py index 698dfba69..709a40ee7 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -2599,6 +2599,9 @@ def logHTTPTraffic(requestLogMsg, responseLogMsg): """ Logs HTTP traffic to the output file """ + threadData = getCurrentThreadData() + assert threadData.requestCollector is not None, "Request collector should be initialized by now" + threadData.requestCollector.collectRequest(requestLogMsg, responseLogMsg) if not conf.trafficFile: return diff --git a/lib/core/option.py b/lib/core/option.py index 5ae0f5aca..c0c59eb05 100755 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -149,6 +149,7 @@ from lib.request.pkihandler import HTTPSPKIAuthHandler from lib.request.rangehandler import HTTPRangeHandler from lib.request.redirecthandler import SmartRedirectHandler from lib.request.templates import getPageTemplate +from lib.utils.collect import RequestCollectorFactory from lib.utils.crawler import crawl from lib.utils.deps import checkDependencies from lib.utils.search import search @@ -1844,6 +1845,7 @@ def _setConfAttributes(): conf.scheme = None conf.tests = [] conf.trafficFP = None + conf.requestCollectorFactory = None conf.wFileType = None def _setKnowledgeBaseAttributes(flushAll=True): @@ -2228,6 +2230,11 @@ def _setTrafficOutputFP(): conf.trafficFP = openFile(conf.trafficFile, "w+") +def _setupRequestCollector(): + conf.requestCollectorFactory = RequestCollectorFactory(collect=conf.collectRequests) + threadData = getCurrentThreadData() + threadData.requestCollector = conf.requestCollectorFactory.create() + def _setDNSServer(): if not conf.dnsDomain: return @@ -2604,6 +2611,7 @@ def init(): _setTamperingFunctions() _setWafFunctions() _setTrafficOutputFP() + _setupRequestCollector() _resolveCrossReferences() _checkWebSocket() diff --git a/lib/core/optiondict.py b/lib/core/optiondict.py index fd85eff38..db7834cf9 100644 --- a/lib/core/optiondict.py +++ b/lib/core/optiondict.py @@ -197,6 +197,7 @@ optDict = { "binaryFields": "string", "charset": "string", "checkInternet": "boolean", + "collectRequests": "string", "crawlDepth": "integer", "crawlExclude": "string", "csvDel": "string", diff --git a/lib/core/threads.py b/lib/core/threads.py index 8f89fb1b8..b3566b955 100644 --- a/lib/core/threads.py +++ b/lib/core/threads.py @@ -38,6 +38,8 @@ class _ThreadData(threading.local): Resets thread data model """ + self.requestCollector = None + self.disableStdOut = False self.hashDBCursor = None self.inTransaction = False diff --git a/lib/parse/cmdline.py b/lib/parse/cmdline.py index e02cac62f..3e97bfb56 100644 --- a/lib/parse/cmdline.py +++ b/lib/parse/cmdline.py @@ -631,6 +631,10 @@ def cmdLineParser(argv=None): action="store_true", help="Never ask for user input, use the default behaviour") + general.add_option("--collect-requests", dest="collectRequests", + action="store_true", + help="Collect requests in HAR format") + general.add_option("--binary-fields", dest="binaryFields", help="Result fields having binary values (e.g. \"digest\")") diff --git a/lib/utils/collect.py b/lib/utils/collect.py new file mode 100644 index 000000000..32b42d595 --- /dev/null +++ b/lib/utils/collect.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/) +See the file 'doc/COPYING' for copying permission +""" + +from lib.core.data import logger + + +class RequestCollectorFactory: + + def __init__(self, collect=False): + self.collect = collect + + def create(self): + collector = RequestCollector() + + if not self.collect: + collector.collectRequest = self._noop + + return collector + + @staticmethod + def _noop(*args, **kwargs): + pass + + +class RequestCollector: + + def collectRequest(self, requestMessage, responseMessage): + logger.info("Received request/response: %s/%s", len(requestMessage), len(responseMessage)) From 0d756a88239f3525c9c6c576f983c7bd451ee35f Mon Sep 17 00:00:00 2001 From: Louis-Philippe Huberdeau Date: Fri, 23 Jun 2017 11:50:21 -0400 Subject: [PATCH 2/5] Parse request data and convert to HAR, include in injection data --- lib/controller/checks.py | 2 + lib/utils/collect.py | 177 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 178 insertions(+), 1 deletion(-) diff --git a/lib/controller/checks.py b/lib/controller/checks.py index 3e4698c5c..a1f219baa 100644 --- a/lib/controller/checks.py +++ b/lib/controller/checks.py @@ -117,6 +117,7 @@ def checkSqlInjection(place, parameter, value): while tests: test = tests.pop(0) + threadData.requestCollector.reset() try: if kb.endDetection: @@ -700,6 +701,7 @@ def checkSqlInjection(place, parameter, value): injection.data[stype].matchRatio = kb.matchRatio injection.data[stype].trueCode = trueCode injection.data[stype].falseCode = falseCode + injection.data[stype].collectedRequests = threadData.requestCollector.obtain() injection.conf.textOnly = conf.textOnly injection.conf.titles = conf.titles diff --git a/lib/utils/collect.py b/lib/utils/collect.py index 32b42d595..a913b63a4 100644 --- a/lib/utils/collect.py +++ b/lib/utils/collect.py @@ -5,7 +5,11 @@ Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/) See the file 'doc/COPYING' for copying permission """ +from BaseHTTPServer import BaseHTTPRequestHandler +from StringIO import StringIO + from lib.core.data import logger +from lib.core.settings import VERSION class RequestCollectorFactory: @@ -18,6 +22,8 @@ class RequestCollectorFactory: if not self.collect: collector.collectRequest = self._noop + else: + logger.info("Request collection is enabled.") return collector @@ -28,5 +34,174 @@ class RequestCollectorFactory: class RequestCollector: + def __init__(self): + self.reset() + def collectRequest(self, requestMessage, responseMessage): - logger.info("Received request/response: %s/%s", len(requestMessage), len(responseMessage)) + self.messages.append(RawPair(requestMessage, responseMessage)) + + def reset(self): + self.messages = [] + + def obtain(self): + if self.messages: + return {"log": { + "version": "1.2", + "creator": {"name": "SQLMap", "version": VERSION}, + "entries": [pair.toEntry().toDict() for pair in self.messages], + }} + + +class RawPair: + + def __init__(self, request, response): + self.request = request + self.response = response + + def toEntry(self): + return Entry(request=Request.parse(self.request), + response=Response.parse(self.response)) + + +class Entry: + + def __init__(self, request, response): + self.request = request + self.response = response + + def toDict(self): + return { + "request": self.request.toDict(), + "response": self.response.toDict(), + } + + +class Request: + + def __init__(self, method, path, httpVersion, headers, postBody=None, raw=None, comment=None): + self.method = method + self.path = path + self.httpVersion = httpVersion + self.headers = headers or {} + self.postBody = postBody + self.comment = comment + self.raw = raw + + @classmethod + def parse(cls, raw): + request = HTTPRequest(raw) + return cls(method=request.command, + path=request.path, + httpVersion=request.request_version, + headers=request.headers, + postBody=request.rfile.read(), + comment=request.comment, + raw=raw) + + @property + def url(self): + host = self.headers.get('Host', 'unknown') + return "http://%s%s" % (host, self.path) + + def toDict(self): + out = { + "method": self.method, + "url": self.url, + "headers": [dict(name=key, value=value) for key, value in self.headers.items()], + "comment": self.comment, + "_raw": self.raw, + } + if self.postBody: + contentType = self.headers.get('Content-Type') + out["postData"] = { + "mimeType": contentType, + "text": self.postBody, + } + return out + + +class Response: + + def __init__(self): + pass + + @classmethod + def parse(cls, raw): + return cls() + + def toDict(self): + return { + } + + +class HTTPRequest(BaseHTTPRequestHandler): + # Original source: + # https://stackoverflow.com/questions/4685217/parse-raw-http-headers + + def __init__(self, request_text): + self.comment = None + self.rfile = StringIO(request_text) + self.raw_requestline = self.rfile.readline() + + if self.raw_requestline.startswith("HTTP request ["): + self.comment = self.raw_requestline + self.raw_requestline = self.rfile.readline() + + self.error_code = self.error_message = None + self.parse_request() + + def send_error(self, code, message): + self.error_code = code + self.error_message = message + + +if __name__ == '__main__': + import unittest + + class RequestParseTest(unittest.TestCase): + + def test_basic_request(self): + req = Request.parse("GET /test HTTP/1.0\r\n" + "Host: test\r\n" + "Connection: close") + self.assertEqual("GET", req.method) + self.assertEqual("/test", req.path) + self.assertEqual("close", req.headers['Connection']) + self.assertEqual("test", req.headers['Host']) + self.assertEqual("HTTP/1.0", req.httpVersion) + + def test_with_request_as_logged_by_sqlmap(self): + raw = "HTTP request [#75]:\nPOST /create.php HTTP/1.1\nHost: 240.0.0.2\nAccept-encoding: gzip,deflate\nCache-control: no-cache\nContent-type: application/x-www-form-urlencoded; charset=utf-8\nAccept: */*\nUser-agent: Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.215 Safari/534.10\nCookie: PHPSESSID=65c4a9cfbbe91f2d975d50ce5e8d1026\nContent-length: 138\nConnection: close\n\nname=test%27%29%3BSELECT%20LIKE%28%27ABCDEFG%27%2CUPPER%28HEX%28RANDOMBLOB%28000000000%2F2%29%29%29%29--&csrfmiddlewaretoken=594d26cfa3fad\n" # noqa + req = Request.parse(raw) + self.assertEqual("POST", req.method) + self.assertEqual("138", req.headers["Content-Length"]) + self.assertIn("csrfmiddlewaretoken", req.postBody) + self.assertEqual("HTTP request [#75]:\n", req.comment) + + class RequestRenderTest(unittest.TestCase): + def test_render_get_request(self): + req = Request(method="GET", + path="/test.php", + headers={"Host": "example.com", "Content-Length": "0"}, + httpVersion="HTTP/1.1", + comment="Hello World") + out = req.toDict() + self.assertEqual("GET", out["method"]) + self.assertEqual("http://example.com/test.php", out["url"]) + self.assertIn({"name": "Host", "value": "example.com"}, out["headers"]) + self.assertEqual("Hello World", out["comment"]) + + def test_render_with_post_body(self): + req = Request(method="POST", + path="/test.php", + headers={"Host": "example.com", + "Content-Type": "application/x-www-form-urlencoded; charset=utf-8"}, + httpVersion="HTTP/1.1", + postBody="name=test&csrfmiddlewaretoken=594d26cfa3fad\n") + out = req.toDict() + self.assertEqual(out["postData"], { + "mimeType": "application/x-www-form-urlencoded; charset=utf-8", + "text": "name=test&csrfmiddlewaretoken=594d26cfa3fad\n", + }) + + unittest.main(buffer=False) From fae965f8b60097afa1f38b3dfd79e816702cc61d Mon Sep 17 00:00:00 2001 From: Louis-Philippe Huberdeau Date: Fri, 23 Jun 2017 13:28:22 -0400 Subject: [PATCH 3/5] Parse and build the response block --- lib/utils/collect.py | 93 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 4 deletions(-) diff --git a/lib/utils/collect.py b/lib/utils/collect.py index a913b63a4..7cb328883 100644 --- a/lib/utils/collect.py +++ b/lib/utils/collect.py @@ -6,7 +6,10 @@ See the file 'doc/COPYING' for copying permission """ from BaseHTTPServer import BaseHTTPRequestHandler +from httplib import HTTPResponse from StringIO import StringIO +import base64 +import re from lib.core.data import logger from lib.core.settings import VERSION @@ -122,18 +125,68 @@ class Request: class Response: - def __init__(self): - pass + extract_status = re.compile(r'\((\d{3}) (.*)\)') + + def __init__(self, httpVersion, status, statusText, headers, content, raw=None, comment=None): + self.raw = raw + self.httpVersion = httpVersion + self.status = status + self.statusText = statusText + self.headers = headers + self.content = content + self.comment = comment @classmethod def parse(cls, raw): - return cls() + altered = raw + comment = None + + if altered.startswith("HTTP response ["): + io = StringIO(raw) + first_line = io.readline() + parts = cls.extract_status.search(first_line) + status_line = "HTTP/1.0 %s %s" % (parts.group(1), parts.group(2)) + remain = io.read() + altered = status_line + "\n" + remain + comment = first_line + + response = HTTPResponse(FakeSocket(altered)) + response.begin() + return cls(httpVersion="HTTP/1.1" if response.version == 11 else "HTTP/1.0", + status=response.status, + statusText=response.reason, + headers=response.msg, + content=response.read(-1), + comment=comment, + raw=raw) def toDict(self): return { + "httpVersion": self.httpVersion, + "status": self.status, + "statusText": self.statusText, + "headers": [dict(name=key, value=value) for key, value in self.headers.items()], + "content": { + "mimeType": self.headers.get('Content-Type'), + "encoding": "base64", + "text": base64.b64encode(self.content), + }, + "comment": self.comment, + "_raw": self.raw, } +class FakeSocket: + # Original source: + # https://stackoverflow.com/questions/24728088/python-parse-http-response-string + + def __init__(self, response_text): + self._file = StringIO(response_text) + + def makefile(self, *args, **kwargs): + return self._file + + class HTTPRequest(BaseHTTPRequestHandler): # Original source: # https://stackoverflow.com/questions/4685217/parse-raw-http-headers @@ -171,7 +224,7 @@ if __name__ == '__main__': self.assertEqual("HTTP/1.0", req.httpVersion) def test_with_request_as_logged_by_sqlmap(self): - raw = "HTTP request [#75]:\nPOST /create.php HTTP/1.1\nHost: 240.0.0.2\nAccept-encoding: gzip,deflate\nCache-control: no-cache\nContent-type: application/x-www-form-urlencoded; charset=utf-8\nAccept: */*\nUser-agent: Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.215 Safari/534.10\nCookie: PHPSESSID=65c4a9cfbbe91f2d975d50ce5e8d1026\nContent-length: 138\nConnection: close\n\nname=test%27%29%3BSELECT%20LIKE%28%27ABCDEFG%27%2CUPPER%28HEX%28RANDOMBLOB%28000000000%2F2%29%29%29%29--&csrfmiddlewaretoken=594d26cfa3fad\n" # noqa + raw = "HTTP request [#75]:\nPOST /create.php HTTP/1.1\nHost: 127.0.0.1\nAccept-encoding: gzip,deflate\nCache-control: no-cache\nContent-type: application/x-www-form-urlencoded; charset=utf-8\nAccept: */*\nUser-agent: Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.215 Safari/534.10\nCookie: PHPSESSID=65c4a9cfbbe91f2d975d50ce5e8d1026\nContent-length: 138\nConnection: close\n\nname=test%27%29%3BSELECT%20LIKE%28%27ABCDEFG%27%2CUPPER%28HEX%28RANDOMBLOB%280.0.10000%2F2%29%29%29%29--&csrfmiddlewaretoken=594d26cfa3fad\n" # noqa req = Request.parse(raw) self.assertEqual("POST", req.method) self.assertEqual("138", req.headers["Content-Length"]) @@ -204,4 +257,36 @@ if __name__ == '__main__': "text": "name=test&csrfmiddlewaretoken=594d26cfa3fad\n", }) + class ResponseParseTest(unittest.TestCase): + def test_parse_standard_http_response(self): + raw = "HTTP/1.1 404 Not Found\nContent-length: 518\nX-powered-by: PHP/5.6.30\nContent-encoding: gzip\nExpires: Thu, 19 Nov 1981 08:52:00 GMT\nVary: Accept-Encoding\nUri: http://127.0.0.1/\nServer: Apache/2.4.10 (Debian)\nConnection: close\nPragma: no-cache\nCache-control: no-store, no-cache, must-revalidate, post-check=0, pre-check=0\nDate: Fri, 23 Jun 2017 16:18:17 GMT\nContent-type: text/html; charset=UTF-8\n\n\nTest\n" # noqa + resp = Response.parse(raw) + self.assertEqual(resp.status, 404) + self.assertEqual(resp.statusText, "Not Found") + + def test_parse_response_as_logged_by_sqlmap(self): + raw = "HTTP response [#74] (200 OK):\nContent-length: 518\nX-powered-by: PHP/5.6.30\nContent-encoding: gzip\nExpires: Thu, 19 Nov 1981 08:52:00 GMT\nVary: Accept-Encoding\nUri: http://127.0.0.1/\nServer: Apache/2.4.10 (Debian)\nConnection: close\nPragma: no-cache\nCache-control: no-store, no-cache, must-revalidate, post-check=0, pre-check=0\nDate: Fri, 23 Jun 2017 16:18:17 GMT\nContent-type: text/html; charset=UTF-8\n\n\nTest\n" # noqa + resp = Response.parse(raw) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.statusText, "OK") + self.assertEqual(resp.headers["Content-Length"], "518") + self.assertIn("Test", resp.content) + self.assertEqual("HTTP response [#74] (200 OK):\n", resp.comment) + + class ResponseRenderTest(unittest.TestCase): + def test_simple_page_encoding(self): + resp = Response(status=200, statusText="OK", + httpVersion="HTTP/1.1", + headers={"Content-Type": "text/html"}, + content="Hello\n") + out = resp.toDict() + self.assertEqual(200, out["status"]) + self.assertEqual("OK", out["statusText"]) + self.assertIn({"name": "Content-Type", "value": "text/html"}, out["headers"]) + self.assertEqual(out["content"], { + "mimeType": "text/html", + "encoding": "base64", + "text": "PGh0bWw+PGJvZHk+SGVsbG88L2JvZHk+PC9odG1sPgo=", + }) + unittest.main(buffer=False) From dd19527e9cbda0eb6901736a25a08a0f57cd19e6 Mon Sep 17 00:00:00 2001 From: Louis-Philippe Huberdeau Date: Thu, 29 Jun 2017 09:00:02 -0400 Subject: [PATCH 4/5] Remove debug _raw entry from output --- lib/utils/collect.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/utils/collect.py b/lib/utils/collect.py index 7cb328883..b88ef4111 100644 --- a/lib/utils/collect.py +++ b/lib/utils/collect.py @@ -112,7 +112,6 @@ class Request: "url": self.url, "headers": [dict(name=key, value=value) for key, value in self.headers.items()], "comment": self.comment, - "_raw": self.raw, } if self.postBody: contentType = self.headers.get('Content-Type') @@ -172,7 +171,6 @@ class Response: "text": base64.b64encode(self.content), }, "comment": self.comment, - "_raw": self.raw, } From b6969df52ac13cace7112ff53156171f7563acfd Mon Sep 17 00:00:00 2001 From: Louis-Philippe Huberdeau Date: Thu, 29 Jun 2017 10:14:20 -0400 Subject: [PATCH 5/5] Add missing httpVersion in request render, avoid encoding to base64 unless binary data is included --- lib/utils/collect.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/lib/utils/collect.py b/lib/utils/collect.py index b88ef4111..fc6915061 100644 --- a/lib/utils/collect.py +++ b/lib/utils/collect.py @@ -108,6 +108,7 @@ class Request: def toDict(self): out = { + "httpVersion": self.httpVersion, "method": self.method, "url": self.url, "headers": [dict(name=key, value=value) for key, value in self.headers.items()], @@ -160,16 +161,22 @@ class Response: raw=raw) def toDict(self): + content = { + "mimeType": self.headers.get('Content-Type'), + "text": self.content, + } + + binary = set(['\0', '\1']) + if any(c in binary for c in self.content): + content["encoding"] = "base64" + content["text"] = base64.b64encode(self.content) + return { "httpVersion": self.httpVersion, "status": self.status, "statusText": self.statusText, "headers": [dict(name=key, value=value) for key, value in self.headers.items()], - "content": { - "mimeType": self.headers.get('Content-Type'), - "encoding": "base64", - "text": base64.b64encode(self.content), - }, + "content": content, "comment": self.comment, } @@ -241,6 +248,7 @@ if __name__ == '__main__': self.assertEqual("http://example.com/test.php", out["url"]) self.assertIn({"name": "Host", "value": "example.com"}, out["headers"]) self.assertEqual("Hello World", out["comment"]) + self.assertEqual("HTTP/1.1", out["httpVersion"]) def test_render_with_post_body(self): req = Request(method="POST", @@ -276,15 +284,26 @@ if __name__ == '__main__': resp = Response(status=200, statusText="OK", httpVersion="HTTP/1.1", headers={"Content-Type": "text/html"}, - content="Hello\n") + content="\nHello\n") out = resp.toDict() self.assertEqual(200, out["status"]) self.assertEqual("OK", out["statusText"]) self.assertIn({"name": "Content-Type", "value": "text/html"}, out["headers"]) self.assertEqual(out["content"], { "mimeType": "text/html", + "text": "\nHello\n", + }) + + def test_simple_body_contains_binary_data(self): + resp = Response(status=200, statusText="OK", + httpVersion="HTTP/1.1", + headers={"Content-Type": "application/octet-stream"}, + content="test\0abc") + out = resp.toDict() + self.assertEqual(out["content"], { "encoding": "base64", - "text": "PGh0bWw+PGJvZHk+SGVsbG88L2JvZHk+PC9odG1sPgo=", + "mimeType": "application/octet-stream", + "text": "dGVzdABhYmM=", }) unittest.main(buffer=False)