Adding initial hook to receive the request/response pairs

lib/core/common.py

@@ -2599,6 +2599,9 @@ def logHTTPTraffic(requestLogMsg, responseLogMsg):
     """
     Logs HTTP traffic to the output file
     """
+    threadData = getCurrentThreadData()
+    assert threadData.requestCollector is not None, "Request collector should be initialized by now"
+    threadData.requestCollector.collectRequest(requestLogMsg, responseLogMsg)
 
     if not conf.trafficFile:
         return

lib/core/option.py

@@ -149,6 +149,7 @@ from lib.request.pkihandler import HTTPSPKIAuthHandler
 from lib.request.rangehandler import HTTPRangeHandler
 from lib.request.redirecthandler import SmartRedirectHandler
 from lib.request.templates import getPageTemplate
+from lib.utils.collect import RequestCollectorFactory
 from lib.utils.crawler import crawl
 from lib.utils.deps import checkDependencies
 from lib.utils.search import search
@@ -1844,6 +1845,7 @@ def _setConfAttributes():
     conf.scheme = None
     conf.tests = []
     conf.trafficFP = None
+    conf.requestCollectorFactory = None
     conf.wFileType = None
 
 def _setKnowledgeBaseAttributes(flushAll=True):
@@ -2228,6 +2230,11 @@ def _setTrafficOutputFP():
 
         conf.trafficFP = openFile(conf.trafficFile, "w+")
 
+def _setupRequestCollector():
+    conf.requestCollectorFactory = RequestCollectorFactory(collect=conf.collectRequests)
+    threadData = getCurrentThreadData()
+    threadData.requestCollector = conf.requestCollectorFactory.create()
+
 def _setDNSServer():
     if not conf.dnsDomain:
         return
@@ -2604,6 +2611,7 @@ def init():
     _setTamperingFunctions()
     _setWafFunctions()
     _setTrafficOutputFP()
+    _setupRequestCollector()
     _resolveCrossReferences()
     _checkWebSocket()
 

lib/core/optiondict.py

@@ -197,6 +197,7 @@ optDict = {
                                "binaryFields":      "string",
                                "charset":           "string",
                                "checkInternet":     "boolean",
+                               "collectRequests":   "string",
                                "crawlDepth":        "integer",
                                "crawlExclude":      "string",
                                "csvDel":            "string",

lib/core/threads.py

@@ -38,6 +38,8 @@ class _ThreadData(threading.local):
         Resets thread data model
         """
 
+        self.requestCollector = None
+
         self.disableStdOut = False
         self.hashDBCursor = None
         self.inTransaction = False

lib/parse/cmdline.py

@@ -631,6 +631,10 @@ def cmdLineParser(argv=None):
                             action="store_true",
                             help="Never ask for user input, use the default behaviour")
 
+        general.add_option("--collect-requests", dest="collectRequests",
+                           action="store_true",
+                           help="Collect requests in HAR format")
+
         general.add_option("--binary-fields", dest="binaryFields",
                           help="Result fields having binary values (e.g. \"digest\")")
 

lib/utils/collect.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+
+"""
+Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+See the file 'doc/COPYING' for copying permission
+"""
+
+from lib.core.data import logger
+
+
+class RequestCollectorFactory:
+
+    def __init__(self, collect=False):
+        self.collect = collect
+
+    def create(self):
+        collector = RequestCollector()
+
+        if not self.collect:
+            collector.collectRequest = self._noop
+
+        return collector
+
+    @staticmethod
+    def _noop(*args, **kwargs):
+        pass
+
+
+class RequestCollector:
+
+    def collectRequest(self, requestMessage, responseMessage):
+        logger.info("Received request/response: %s/%s", len(requestMessage), len(responseMessage))