added support for displaying HTTP error codes (particularly interesting ones are 403 and 406 which screw up data retrieval and DBMS fingerprinting badly)

lib/controller/controller.py

@@ -23,6 +23,7 @@ from lib.core.common import getUnicode
 from lib.core.common import paramToDict
 from lib.core.common import parseTargetUrl
 from lib.core.common import readInput
+from lib.core.common import showHttpErrorCodes
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
@@ -432,6 +433,9 @@ def start():
                 logger.critical(e)
                 return False
 
+        finally:
+            showHttpErrorCodes()
+
     if conf.loggedToOut:
         logger.info("Fetched data logged to text files under '%s'" % conf.outputPath)
 

lib/core/common.py

@@ -21,6 +21,7 @@ import urlparse
 import ntpath
 import posixpath
 import subprocess
+import httplib
 
 from ConfigParser import DEFAULTSECT
 from ConfigParser import RawConfigParser
@@ -419,14 +420,13 @@ def filePathToString(filePath):
     return strRepl
 
 def dataToStdout(data, forceOutput=False):
-    if (forceOutput or conf.verbose > 0)\
+    if not ('threadException' in kb and kb.threadException):
-        and not ('threadException' in kb and kb.threadException)\
+        if forceOutput or (conf.verbose > 0) and not ('disableStdOut' in kb and kb.disableStdOut):
-        and not ('disableStdOut' in kb and kb.disableStdOut):
+            try:
-        try:
+                sys.stdout.write(data)
-            sys.stdout.write(data)
+                sys.stdout.flush()
-            sys.stdout.flush()
+            except UnicodeEncodeError:
-        except UnicodeEncodeError:
+                print data.encode(conf.dataEncoding)
-            print data.encode(conf.dataEncoding)
 
 def dataToSessionFile(data):
     if not conf.sessionFile:
@@ -1956,3 +1956,11 @@ def unicodeToSafeHTMLValue(value):
 
 def getErrorParsedDBMS():
     return kb.htmlFp[0] if kb.htmlFp else None
+
+def showHttpErrorCodes():
+    if kb.httpErrorCodes:
+        warnMsg = "HTTP error codes detected during testing:\n"
+        warnMsg += ", ".join("%d (%s) - %d times" % (code, httplib.responses[code]\
+          if code in httplib.responses else '?', count)\
+          for code, count in kb.httpErrorCodes.items())
+        logger.warn(warnMsg)

lib/core/option.py

@@ -1156,6 +1156,7 @@ def __setKnowledgeBaseAttributes(flushAll=True):
     kb.docRoot         = None
     kb.dynamicMarkings = []
     kb.endDetection    = False
+    kb.httpErrorCodes  = {}
     kb.errorIsNone     = True
     kb.formNames       = []
     kb.headersCount    = 0

lib/request/connect.py

@@ -252,6 +252,10 @@ class Connect:
 
             threadData.lastHTTPError = (threadData.lastRequestUID, code)
 
+            if code not in kb.httpErrorCodes:
+                kb.httpErrorCodes[code] = 0
+            kb.httpErrorCodes[code] += 1
+
             try:
                 page = e.read()
                 responseHeaders = e.info()

sqlmap.py

@@ -115,10 +115,10 @@ def main():
         closeDumper(True)
 
     finally:
+        dataToStdout("\n[*] shutting down at: %s\n\n" % time.strftime("%X"), forceOutput=True)
+
         kb.threadContinue = False
         kb.threadException = True
 
-    dataToStdout("\n[*] shutting down at: %s\n\n" % time.strftime("%X"), forceOutput=True)
-
 if __name__ == "__main__":
     main()