bug fix, refactoring and improved extractErrorMessage capabilities

lib/core/common.py

@@ -59,6 +59,7 @@ from lib.core.settings import DESCRIPTION
 from lib.core.settings import IS_WIN
 from lib.core.settings import PLATFORM
 from lib.core.settings import SITE
+from lib.core.settings import ERROR_PARSING_REGEXES
 from lib.core.settings import SQL_STATEMENTS
 from lib.core.settings import SUPPORTED_DBMS
 from lib.core.settings import UNKNOWN_DBMS_VERSION
@@ -1584,7 +1585,7 @@ def extractErrorMessage(page):
     retVal = None
 
     if isinstance(page, basestring):
-        for regex in (r"<b>[^<]*(fatal|error|warning|exception)[^<]*</b>:?\s+(?P<result>.+?)<br\s*/?\s*>", r"<li>Error Type:<br>(?P<result>.+?)</li>"):
+        for regex in ERROR_PARSING_REGEXES:
             match = re.search(regex, page, re.DOTALL | re.IGNORECASE)
 
             if match:
@@ -1661,6 +1662,7 @@ def logHTTPTraffic(requestLogMsg, responseLogMsg):
 
     kb.locks.logLock.release()
 
+# cross-linked method
 def getPageTemplate(payload, place):
     pass
 

lib/core/settings.py

@@ -160,3 +160,9 @@ SQL_STATEMENTS      = {
                              "commit ",
                              "rollback ",      ),
                      }
+
+ERROR_PARSING_REGEXES = (   
+                            r"<b>[^<]*(fatal|error|warning|exception)[^<]*</b>:?\s+(?P<result>.+?)<br\s*/?\s*>", 
+                            r"<li>Error Type:<br>(?P<result>.+?)</li>", 
+                            r"error '[0-9a-f]{8}'((<[^>]+>)|\s)+(?P<result>[^<>]+)"
+                        )

lib/request/basic.py

@@ -14,10 +14,12 @@ import re
 import StringIO
 import zlib
 
+from lib.core.common import extractErrorMessage
 from lib.core.common import getCompiledRegex
 from lib.core.common import getUnicode
 from lib.core.common import isWindowsDriveLetterPath
 from lib.core.common import posixToNtSlashes
+from lib.core.common import sanitizeAsciiString
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
@@ -133,3 +135,14 @@ def decodePage(page, contentEncoding, contentType):
             page = getUnicode(page, charset)
 
     return page
+
+def processResponse(page, responseHeaders):
+    page = sanitizeAsciiString(page)
+    page = getUnicode(page)
+    parseResponse(page, responseHeaders)
+    if conf.parseErrors:
+        msg = extractErrorMessage(page)
+
+        if msg:
+            logger.info("parsed error message: '%s'" % msg) 
+    return page

lib/request/connect.py

@@ -20,7 +20,6 @@ from lib.core.agent import agent
 from lib.core.common import average
 from lib.core.common import calculateDeltaSeconds
 from lib.core.common import clearConsoleLine
-from lib.core.common import extractErrorMessage
 from lib.core.common import getCurrentThreadData
 from lib.core.common import getFilteredPageContent
 from lib.core.common import getUnicode
@@ -33,7 +32,6 @@ from lib.core.common import urlEncodeCookieValues
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
-from lib.core.common import sanitizeAsciiString
 from lib.core.enums import HTTPMETHOD
 from lib.core.enums import NULLCONNECTION
 from lib.core.enums import PLACE
@@ -43,7 +41,7 @@ from lib.core.settings import MIN_TIME_RESPONSES
 from lib.core.threads import getCurrentThreadData
 from lib.request.basic import decodePage
 from lib.request.basic import forgeHeaders
-from lib.request.basic import parseResponse
+from lib.request.basic import processResponse
 from lib.request.direct import direct
 from lib.request.comparison import comparison
 from lib.request.methodrequest import MethodRequest
@@ -270,9 +268,7 @@ class Connect:
             else:
                 debugMsg = "got HTTP error code: %d (%s)" % (code, status)
                 logger.debug(debugMsg)
-                page = sanitizeAsciiString(page)
+                page = processResponse(page, responseHeaders)
-                page = getUnicode(page)
-                parseResponse(page, responseHeaders)
                 return page, responseHeaders
 
         except (urllib2.URLError, socket.error, socket.timeout, httplib.BadStatusLine, httplib.IncompleteRead), e:
@@ -316,9 +312,7 @@ class Connect:
 
         socket.setdefaulttimeout(conf.timeout)
 
-        page = sanitizeAsciiString(page)
+        page = processResponse(page, responseHeaders)
-        page = getUnicode(page)
-        parseResponse(page, responseHeaders)
 
         responseMsg += "[#%d] (%d %s):\n" % (threadData.lastRequestUID, code, status)
         if responseHeaders:
@@ -332,12 +326,6 @@ class Connect:
 
         logger.log(7, responseMsg)
 
-        if conf.parseErrors:
-            msg = extractErrorMessage(page)
-
-            if msg:
-                logger.info("parsed error message: '%s'" % msg)
-
         return page, responseHeaders
 
     @staticmethod