important fix for a bug reported by x <deep_freeze@mail.ru> (along with normal fixes, getUnicode now uses kb.pageEncoding)

doc/THANKS

@@ -442,6 +442,9 @@ ToR <sstidus@email.it>
 ultramegaman <seclists@ultramegaman.com>
     for reporting a minor bug
 
+x <deep_freeze@mail.ru>
+    for reporting a bug
+
 == Organizations ==
 
 Black Hat team <info@blackhat.com>

lib/core/common.py

@@ -1510,10 +1510,8 @@ def getUnicode(value, encoding=None):
     u'1'
     """
 
-    if encoding is None:
-        encoding = conf.dataEncoding if 'dataEncoding' in conf else "utf-8"
     if isinstance(value, basestring):
-        return value if isinstance(value, unicode) else unicode(value, encoding, errors='replace')
+        return value if isinstance(value, unicode) else unicode(value, encoding or kb.pageEncoding or "utf-8", errors='replace')
     else:
         return unicode(value) # encoding ignored for non-basestring instances
 

lib/core/option.py

@@ -1183,6 +1183,7 @@ def __setKnowledgeBaseAttributes(flushAll=True):
     kb.osVersion       = None
     kb.osSP            = None
 
+    kb.pageEncoding    = "utf-8"
     kb.pageStable      = None
     kb.partRun         = None
     kb.proxyAuthHeader = None

lib/request/basic.py

@@ -132,7 +132,8 @@ def decodePage(page, contentEncoding, contentType):
         charset = checkCharEncoding(contentType.split('charset=')[-1])
 
         if charset:
-            page = getUnicode(page, charset)
+            kb.pageEncoding = charset
+            page = getUnicode(page)
 
     return page
 

lib/request/connect.py

@@ -233,9 +233,9 @@ class Connect:
             # Get HTTP response
             page = conn.read()
             code = conn.code
-            status = conn.msg
             responseHeaders = conn.info()
             page = decodePage(page, responseHeaders.get("Content-Encoding"), responseHeaders.get("Content-Type"))
+            status = getUnicode(conn.msg)
 
             # Explicit closing of connection object
             if not conf.keepAlive:
@@ -247,18 +247,11 @@ class Connect:
                     logger.warn(warnMsg)
 
         except urllib2.HTTPError, e:
-            code = e.code
+            page = None
-            status = e.msg
-
-            threadData.lastHTTPError = (threadData.lastRequestUID, code)
-
-            if code not in kb.httpErrorCodes:
-                kb.httpErrorCodes[code] = 0
-            kb.httpErrorCodes[code] += 1
-
             try:
                 page = e.read()
                 responseHeaders = e.info()
+                page = decodePage(page, responseHeaders.get("Content-Encoding"), responseHeaders.get("Content-Type"))
             except socket.timeout:
                 warnMsg  = "connection timed out while trying "
                 warnMsg += "to get error page information (%d)" % code
@@ -267,9 +260,19 @@ class Connect:
             except:
                 pass
 
+            code = e.code
+            threadData.lastHTTPError = (threadData.lastRequestUID, code)
+
+            if code not in kb.httpErrorCodes:
+                kb.httpErrorCodes[code] = 0
+            kb.httpErrorCodes[code] += 1
+
+            status = getUnicode(e.msg)
             responseMsg += "[#%d] (%d %s):\n" % (threadData.lastRequestUID, code, status)
+
             if responseHeaders:
-                logHeaders = "\n".join(["%s: %s" % (key.capitalize() if isinstance(key, basestring) else key, value) for (key, value) in responseHeaders.items()])
+                logHeaders = "\n".join(["%s: %s" % (key.capitalize() if isinstance(key, basestring) else key, getUnicode(value)) for (key, value) in responseHeaders.items()])
+
             logHTTPTraffic(requestMsg, "%s%s\n\n%s" % (responseMsg, logHeaders, page))
 
             if conf.verbose <= 5:
@@ -342,7 +345,7 @@ class Connect:
 
         responseMsg += "[#%d] (%d %s):\n" % (threadData.lastRequestUID, code, status)
         if responseHeaders:
-            logHeaders = "\n".join(["%s: %s" % (key.capitalize() if isinstance(key, basestring) else key, value) for (key, value) in responseHeaders.items()])
+            logHeaders = "\n".join(["%s: %s" % (key.capitalize() if isinstance(key, basestring) else key, getUnicode(value)) for (key, value) in responseHeaders.items()])
         logHTTPTraffic(requestMsg, "%s%s\n\n%s" % (responseMsg, logHeaders, page))
 
         if conf.verbose <= 5: