From 08ccbf2c1ea4ba892efabef142c0e53da21fe5a0 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 3 Jan 2011 22:02:58 +0000 Subject: [PATCH] important fix for a bug reported by x (along with normal fixes, getUnicode now uses kb.pageEncoding) --- doc/THANKS | 3 +++ lib/core/common.py | 4 +--- lib/core/option.py | 1 + lib/request/basic.py | 3 ++- lib/request/connect.py | 27 +++++++++++++++------------ 5 files changed, 22 insertions(+), 16 deletions(-) diff --git a/doc/THANKS b/doc/THANKS index 2aaac8c83..1f1733e00 100644 --- a/doc/THANKS +++ b/doc/THANKS @@ -442,6 +442,9 @@ ToR ultramegaman for reporting a minor bug +x + for reporting a bug + == Organizations == Black Hat team diff --git a/lib/core/common.py b/lib/core/common.py index dfda08e02..f9476c7ad 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -1510,10 +1510,8 @@ def getUnicode(value, encoding=None): u'1' """ - if encoding is None: - encoding = conf.dataEncoding if 'dataEncoding' in conf else "utf-8" if isinstance(value, basestring): - return value if isinstance(value, unicode) else unicode(value, encoding, errors='replace') + return value if isinstance(value, unicode) else unicode(value, encoding or kb.pageEncoding or "utf-8", errors='replace') else: return unicode(value) # encoding ignored for non-basestring instances diff --git a/lib/core/option.py b/lib/core/option.py index 67671f5b5..68a1413cc 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -1183,6 +1183,7 @@ def __setKnowledgeBaseAttributes(flushAll=True): kb.osVersion = None kb.osSP = None + kb.pageEncoding = "utf-8" kb.pageStable = None kb.partRun = None kb.proxyAuthHeader = None diff --git a/lib/request/basic.py b/lib/request/basic.py index 92be0e2bb..40c1b567e 100644 --- a/lib/request/basic.py +++ b/lib/request/basic.py @@ -132,7 +132,8 @@ def decodePage(page, contentEncoding, contentType): charset = checkCharEncoding(contentType.split('charset=')[-1]) if charset: - page = getUnicode(page, charset) + kb.pageEncoding = charset + page = getUnicode(page) return page diff --git a/lib/request/connect.py b/lib/request/connect.py index 12f9e2b9c..3278f5bf9 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -233,9 +233,9 @@ class Connect: # Get HTTP response page = conn.read() code = conn.code - status = conn.msg responseHeaders = conn.info() page = decodePage(page, responseHeaders.get("Content-Encoding"), responseHeaders.get("Content-Type")) + status = getUnicode(conn.msg) # Explicit closing of connection object if not conf.keepAlive: @@ -247,18 +247,11 @@ class Connect: logger.warn(warnMsg) except urllib2.HTTPError, e: - code = e.code - status = e.msg - - threadData.lastHTTPError = (threadData.lastRequestUID, code) - - if code not in kb.httpErrorCodes: - kb.httpErrorCodes[code] = 0 - kb.httpErrorCodes[code] += 1 - + page = None try: page = e.read() responseHeaders = e.info() + page = decodePage(page, responseHeaders.get("Content-Encoding"), responseHeaders.get("Content-Type")) except socket.timeout: warnMsg = "connection timed out while trying " warnMsg += "to get error page information (%d)" % code @@ -267,9 +260,19 @@ class Connect: except: pass + code = e.code + threadData.lastHTTPError = (threadData.lastRequestUID, code) + + if code not in kb.httpErrorCodes: + kb.httpErrorCodes[code] = 0 + kb.httpErrorCodes[code] += 1 + + status = getUnicode(e.msg) responseMsg += "[#%d] (%d %s):\n" % (threadData.lastRequestUID, code, status) + if responseHeaders: - logHeaders = "\n".join(["%s: %s" % (key.capitalize() if isinstance(key, basestring) else key, value) for (key, value) in responseHeaders.items()]) + logHeaders = "\n".join(["%s: %s" % (key.capitalize() if isinstance(key, basestring) else key, getUnicode(value)) for (key, value) in responseHeaders.items()]) + logHTTPTraffic(requestMsg, "%s%s\n\n%s" % (responseMsg, logHeaders, page)) if conf.verbose <= 5: @@ -342,7 +345,7 @@ class Connect: responseMsg += "[#%d] (%d %s):\n" % (threadData.lastRequestUID, code, status) if responseHeaders: - logHeaders = "\n".join(["%s: %s" % (key.capitalize() if isinstance(key, basestring) else key, value) for (key, value) in responseHeaders.items()]) + logHeaders = "\n".join(["%s: %s" % (key.capitalize() if isinstance(key, basestring) else key, getUnicode(value)) for (key, value) in responseHeaders.items()]) logHTTPTraffic(requestMsg, "%s%s\n\n%s" % (responseMsg, logHeaders, page)) if conf.verbose <= 5: