Fix for an Issue #129

lib/controller/controller.py

@@ -31,8 +31,8 @@ from lib.core.common import parseTargetUrl
 from lib.core.common import randomStr
 from lib.core.common import readInput
 from lib.core.common import showHttpErrorCodes
-from lib.core.convert import urlencode
-from lib.core.convert import urldecode
+from lib.core.common import urlencode
+from lib.core.common import urldecode
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger

lib/core/common.py

@@ -52,9 +52,9 @@ from lib.core.data import paths
 from lib.core.convert import base64pickle
 from lib.core.convert import base64unpickle
 from lib.core.convert import htmlunescape
+from lib.core.convert import stdoutencode
 from lib.core.convert import unicodeencode
-from lib.core.convert import urldecode
-from lib.core.convert import urlencode
+from lib.core.convert import utf8encode
 from lib.core.decorators import cachedmethod
 from lib.core.enums import CHARSET_TYPE
 from lib.core.enums import DBMS
@@ -125,6 +125,8 @@ from lib.core.settings import SENSITIVE_DATA_REGEX
 from lib.core.settings import TEXT_TAG_REGEX
 from lib.core.settings import UNION_UNIQUE_FIFO_LENGTH
 from lib.core.settings import URI_QUESTION_MARKER
+from lib.core.settings import URLENCODE_CHAR_LIMIT
+from lib.core.settings import URLENCODE_FAILSAFE_CHARS
 from lib.core.threads import getCurrentThreadData
 from thirdparty.clientform.clientform import ParseResponse
 from thirdparty.clientform.clientform import ParseError
@@ -721,29 +723,10 @@ def dataToStdout(data, forceOutput=False, bold=False):
 
     if not kb.get("threadException"):
         if forceOutput or not getCurrentThreadData().disableStdOut:
-            try:
-                if kb.get("multiThreadMode"):
-                    logging._acquireLock()
-                # Reference: http://bugs.python.org/issue1602
-                if IS_WIN:
-                    output = data.encode('ascii', "replace")
-
-                    if output != data:
-                        warnMsg = "cannot properly display Unicode characters "
-                        warnMsg += "inside Windows OS command prompt "
-                        warnMsg += "(http://bugs.python.org/issue1602). All "
-                        warnMsg += "unhandled occurances will result in "
-                        warnMsg += "replacement with '?' character. Please, find "
-                        warnMsg += "proper character representation inside "
-                        warnMsg += "corresponding output files. "
-                        singleTimeWarnMessage(warnMsg)
-
-                    message = output
-                else:
-                    message = data.encode(sys.stdout.encoding)
-            except:
-                message = data.encode(UNICODE_ENCODING)
+            if kb.get("multiThreadMode"):
+                logging._acquireLock()
 
+            message = stdoutencode(data)
             sys.stdout.write(setColor(message, bold))
             sys.stdout.flush()
 
@@ -2010,6 +1993,57 @@ def extractErrorMessage(page):
 
     return retVal
 
+def urldecode(value, encoding=None):
+    result = None
+
+    if value:
+        try:
+            # for cases like T%C3%BCrk%C3%A7e
+            value = str(value)
+        except ValueError:
+            pass
+        finally:
+            result = urllib.unquote_plus(value)
+
+    if isinstance(result, str):
+        result = unicode(result, encoding or UNICODE_ENCODING, "replace")
+
+    return result
+
+def urlencode(value, safe="%&=", convall=False, limit=False):
+    if conf.direct or PLACE.SOAP in conf.paramDict:
+        return value
+
+    count = 0
+    result = None if value is None else ""
+
+    if value:
+        if convall or safe is None:
+            safe = ""
+
+        # corner case when character % really needs to be
+        # encoded (when not representing url encoded char)
+        # except in cases when tampering scripts are used
+        if all(map(lambda x: '%' in x, [safe, value])) and not kb.tamperFunctions:
+            value = re.sub("%(?![0-9a-fA-F]{2})", "%25", value)
+
+        while True:
+            result = urllib.quote(utf8encode(value), safe)
+
+            if limit and len(result) > URLENCODE_CHAR_LIMIT:
+                if count >= len(URLENCODE_FAILSAFE_CHARS):
+                    break
+
+                while count < len(URLENCODE_FAILSAFE_CHARS):
+                    safe += URLENCODE_FAILSAFE_CHARS[count]
+                    count += 1
+                    if safe[-1] in value:
+                        break
+            else:
+                break
+
+    return result
+
 def beep():
     """
     Does an audible beep sound
@@ -2094,11 +2128,7 @@ def logHTTPTraffic(requestLogMsg, responseLogMsg):
         dataToTrafficFile("%s%s" % (responseLogMsg, os.linesep))
         dataToTrafficFile("%s%s%s%s" % (os.linesep, 76 * '#', os.linesep, os.linesep))
 
-def getPageTemplate(payload, place):
-    """
-    Cross-linked method
-    """
-
+def getPageTemplate(payload, place):  # Cross-linked function
     pass
 
 def getPublicTypeMembers(type_, onlyValues=False):

lib/core/convert.py

@@ -17,12 +17,9 @@ import sys
 import struct
 import urllib
 
-from lib.core.data import conf
-from lib.core.data import kb
 from lib.core.enums import PLACE
+from lib.core.settings import IS_WIN
 from lib.core.settings import UNICODE_ENCODING
-from lib.core.settings import URLENCODE_CHAR_LIMIT
-from lib.core.settings import URLENCODE_FAILSAFE_CHARS
 
 def base64decode(value):
     return value.decode("base64")
@@ -62,57 +59,6 @@ def sha1hash(value):
     else:
         return sha.new(value).hexdigest()
 
-def urldecode(value, encoding=None):
-    result = None
-
-    if value:
-        try:
-            # for cases like T%C3%BCrk%C3%A7e
-            value = str(value)
-        except ValueError:
-            pass
-        finally:
-            result = urllib.unquote_plus(value)
-
-    if isinstance(result, str):
-        result = unicode(result, encoding or UNICODE_ENCODING, "replace")
-
-    return result
-
-def urlencode(value, safe="%&=", convall=False, limit=False):
-    if conf.direct or PLACE.SOAP in conf.paramDict:
-        return value
-
-    count = 0
-    result = None if value is None else ""
-
-    if value:
-        if convall or safe is None:
-            safe = ""
-
-        # corner case when character % really needs to be
-        # encoded (when not representing url encoded char)
-        # except in cases when tampering scripts are used
-        if all(map(lambda x: '%' in x, [safe, value])) and not kb.tamperFunctions:
-            value = re.sub("%(?![0-9a-fA-F]{2})", "%25", value)
-
-        while True:
-            result = urllib.quote(utf8encode(value), safe)
-
-            if limit and len(result) > URLENCODE_CHAR_LIMIT:
-                if count >= len(URLENCODE_FAILSAFE_CHARS):
-                    break
-
-                while count < len(URLENCODE_FAILSAFE_CHARS):
-                    safe += URLENCODE_FAILSAFE_CHARS[count]
-                    count += 1
-                    if safe[-1] in value:
-                        break
-            else:
-                break
-
-    return result
-
 def unicodeencode(value, encoding=None):
     """
     Return 8-bit string representation of the supplied unicode value:
@@ -145,3 +91,32 @@ def htmlunescape(value):
         codes = (('&lt;', '<'), ('&gt;', '>'), ('&quot;', '"'), ('&nbsp;', ' '), ('&amp;', '&'))
         retVal = reduce(lambda x, y: x.replace(y[0], y[1]), codes, retVal)
     return retVal
+
+def singleTimeWarnMessage(message):  # Cross-linked function
+    pass
+
+def stdoutencode(data):
+    retVal = None
+
+    try:
+        # Reference: http://bugs.python.org/issue1602
+        if IS_WIN:
+            output = data.encode('ascii', "replace")
+
+            if output != data:
+                warnMsg = "cannot properly display Unicode characters "
+                warnMsg += "inside Windows OS command prompt "
+                warnMsg += "(http://bugs.python.org/issue1602). All "
+                warnMsg += "unhandled occurances will result in "
+                warnMsg += "replacement with '?' character. Please, find "
+                warnMsg += "proper character representation inside "
+                warnMsg += "corresponding output files. "
+                singleTimeWarnMessage(warnMsg)
+
+            retVal = output
+        else:
+            retVal = data.encode(sys.stdout.encoding)
+    except:
+        retVal = data.encode(UNICODE_ENCODING)
+
+    return retVal

lib/core/option.py

@@ -19,6 +19,7 @@ import urlparse
 
 import lib.core.common
 import lib.core.threads
+import lib.core.convert
 
 from lib.controller.checks import checkConnection
 from lib.core.common import Backend
@@ -46,9 +47,10 @@ from lib.core.common import resetCookieJar
 from lib.core.common import runningAsAdmin
 from lib.core.common import sanitizeStr
 from lib.core.common import setOptimize
+from lib.core.common import singleTimeWarnMessage
 from lib.core.common import UnicodeRawConfigParser
-from lib.core.convert import urldecode
-from lib.core.convert import urlencode
+from lib.core.common import urldecode
+from lib.core.common import urlencode
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
@@ -1970,6 +1972,7 @@ def __basicOptionValidation():
 def __resolveCrossReferences():
     lib.core.threads.readInput = readInput
     lib.core.common.getPageTemplate = getPageTemplate
+    lib.core.convert.singleTimeWarnMessage = singleTimeWarnMessage
 
 def init(inputOptions=AttribDict(), overrideOptions=False):
     """

lib/core/target.py

@@ -18,7 +18,7 @@ from lib.core.common import intersect
 from lib.core.common import paramToDict
 from lib.core.common import readInput
 from lib.core.common import resetCookieJar
-from lib.core.convert import urldecode
+from lib.core.common import urldecode
 from lib.core.data import cmdLineOptions
 from lib.core.data import conf
 from lib.core.data import kb

lib/request/connect.py

@@ -32,8 +32,8 @@ from lib.core.common import singleTimeWarnMessage
 from lib.core.common import stdev
 from lib.core.common import urlEncodeCookieValues
 from lib.core.common import wasLastRequestDelayed
-from lib.core.convert import unicodeencode
-from lib.core.convert import urlencode
+from lib.core.common import unicodeencode
+from lib.core.common import urlencode
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger

lib/utils/checkpayload.py

@@ -8,7 +8,7 @@ See the file 'doc/COPYING' for copying permission
 import re
 
 from lib.core.common import readXmlFile
-from lib.core.convert import urldecode
+from lib.core.common import urldecode
 from lib.core.data import paths
 from lib.core.data import logger
 

lib/utils/google.py

@@ -14,8 +14,8 @@ import urllib2
 
 from lib.core.common import getUnicode
 from lib.core.common import readInput
-from lib.core.convert import urldecode
-from lib.core.convert import urlencode
+from lib.core.common import urldecode
+from lib.core.common import urlencode
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger

thirdparty/ansistrm/ansistrm.py

@@ -5,6 +5,8 @@ import logging
 import os
 import re
 
+from lib.core.convert import stdoutencode
+
 class ColorizingStreamHandler(logging.StreamHandler):
     # color names to indices
     color_map = {
@@ -45,7 +47,7 @@ class ColorizingStreamHandler(logging.StreamHandler):
 
     def emit(self, record):
         try:
-            message = self.format(record)
+            message = stdoutencode(self.format(record))
             stream = self.stream
 
             if not self.is_tty: