update for a previous update (putting conf.dataEncoding in getUnicode wherever we know that data won't be 'touched' or 'used' in anyway related to the current web page - if not sure, just leave it as it is)

lib/core/common.py

@@ -89,7 +89,7 @@ class UnicodeRawConfigParser(RawConfigParser):
             fp.write("[%s]\n" % DEFAULTSECT)
 
             for (key, value) in self._defaults.items():
-                fp.write("%s = %s\n" % (key, getUnicode(value).replace('\n', '\n\t')))
+                fp.write("%s = %s\n" % (key, getUnicode(value, conf.dataEncoding).replace('\n', '\n\t')))
 
             fp.write("\n")
 
@@ -101,7 +101,7 @@ class UnicodeRawConfigParser(RawConfigParser):
                     if value is None:
                         fp.write("%s\n" % (key))
                     else:
-                        fp.write("%s = %s\n" % (key, getUnicode(value).replace('\n', '\n\t')))
+                        fp.write("%s = %s\n" % (key, getUnicode(value, conf.dataEncoding).replace('\n', '\n\t')))
 
             fp.write("\n")
 
@@ -498,7 +498,7 @@ def readInput(message, default=None):
         message += " "
 
     if conf.batch and default:
-        infoMsg = "%s%s" % (message, getUnicode(default))
+        infoMsg = "%s%s" % (message, getUnicode(default, conf.dataEncoding))
         logger.info(infoMsg)
 
         debugMsg = "used the default behaviour, running in batch mode"
@@ -1511,7 +1511,7 @@ def getUnicode(value, encoding=None):
     """
 
     if isinstance(value, basestring):
-        return value if isinstance(value, unicode) else unicode(value, encoding or kb.pageEncoding or "utf-8", errors='replace')
+        return value if isinstance(value, unicode) else unicode(value, encoding or kb.pageEncoding or conf.dataEncoding, errors='replace')
     else:
         return unicode(value) # encoding ignored for non-basestring instances
 

lib/core/option.py

@@ -1183,7 +1183,7 @@ def __setKnowledgeBaseAttributes(flushAll=True):
     kb.osVersion       = None
     kb.osSP            = None
 
-    kb.pageEncoding    = "utf-8"
+    kb.pageEncoding    = None
     kb.pageStable      = None
     kb.partRun         = None
     kb.proxyAuthHeader = None

lib/core/profiling.py

@@ -27,7 +27,7 @@ def profile(profileOutputFile=None, dotOutputFile=None, imageOutputFile=None):
         import gtk
         import pydot
     except ImportError, e:
-        errMsg = "profiling requires third-party libraries (%s)" % getUnicode(e)
+        errMsg = "profiling requires third-party libraries (%s)" % getUnicode(e, conf.dataEncoding)
         logger.error(errMsg)
         return
 

lib/core/update.py

@@ -201,7 +201,7 @@ def __updateSqlmap():
         logger.debug(debugMsg)
 
         def notify(event_dict):
-            action = getUnicode(event_dict['action'])
+            action = getUnicode(event_dict['action'], conf.dataEncoding)
             index = action.find('_')
             prefix = action[index + 1].upper() if index != -1 else action.capitalize()
 
@@ -211,7 +211,7 @@ def __updateSqlmap():
             if action.find('_completed') == -1:
                 dataToStdout("%s\t%s\n" % (prefix, event_dict['path']))
             else:
-                revision = getUnicode(event_dict['revision'])
+                revision = getUnicode(event_dict['revision'], conf.dataEncoding)
                 index = revision.find('number ')
 
                 if index != -1:

lib/request/direct.py

@@ -45,7 +45,7 @@ def direct(query, content=True):
             output = timeout(func=conf.dbmsConnector.select, args=(query,), duration=conf.timeout, default=None)
 
         infoMsg  = "resumed from file '%s': " % conf.sessionFile
-        infoMsg += "%s..." % getUnicode(output)[:20]
+        infoMsg += "%s..." % getUnicode(output, conf.dataEncoding)[:20]
         logger.info(infoMsg)
     else:
         output = timeout(func=conf.dbmsConnector.select, args=(query,), duration=conf.timeout, default=None)
@@ -61,7 +61,7 @@ def direct(query, content=True):
                 out = list(output)[0][0]
                 if isinstance(out, str):
                     out = utf8decode(out)
-                return getUnicode(out)
+                return getUnicode(out, conf.dataEncoding)
             else:
                 return list(output)
         else:

lib/takeover/metasploit.py

@@ -141,7 +141,7 @@ class Metasploit:
 
         if not choice:
             if lst:
-                choice = getUnicode(default)
+                choice = getUnicode(default, conf.dataEncoding)
             else:
                 return default
 

lib/techniques/error/use.py

@@ -13,7 +13,6 @@ import time
 from lib.core.agent import agent
 from lib.core.common import dataToStdout
 from lib.core.common import extractRegexResult
-from lib.core.common import getUnicode
 from lib.core.common import initTechnique
 from lib.core.common import randomInt
 from lib.core.common import replaceNewlineTabs

lib/utils/google.py

@@ -111,7 +111,7 @@ class Google:
             responseMsg = "HTTP response (%s - %d):\n" % (status, code)
 
             if conf.verbose <= 4:
-                responseMsg += getUnicode(responseHeaders)
+                responseMsg += getUnicode(responseHeaders, conf.dataEncoding)
             elif conf.verbose > 4:
                 responseMsg += "%s\n%s\n" % (responseHeaders, page)
 

plugins/dbms/maxdb/fingerprint.py

@@ -13,7 +13,6 @@ from lib.core.agent import agent
 from lib.core.common import formatDBMSfp
 from lib.core.common import formatFingerprint
 from lib.core.common import getErrorParsedDBMSesFormatted
-from lib.core.common import getUnicode
 from lib.core.common import randomInt
 from lib.core.common import randomRange
 from lib.core.data import conf