more unicode refactoring

lib/controller/checks.py

@@ -27,6 +27,7 @@ import socket
 import time
 
 from lib.core.agent import agent
+from lib.core.common import getUnicode
 from lib.core.common import randomInt
 from lib.core.common import randomStr
 from lib.core.convert import md5hash
@@ -257,7 +258,7 @@ def checkDynParam(place, parameter, value):
     logger.info(infoMsg)
 
     randInt = randomInt()
-    payload = agent.payload(place, parameter, value, unicode(randInt))
+    payload = agent.payload(place, parameter, value, getUnicode(randInt))
     dynResult1 = Request.queryPage(payload, place)
 
     if True == dynResult1:
@@ -395,7 +396,7 @@ def checkConnection():
         conf.seqMatcher.set_seq1(page)
 
     except sqlmapConnectionException, errMsg:
-        errMsg = unicode(errMsg)
+        errMsg = getUnicode(errMsg)
         raise sqlmapConnectionException, errMsg
 
     return True

lib/controller/controller.py

@@ -29,6 +29,7 @@ from lib.controller.checks import checkStability
 from lib.controller.checks import checkString
 from lib.controller.checks import checkRegexp
 from lib.controller.checks import checkConnection
+from lib.core.common import getUnicode
 from lib.core.common import paramToDict
 from lib.core.common import parseTargetUrl
 from lib.core.common import readInput
@@ -155,7 +156,7 @@ def start():
 
             if not conf.dropSetCookie:
                 for _, cookie in enumerate(conf.cj):
-                    cookie = unicode(cookie)
+                    cookie = getUnicode(cookie)
                     index  = cookie.index(" for ")
 
                     cookieStr += "%s;" % cookie[8:index]
@@ -267,7 +268,7 @@ def start():
                     action()
 
         except exceptionsTuple, e:
-            e = unicode(e)
+            e = getUnicode(e)
 
             if conf.multipleTargets:
                 e += ", skipping to next url"

lib/core/common.py

@@ -1354,7 +1354,7 @@ def getCommonStart(strings=[]):
 
 def getUnicode(value):
     if isinstance(value, basestring):
-        return value if isinstance(value, unicode) else unicode(value, conf.dataEncoding)
+        return value if isinstance(value, unicode) else unicode(value, conf.dataEncoding if 'dataEncoding' in conf else "utf-8")
     else:
         return unicode(value)
 

lib/core/progress.py

@@ -22,6 +22,7 @@ with sqlmap; if not, write to the Free Software Foundation, Inc., 51
 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 """
 
+from lib.core.common import getUnicode
 from lib.core.common import dataToStdout
 from lib.core.data import conf
 
@@ -80,7 +81,7 @@ class ProgressBar:
                                           " " * (allFull - numHashes))
 
         # Add the percentage at the beginning of the progress bar
-        percentString = unicode(percentDone) + "%"
+        percentString = getUnicode(percentDone) + "%"
         self.__progBar = "%s %s" % (percentString, self.__progBar)
 
     def draw(self, eta=0):
@@ -102,4 +103,4 @@ class ProgressBar:
         This method returns the progress bar string
         """
 
-        return unicode(self.__progBar)
+        return getUnicode(self.__progBar)

lib/core/update.py

@@ -39,6 +39,7 @@ from xml.dom.minidom import Document
 from subprocess import PIPE
 from subprocess import Popen as execute
 
+from lib.core.common import getUnicode
 from lib.core.common import dataToStdout
 from lib.core.common import pollProcess
 from lib.core.common import readInput
@@ -215,7 +216,7 @@ def __updateSqlmap():
         logger.debug(debugMsg)
 
         def notify(event_dict):
-            action = unicode(event_dict['action'])
+            action = getUnicode(event_dict['action'])
             index = action.find('_')
             prefix = action[index + 1].upper() if index != -1 else action.capitalize()
 
@@ -225,7 +226,7 @@ def __updateSqlmap():
             if action.find('_completed') == -1:
                 print "%s\t%s" % (prefix, event_dict['path'])
             else:
-                revision = unicode(event_dict['revision'])
+                revision = getUnicode(event_dict['revision'])
                 index = revision.find('number ')
 
                 if index != -1:

lib/request/basic.py

@@ -94,7 +94,7 @@ def decodePage(page, encoding):
     Decode gzip/deflate HTTP response
     """
 
-    if unicode(encoding).lower() in ('gzip', 'x-gzip', 'deflate'):
+    if isinstance(encoding, basestring) and encoding.lower() in ('gzip', 'x-gzip', 'deflate'):
         if encoding == 'deflate':
             # http://stackoverflow.com/questions/1089662/python-inflate-and-deflate-implementations
             data = StringIO.StringIO(zlib.decompress(page, -15))

lib/request/direct.py

@@ -24,6 +24,7 @@ Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 
 from lib.core.agent import agent
 from lib.core.common import dataToSessionFile
+from lib.core.common import getUnicode
 from lib.core.convert import base64pickle
 from lib.core.convert import base64unpickle
 from lib.core.convert import utf8decode
@@ -55,7 +56,7 @@ def direct(query, content=True):
         output = base64unpickle(kb.resumedQueries[conf.hostname][query][:-1])
 
         infoMsg  = "resumed from file '%s': " % conf.sessionFile
-        infoMsg += "%s..." % unicode(output)[:20]
+        infoMsg += "%s..." % getUnicode(output)[:20]
         logger.info(infoMsg)
     elif select:
         output = timeout(func=conf.dbmsConnector.select, args=(query,), duration=conf.timeout, default=None)

lib/takeover/metasploit.py

@@ -36,6 +36,7 @@ from subprocess import Popen as execute
 from lib.core.common import dataToStdout
 from lib.core.common import getLocalIP
 from lib.core.common import getRemoteIP
+from lib.core.common import getUnicode
 from lib.core.common import normalizePath
 from lib.core.common import ntToPosixSlashes
 from lib.core.common import pollProcess
@@ -157,7 +158,7 @@ class Metasploit:
 
         if not choice:
             if lst:
-                choice = unicode(default)
+                choice = getUnicode(default)
             else:
                 return default
 

lib/techniques/inband/union/use.py

@@ -27,6 +27,7 @@ import time
 
 from lib.core.agent import agent
 from lib.core.common import calculateDeltaSeconds
+from lib.core.common import getUnicode
 from lib.core.common import parseUnionPage
 from lib.core.data import conf
 from lib.core.data import kb
@@ -227,7 +228,7 @@ def unionUse(expression, direct=False, unescape=True, resetCounter=False, nullCh
         # sql injection output
         startPosition = resultPage.index(temp.start)
         endPosition = resultPage.rindex(temp.stop) + len(temp.stop)
-        value = unicode(resultPage[startPosition:endPosition])
+        value = getUnicode(resultPage[startPosition:endPosition])
 
         duration = calculateDeltaSeconds(start)
 

lib/utils/google.py

@@ -27,6 +27,7 @@ import re
 import socket
 import urllib2
 
+from lib.core.common import getUnicode
 from lib.core.convert import urlencode
 from lib.core.data import conf
 from lib.core.data import kb
@@ -123,7 +124,7 @@ class Google:
             responseMsg = "HTTP response (%s - %d):\n" % (status, code)
     
             if conf.verbose <= 4:
-                responseMsg += unicode(responseHeaders)
+                responseMsg += getUnicode(responseHeaders)
             elif conf.verbose > 4:
                 responseMsg += "%s\n%s\n" % (responseHeaders, page)
     

plugins/dbms/firebird/fingerprint.py

@@ -28,6 +28,7 @@ from lib.core.agent import agent
 from lib.core.common import formatDBMSfp
 from lib.core.common import formatFingerprint
 from lib.core.common import getHtmlErrorFp
+from lib.core.common import getUnicode
 from lib.core.common import randomInt
 from lib.core.common import randomRange
 from lib.core.data import conf
@@ -94,7 +95,7 @@ class Fingerprint(GenericFingerprint):
         for i in xrange(len(table)):
             version, checks = table[i]
             failed = False
-            check = checks[randomRange(0,len(checks)-1)].replace("%d", unicode(randomRange(1,100)))
+            check = checks[randomRange(0,len(checks)-1)].replace("%d", getUnicode(randomRange(1,100)))
             payload = agent.fullPayload(check)
             result  = Request.queryPage(payload)
             if result:

plugins/dbms/mssqlserver/fingerprint.py

@@ -26,6 +26,7 @@ from lib.core.agent import agent
 from lib.core.common import formatDBMSfp
 from lib.core.common import formatFingerprint
 from lib.core.common import getHtmlErrorFp
+from lib.core.common import getUnicode
 from lib.core.common import randomInt
 from lib.core.data import conf
 from lib.core.data import kb
@@ -217,7 +218,7 @@ class Fingerprint(GenericFingerprint):
 
         for sp in sps:
             query  =  "(SELECT LEN(%s) FROM %s WHERE %s " % (self.tblField, self.fileTblName, self.tblField)
-            query += "LIKE '%Service Pack " + unicode(sp) + "%')>0"
+            query += "LIKE '%Service Pack " + getUnicode(sp) + "%')>0"
             query  = agent.forgeCaseStatement(query)
 
             if inject.getValue(query, charsetType=1) == "1":

plugins/dbms/mysql/fingerprint.py

@@ -28,6 +28,7 @@ from lib.core.agent import agent
 from lib.core.common import formatDBMSfp
 from lib.core.common import formatFingerprint
 from lib.core.common import getHtmlErrorFp
+from lib.core.common import getUnicode
 from lib.core.common import randomInt
 from lib.core.data import conf
 from lib.core.data import kb
@@ -76,7 +77,7 @@ class Fingerprint(GenericFingerprint):
 
             for version in range(element[0], element[1] + 1):
                 randInt = randomInt()
-                version = unicode(version)
+                version = getUnicode(version)
                 query   = agent.prefixQuery(" /*!%s AND %d=%d*/" % (version, randInt, randInt + 1))
                 query   = agent.postfixQuery(query)
                 payload = agent.payload(newValue=query)
@@ -168,7 +169,7 @@ class Fingerprint(GenericFingerprint):
         infoMsg = "testing MySQL"
         logger.info(infoMsg)
 
-        randInt = unicode(randomInt(1))
+        randInt = getUnicode(randomInt(1))
         payload = agent.fullPayload(" AND CONNECTION_ID()=CONNECTION_ID()")
         result  = Request.queryPage(payload)
 

plugins/dbms/postgresql/fingerprint.py

@@ -28,6 +28,7 @@ from lib.core.agent import agent
 from lib.core.common import formatDBMSfp
 from lib.core.common import formatFingerprint
 from lib.core.common import getHtmlErrorFp
+from lib.core.common import getUnicode
 from lib.core.common import randomInt
 from lib.core.data import conf
 from lib.core.data import kb
@@ -97,7 +98,7 @@ class Fingerprint(GenericFingerprint):
         infoMsg = "testing PostgreSQL"
         logger.info(infoMsg)
 
-        randInt = unicode(randomInt(1))
+        randInt = getUnicode(randomInt(1))
 
         payload = agent.fullPayload(" AND %s::int=%s" % (randInt, randInt))
         result  = Request.queryPage(payload)

plugins/generic/enumeration.py

@@ -26,6 +26,7 @@ import re
 
 from lib.core.agent import agent
 from lib.core.common import getRange
+from lib.core.common import getUnicode
 from lib.core.common import parsePasswordHash
 from lib.core.common import readInput
 from lib.core.common import safeStringFormat
@@ -440,7 +441,7 @@ class Enumeration:
 
                             # In PostgreSQL we get 1 if the privilege is
                             # True, 0 otherwise
-                            if kb.dbms == "PostgreSQL" and unicode(privilege).isdigit():
+                            if kb.dbms == "PostgreSQL" and getUnicode(privilege).isdigit():
                                 for position, pgsqlPriv in pgsqlPrivs:
                                     if count == position and int(privilege) == 1:
                                         privileges.add(pgsqlPriv)
@@ -1058,7 +1059,7 @@ class Enumeration:
                         else:
                             colEntry = entry[index]
 
-                        colEntryLen = len(unicode(colEntry))
+                        colEntryLen = len(getUnicode(colEntry))
                         maxLen = max(colLen, colEntryLen)
 
                         if maxLen > kb.data.dumpedTable[column]["length"]:

sqlmap.py

@@ -44,6 +44,7 @@ except ImportError, _:
 
 from lib.controller.controller import start
 from lib.core.common import banner
+from lib.core.common import getUnicode
 from lib.core.common import profile
 from lib.core.common import setPaths
 from lib.core.common import weAreFrozen
@@ -63,7 +64,7 @@ def modulePath():
     """
 
     if weAreFrozen():
-        return os.path.dirname(unicode(sys.executable, sys.getfilesystemencoding()))
+        return os.path.dirname(getUnicode(sys.executable, sys.getfilesystemencoding()))
     else:
         return os.path.dirname(os.path.realpath(__file__))
 
@@ -87,7 +88,7 @@ def main():
         else:
             start()
     except exceptionsTuple, e:
-        e = unicode(e)
+        e = getUnicode(e)
         logger.error(e)
         closeDumper(False, e)