refactoring

lib/core/common.py

@@ -59,6 +59,7 @@ from lib.core.exception import sqlmapMissingDependence
 from lib.core.exception import sqlmapSyntaxException
 from lib.core.optiondict import optDict
 from lib.core.settings import INFERENCE_UNKNOWN_CHAR
+from lib.core.settings import UNICODE_ENCODING
 from lib.core.settings import DESCRIPTION
 from lib.core.settings import IS_WIN
 from lib.core.settings import PLATFORM
@@ -99,7 +100,7 @@ class UnicodeRawConfigParser(RawConfigParser):
             fp.write("[%s]\n" % DEFAULTSECT)
 
             for (key, value) in self._defaults.items():
-                fp.write("%s = %s\n" % (key, getUnicode(value, conf.dataEncoding).replace('\n', '\n\t')))
+                fp.write("%s = %s\n" % (key, getUnicode(value, UNICODE_ENCODING).replace('\n', '\n\t')))
 
             fp.write("\n")
 
@@ -111,7 +112,7 @@ class UnicodeRawConfigParser(RawConfigParser):
                     if value is None:
                         fp.write("%s\n" % (key))
                     else:
-                        fp.write("%s = %s\n" % (key, getUnicode(value, conf.dataEncoding).replace('\n', '\n\t')))
+                        fp.write("%s = %s\n" % (key, getUnicode(value, UNICODE_ENCODING).replace('\n', '\n\t')))
 
             fp.write("\n")
 
@@ -584,9 +585,9 @@ def dataToStdout(data, forceOutput=False):
     if not ('threadException' in kb and kb.threadException):
         if forceOutput or (conf.verbose > 0) and not getCurrentThreadData().disableStdOut:
             try:
-                sys.stdout.write(data)
+                sys.stdout.write(data.encode(sys.stdout.encoding))
-            except UnicodeEncodeError:
+            except:
-                sys.stdout.write(data.encode(conf.dataEncoding))
+                sys.stdout.write(data.encode(UNICODE_ENCODING, errors="replace"))
             finally:
                 sys.stdout.flush()
 
@@ -660,7 +661,7 @@ def readInput(message, default=None):
         message += " "
 
     if conf.batch and default:
-        infoMsg = "%s%s" % (message, getUnicode(default, conf.dataEncoding))
+        infoMsg = "%s%s" % (message, getUnicode(default, UNICODE_ENCODING))
         logger.info(infoMsg)
 
         debugMsg = "used the default behaviour, running in batch mode"
@@ -668,7 +669,7 @@ def readInput(message, default=None):
 
         data = default
     else:
-        data = raw_input(message.encode(sys.stdout.encoding or conf.dataEncoding))
+        data = raw_input(message.encode(sys.stdout.encoding or UNICODE_ENCODING))
 
         if not data:
             data = default
@@ -1438,7 +1439,7 @@ def readCachedFileContent(filename, mode='rb'):
 
         if filename not in kb.cache.content:
             checkFile(filename)
-            xfile = codecs.open(filename, mode, conf.dataEncoding)
+            xfile = codecs.open(filename, mode, UNICODE_ENCODING)
             content = xfile.read()
             kb.cache.content[filename] = content
             xfile.close()
@@ -1450,7 +1451,7 @@ def readCachedFileContent(filename, mode='rb'):
 def readXmlFile(xmlFile):
     checkFile(xmlFile)  
 
-    xfile = codecs.open(xmlFile, 'r', conf.dataEncoding)
+    xfile = codecs.open(xmlFile, 'r', UNICODE_ENCODING)
     retVal = minidom.parse(xfile).documentElement
 
     xfile.close()
@@ -1502,7 +1503,7 @@ def initCommonOutputs():
     kb.commonOutputs = {}
     key = None
 
-    cfile = codecs.open(paths.COMMON_OUTPUTS, 'r', conf.dataEncoding)
+    cfile = codecs.open(paths.COMMON_OUTPUTS, 'r', UNICODE_ENCODING)
 
     for line in cfile.readlines(): # xreadlines doesn't return unicode strings when codec.open() is used
         if line.find('#') != -1:
@@ -1528,7 +1529,7 @@ def getFileItems(filename, commentPrefix='#', unicode_=True, lowercase=False, un
     checkFile(filename)
 
     if unicode_:
-        ifile = codecs.open(filename, 'r', conf.dataEncoding)
+        ifile = codecs.open(filename, 'r', UNICODE_ENCODING)
     else:
         ifile = open(filename, 'r')
 
@@ -1683,7 +1684,7 @@ def getUnicode(value, encoding=None):
     if isinstance(value, unicode):
         return value
     elif isinstance(value, basestring):
-        return unicode(value, encoding or conf.dataEncoding, errors="replace")
+        return unicode(value, encoding or UNICODE_ENCODING, errors="replace")
     else:
         return unicode(value) # encoding ignored for non-basestring instances
 
@@ -2260,7 +2261,7 @@ def openFile(filename, mode='r'):
     """
 
     try:
-        return codecs.open(filename, mode, conf.dataEncoding)
+        return codecs.open(filename, mode, UNICODE_ENCODING)
     except IOError:
         errMsg = "there has been a file opening error for filename '%s'. " % filename
         errMsg += "Please check %s permissions on a file " % ("write" if \

lib/core/dump.py

@@ -20,6 +20,7 @@ from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.replication import Replication
+from lib.core.settings import UNICODE_ENCODING
 
 class Dump:
     """
@@ -46,7 +47,7 @@ class Dump:
 
     def setOutputFile(self):
         self.__outputFile = "%s%slog" % (conf.outputPath, os.sep)
-        self.__outputFP = codecs.open(self.__outputFile, "ab", conf.dataEncoding)
+        self.__outputFP = codecs.open(self.__outputFile, "ab", UNICODE_ENCODING)
 
     def getOutputFile(self):
         return self.__outputFile

lib/core/option.py

@@ -1137,7 +1137,6 @@ def __setConfAttributes():
 
     conf.boundaries       = []
     conf.cj               = None
-    conf.dataEncoding     = "utf-8"
     conf.dbmsConnector    = None
     conf.dbmsHandler      = None
     conf.dumpPath         = None

lib/core/profiling.py

@@ -12,9 +12,9 @@ import os
 import cProfile
 
 from lib.core.common import getUnicode
-from lib.core.data import conf
 from lib.core.data import logger
 from lib.core.data import paths
+from lib.core.settings import UNICODE_ENCODING
 
 def profile(profileOutputFile=None, dotOutputFile=None, imageOutputFile=None):
     """
@@ -27,7 +27,7 @@ def profile(profileOutputFile=None, dotOutputFile=None, imageOutputFile=None):
         import gtk
         import pydot
     except ImportError, e:
-        errMsg = "profiling requires third-party libraries (%s)" % getUnicode(e, conf.dataEncoding)
+        errMsg = "profiling requires third-party libraries (%s)" % getUnicode(e, UNICODE_ENCODING)
         logger.error(errMsg)
         return
 
@@ -60,7 +60,7 @@ def profile(profileOutputFile=None, dotOutputFile=None, imageOutputFile=None):
 
     # Create dot file by using extra/gprof2dot/gprof2dot.py
     # http://code.google.com/p/jrfonseca/wiki/Gprof2Dot
-    dotFilePointer = codecs.open(dotOutputFile, 'wt', conf.dataEncoding)
+    dotFilePointer = codecs.open(dotOutputFile, 'wt', UNICODE_ENCODING)
     parser = gprof2dot.PstatsParser(profileOutputFile)
     profile = parser.parse()
     profile.prune(0.5/100.0, 0.1/100.0)

lib/core/settings.py

@@ -208,3 +208,6 @@ BURP_SPLITTER = "======================================================"
 
 # Do the url-encoding based on parameter place
 URL_ENCODE_PAYLOAD = { PLACE.GET: True, PLACE.POST: True, PLACE.COOKIE: False, PLACE.UA: True, PLACE.URI: False }
+
+# Encoding used for Unicode data
+UNICODE_ENCODING = "utf8"

lib/core/target.py

@@ -31,6 +31,7 @@ from lib.core.exception import sqlmapSyntaxException
 from lib.core.option import __setDBMS
 from lib.core.option import __setKnowledgeBaseAttributes
 from lib.core.session import resumeConfKb
+from lib.core.settings import UNICODE_ENCODING
 from lib.core.xmldump import dumper as xmldumper
 from lib.request.connect import Connect as Request
 
@@ -139,7 +140,7 @@ def __setOutputResume():
 
     if os.path.exists(conf.sessionFile):
         if not conf.flushSession:
-            readSessionFP = codecs.open(conf.sessionFile, "r", conf.dataEncoding, 'replace')
+            readSessionFP = codecs.open(conf.sessionFile, "r", UNICODE_ENCODING, 'replace')
             __url_cache = set()
             __expression_cache = {}
 
@@ -190,7 +191,7 @@ def __setOutputResume():
                 raise sqlmapFilePathException, errMsg
 
     try:
-        conf.sessionFP = codecs.open(conf.sessionFile, "a", conf.dataEncoding)
+        conf.sessionFP = codecs.open(conf.sessionFile, "a", UNICODE_ENCODING)
         dataToSessionFile("\n[%s]\n" % time.strftime("%X %x"))
     except IOError:
         errMsg = "unable to write on the session file specified"

lib/core/update.py

@@ -31,6 +31,7 @@ from lib.core.data import paths
 from lib.core.exception import sqlmapConnectionException
 from lib.core.exception import sqlmapFilePathException
 from lib.core.settings import MSSQL_VERSIONS_URL
+from lib.core.settings import UNICODE_ENCODING
 from lib.core.subprocessng import pollProcess
 from lib.request.connect import Connect as Request
 
@@ -130,7 +131,7 @@ def __updateMSSQLXML():
                 servicepackElement.appendChild(servicepackText)
 
     # Get the XML old file content to a local variable
-    mssqlXml = codecs.open(paths.MSSQL_XML, "r", conf.dataEncoding)
+    mssqlXml = codecs.open(paths.MSSQL_XML, "r", UNICODE_ENCODING)
     oldMssqlXml = mssqlXml.read()
     oldMssqlXmlSignatures = oldMssqlXml.count("<signature>")
     oldMssqlXmlList = oldMssqlXml.splitlines(1)
@@ -140,12 +141,12 @@ def __updateMSSQLXML():
     shutil.copy(paths.MSSQL_XML, "%s.bak" % paths.MSSQL_XML)
 
     # Save our newly created XML to the signatures file
-    mssqlXml = codecs.open(paths.MSSQL_XML, "w", conf.dataEncoding)
+    mssqlXml = codecs.open(paths.MSSQL_XML, "w", UNICODE_ENCODING)
     doc.writexml(writer=mssqlXml, addindent="    ", newl="\n")
     mssqlXml.close()
 
     # Get the XML new file content to a local variable
-    mssqlXml = codecs.open(paths.MSSQL_XML, "r", conf.dataEncoding)
+    mssqlXml = codecs.open(paths.MSSQL_XML, "r", UNICODE_ENCODING)
     newMssqlXml = mssqlXml.read()
     newMssqlXmlSignatures = newMssqlXml.count("<signature>")
     newMssqlXmlList = newMssqlXml.splitlines(1)
@@ -199,7 +200,7 @@ def __updateSqlmap():
         logger.debug(debugMsg)
 
         def notify(event_dict):
-            action = getUnicode(event_dict['action'], conf.dataEncoding)
+            action = getUnicode(event_dict['action'])
             index = action.find('_')
             prefix = action[index + 1].upper() if index != -1 else action.capitalize()
 
@@ -209,7 +210,7 @@ def __updateSqlmap():
             if action.find('_completed') == -1:
                 dataToStdout("%s\t%s\n" % (prefix, event_dict['path']))
             else:
-                revision = getUnicode(event_dict['revision'], conf.dataEncoding)
+                revision = getUnicode(event_dict['revision'], UNICODE_ENCODING)
                 index = revision.find('number ')
 
                 if index != -1:

lib/core/xmldump.py

@@ -16,6 +16,7 @@ from lib.core.common import restoreDumpMarkedChars
 from lib.core.data import conf
 from lib.core.data import logger
 from lib.core.exception import sqlmapFilePathException
+from lib.core.settings import UNICODE_ENCODING
 
 TECHNIC_ELEM_NAME = "Technic"
 TECHNICS_ELEM_NAME = "Technics"
@@ -489,7 +490,7 @@ class XMLDump:
                     except ExpatError:
                         self.__doc = Document()
 
-                self.__outputFP = codecs.open(self.__outputFile, "w+", conf.dataEncoding)
+                self.__outputFP = codecs.open(self.__outputFile, "w+", UNICODE_ENCODING)
 
                 if self.__root is None:
                     self.__root = self.__doc.createElementNS(NAME_SPACE_ATTR, RESULTS_ELEM_NAME)
@@ -525,7 +526,7 @@ class XMLDump:
                 statusElem.appendChild(errorElem)
 
             self.__addToRoot(statusElem)
-            self.__write(prettyprint.formatXML(self.__doc, encoding=conf.dataEncoding))
+            self.__write(prettyprint.formatXML(self.__doc, encoding=UNICODE_ENCODING))
             self.__outputFP.close()
 
 def closeDumper(status, msg=""):

lib/parse/cmdline.py

@@ -18,6 +18,7 @@ from lib.core.common import getUnicode
 from lib.core.data import logger
 from lib.core.settings import TIME_DEFAULT_DELAY
 from lib.core.settings import VERSION_STRING
+from lib.core.settings import UNICODE_ENCODING
 
 def cmdLineParser():
     """
@@ -545,11 +546,12 @@ def cmdLineParser():
         parser.add_option_group(miscellaneous)
 
         args = []
+        from lib.core.common import dataToStdout
         for arg in sys.argv:
             try:
-                args.append(getUnicode(arg, sys.stdin.encoding or sys.getfilesystemencoding()))
+                args.append(getUnicode(arg, sys.getfilesystemencoding() or sys.stdin.encoding))
             except:
-                args.append(getUnicode(arg, "utf8"))
+                args.append(getUnicode(arg, UNICODE_ENCODING))
         (args, _) = parser.parse_args(args)
 
         if not args.direct and not args.url and not args.list and not args.googleDork and not args.configFile\

lib/parse/configfile.py

@@ -13,10 +13,10 @@ from ConfigParser import NoSectionError
 
 from lib.core.common import checkFile
 from lib.core.common import UnicodeRawConfigParser
-from lib.core.data import conf
 from lib.core.data import logger
 from lib.core.exception import sqlmapMissingMandatoryOptionException
 from lib.core.optiondict import optDict
+from lib.core.settings import UNICODE_ENCODING
 
 config = None
 
@@ -58,7 +58,7 @@ def configFileParser(configFile):
     logger.debug(debugMsg)
 
     checkFile(configFile)
-    configFP = codecs.open(configFile, "rb", conf.dataEncoding)
+    configFP = codecs.open(configFile, "rb", UNICODE_ENCODING)
     config = UnicodeRawConfigParser()
     config.readfp(configFP)
 

lib/request/basic.py

@@ -26,6 +26,7 @@ from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.settings import META_CHARSET_REGEX
 from lib.core.settings import DEFAULT_PAGE_ENCODING
+from lib.core.settings import UNICODE_ENCODING
 from lib.parse.headers import headersParser
 from lib.parse.html import htmlParser
 
@@ -113,7 +114,7 @@ def checkCharEncoding(encoding):
         warnMsg  = "unknown charset '%s'. " % encoding
         warnMsg += "Please report by e-mail to sqlmap-users@lists.sourceforge.net."
         logger.warn(warnMsg)
-        encoding = conf.dataEncoding
+        encoding = UNICODE_ENCODING
 
     return encoding
 

lib/request/direct.py

@@ -19,6 +19,7 @@ from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.enums import DBMS
 from lib.core.settings import SQL_STATEMENTS
+from lib.core.settings import UNICODE_ENCODING
 from lib.utils.timeout import timeout
 
 def direct(query, content=True):
@@ -49,7 +50,7 @@ def direct(query, content=True):
             output = timeout(func=conf.dbmsConnector.select, args=(query,), duration=conf.timeout, default=None)
 
         infoMsg  = "resumed from file '%s': " % conf.sessionFile
-        infoMsg += "%s..." % getUnicode(output, conf.dataEncoding)[:20]
+        infoMsg += "%s..." % getUnicode(output, UNICODE_ENCODING)[:20]
         logger.info(infoMsg)
     else:
         output = timeout(func=conf.dbmsConnector.select, args=(query,), duration=conf.timeout, default=None)
@@ -65,7 +66,7 @@ def direct(query, content=True):
                 out = list(output)[0][0]
                 if isinstance(out, str):
                     out = utf8decode(out)
-                return getUnicode(out, conf.dataEncoding)
+                return getUnicode(out, UNICODE_ENCODING)
             else:
                 return list(output)
         else:

lib/takeover/metasploit.py

@@ -34,6 +34,7 @@ from lib.core.data import logger
 from lib.core.enums import DBMS
 from lib.core.exception import sqlmapDataException
 from lib.core.exception import sqlmapFilePathException
+from lib.core.settings import UNICODE_ENCODING
 from lib.core.subprocessng import blockingReadFromFD
 from lib.core.subprocessng import blockingWriteToFD
 from lib.core.subprocessng import pollProcess
@@ -142,7 +143,7 @@ class Metasploit:
 
         if not choice:
             if lst:
-                choice = getUnicode(default, conf.dataEncoding)
+                choice = getUnicode(default, UNICODE_ENCODING)
             else:
                 return default
 
@@ -341,7 +342,7 @@ class Metasploit:
 
         self.__resource += "exploit\n"
 
-        self.resourceFp = codecs.open(self.resourceFile, "w", conf.dataEncoding)
+        self.resourceFp = codecs.open(self.resourceFile, "w", UNICODE_ENCODING)
         self.resourceFp.write(self.__resource)
         self.resourceFp.close()
 

lib/techniques/inband/union/test.py

@@ -96,6 +96,38 @@ def __unionTestByCharBruteforce(comment, place, parameter, value, prefix, suffix
     SQL injection vulnerability. The test is done up to 50 columns
     on the target database table
     """
+    ratios = []
+    from lib.core.common import popValue
+    from lib.core.common import pushValue
+    from lib.request.comparison import comparison
+    from lib.core.common import stdev
+    from lib.core.common import average
+    pushValue(kb.errorIsNone)
+    kb.errorIsNone = False
+    #for count in range(conf.uColsStart, conf.uColsStop+1):
+    for count in range(conf.uColsStart, conf.uColsStop+1):
+        query = agent.forgeInbandQuery('', -1, count, comment, prefix, suffix, conf.uChar)
+        payload = agent.payload(place=place, parameter=parameter, newValue=query, where=1)
+        page, _ = Request.queryPage(payload, place=place, content=True, raise404=False)
+        ratios.append(comparison(page, True))
+    min_, max_ = min(ratios), max(ratios)
+
+    minIndex = ratios.index(min_)
+    maxIndex = ratios.index(max_)
+    ratios.pop(ratios.index(min_))
+    ratios.pop(ratios.index(max_))
+    deviation = stdev(ratios)
+    lower, upper = average(ratios) - 7 * deviation, average(ratios) + 7 * deviation
+    if min_ < lower:
+        print "NULL count is vulnerable: %d" % (minIndex + conf.uColsStart + 1)
+        pass
+    elif max_ > upper:
+        print "NULL count is vulnerable: %d" % (maxIndex + conf.uColsStart + 1)
+
+    kb.errorIsNone = popValue()
+    return None, None
+
+    #---------------------------------------------------------
 
     validPayload = None
     vector = None

lib/utils/google.py

@@ -21,6 +21,7 @@ from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.exception import sqlmapConnectionException
 from lib.core.exception import sqlmapGenericException
+from lib.core.settings import UNICODE_ENCODING
 from lib.request.basic import decodePage
 
 class Google:
@@ -111,7 +112,7 @@ class Google:
             responseMsg = "HTTP response (%s - %d):\n" % (status, code)
 
             if conf.verbose <= 4:
-                responseMsg += getUnicode(responseHeaders, conf.dataEncoding)
+                responseMsg += getUnicode(responseHeaders, UNICODE_ENCODING)
             elif conf.verbose > 4:
                 responseMsg += "%s\n%s\n" % (responseHeaders, page)
 

lib/utils/hash.py

@@ -17,7 +17,6 @@ from zipfile import ZipFile
 from extra.pydes.pyDes import des
 from extra.pydes.pyDes import CBC
 from lib.core.common import checkFile
-from lib.core.common import conf
 from lib.core.common import clearConsoleLine
 from lib.core.common import dataToStdout
 from lib.core.common import getCompiledRegex
@@ -36,6 +35,7 @@ from lib.core.enums import HASH
 from lib.core.exception import sqlmapUserQuitException
 from lib.core.settings import COMMON_PASSWORD_SUFFIXES
 from lib.core.settings import DUMMY_USER_PREFIX
+from lib.core.settings import UNICODE_ENCODING
 
 def mysql_passwd(password, uppercase=True):
     """
@@ -151,7 +151,7 @@ def oracle_old_passwd(password, username, uppercase=True): # prior to version '1
     IV, pad = "\0"*8, "\0"
 
     if isinstance(username, unicode):
-        username = unicode.encode(username, conf.dataEncoding) #pyDes has issues with unicode strings
+        username = unicode.encode(username, UNICODE_ENCODING) #pyDes has issues with unicode strings
 
     unistr = "".join("\0%s" % c for c in (username + password).upper())
 

plugins/dbms/firebird/connector.py

@@ -12,9 +12,9 @@ try:
 except ImportError, _:
     pass
 
-from lib.core.data import conf
 from lib.core.data import logger
 from lib.core.exception import sqlmapConnectionException
+from lib.core.settings import UNICODE_ENCODING
 
 from plugins.generic.connector import Connector as GenericConnector
 
@@ -39,8 +39,8 @@ class Connector(GenericConnector):
             self.checkFileDb()
 
         try:
-            self.connector = kinterbasdb.connect(host=self.hostname.encode(conf.dataEncoding), database=self.db.encode(conf.dataEncoding), \
+            self.connector = kinterbasdb.connect(host=self.hostname.encode(UNICODE_ENCODING), database=self.db.encode(UNICODE_ENCODING), \
-                user=self.user.encode(conf.dataEncoding), password=self.password.encode(conf.dataEncoding), charset="UTF8") #http://www.daniweb.com/forums/thread248499.html
+                user=self.user.encode(UNICODE_ENCODING), password=self.password.encode(UNICODE_ENCODING), charset="UTF8") #http://www.daniweb.com/forums/thread248499.html
         except kinterbasdb.OperationalError, msg:
             raise sqlmapConnectionException, msg[1]
         self.setCursor()