Minor DREI updates

lib/controller/checks.py

@@ -49,7 +49,6 @@ from lib.core.common import urlencode
 from lib.core.common import wasLastResponseDBMSError
 from lib.core.common import wasLastResponseHTTPError
 from lib.core.compat import xrange
-from lib.core.convert import unicodeencode
 from lib.core.defaults import defaults
 from lib.core.data import conf
 from lib.core.data import kb
@@ -1611,7 +1610,7 @@ def checkConnection(suppressOutput=False):
             kb.errorIsNone = True
 
         if kb.redirectChoice == REDIRECTION.YES and threadData.lastRedirectURL and threadData.lastRedirectURL[0] == threadData.lastRequestUID:
-            if (threadData.lastRedirectURL[1] or "").startswith("https://") and unicodeencode(conf.hostname) in threadData.lastRedirectURL[1]:
+            if (threadData.lastRedirectURL[1] or "").startswith("https://") and conf.hostname in getUnicode(threadData.lastRedirectURL[1]):
                 conf.url = re.sub(r"https?://", "https://", conf.url)
                 match = re.search(r":(\d+)", threadData.lastRedirectURL[1])
                 port = match.group(1) if match else 443

lib/core/common.py

@@ -52,7 +52,6 @@ from lib.core.convert import base64unpickle
 from lib.core.convert import hexdecode
 from lib.core.convert import htmlunescape
 from lib.core.convert import stdoutencode
-from lib.core.convert import unicodeencode
 from lib.core.convert import utf8encode
 from lib.core.data import conf
 from lib.core.data import kb
@@ -894,14 +893,14 @@ def setColor(message, color=None, bold=False, level=None):
     retVal = message
     level = level or extractRegexResult(r"\[(?P<result>%s)\]" % '|'.join(_[0] for _ in getPublicTypeMembers(LOGGING_LEVELS)), message)
 
-    if isinstance(level, unicode):
-        level = unicodeencode(level)
-
     if message and getattr(LOGGER_HANDLER, "is_tty", False):  # colorizing handler
         if bold or color:
             retVal = colored(message, color=color, on_color=None, attrs=("bold",) if bold else None)
         elif level:
-            level = getattr(logging, level, None) if isinstance(level, six.string_types) else level
+            try:
+                level = getattr(logging, level, None)
+            except UnicodeError:
+                level = None
             retVal = LOGGER_HANDLER.colorize(message, level)
 
     return retVal
@@ -989,7 +988,7 @@ def dataToOutFile(filename, data):
 
             try:
                 with open(retVal, "w+b") as f:  # has to stay as non-codecs because data is raw ASCII encoded data
-                    f.write(unicodeencode(data))
+                    f.write(getBytes(data))
             except UnicodeEncodeError as ex:
                 _ = normalizeUnicode(filename)
                 if filename != _:
@@ -2431,7 +2430,7 @@ def getUnicode(value, encoding=None, noneToNull=False):
         except UnicodeDecodeError:
             return six.text_type(str(value), errors="ignore")  # encoding ignored for non-basestring instances
 
-def getBytes(value):
+def getBytes(value, encoding=UNICODE_ENCODING):
     """
     Returns byte representation of provided Unicode value
 
@@ -2446,11 +2445,11 @@ def getBytes(value):
             for char in xrange(0xF0000, 0xF00FF + 1):
                 value = value.replace(unichr(char), "%s%02x" % (SAFE_HEX_MARKER, char - 0xF0000))
 
-            retVal = value.encode(UNICODE_ENCODING)
+            retVal = value.encode(encoding)
 
             retVal = re.sub(r"%s([0-9a-f]{2})" % SAFE_HEX_MARKER, lambda _: _.group(1).decode("hex"), retVal)
         else:
-            retVal = value.encode(UNICODE_ENCODING)
+            retVal = value.encode(encoding)
             retVal = re.sub(r"\\x([0-9a-f]{2})", lambda _: _.group(1).decode("hex"), retVal)
 
     return retVal
@@ -4171,7 +4170,7 @@ def findPageForms(content, url, raise_=False, addToTargets=False):
 
     class _(io.BytesIO):
         def __init__(self, content, url):
-            io.BytesIO.__init__(self, unicodeencode(content, kb.pageEncoding) if isinstance(content, unicode) else content)
+            io.BytesIO.__init__(self, getBytes(content, kb.pageEncoding))
             self._url = url
 
         def geturl(self):

lib/core/dump.py

@@ -17,6 +17,7 @@ from lib.core.common import Backend
 from lib.core.common import checkFile
 from lib.core.common import dataToDumpFile
 from lib.core.common import dataToStdout
+from lib.core.common import getBytes
 from lib.core.common import getSafeExString
 from lib.core.common import getUnicode
 from lib.core.common import isListLike
@@ -26,7 +27,6 @@ from lib.core.common import openFile
 from lib.core.common import prioritySortColumns
 from lib.core.common import randomInt
 from lib.core.common import safeCSValue
-from lib.core.common import unicodeencode
 from lib.core.common import unsafeSQLIdentificatorNaming
 from lib.core.compat import xrange
 from lib.core.data import conf
@@ -422,8 +422,8 @@ class Dump(object):
                 except:
                     warnFile = True
 
-                    _ = unicodeencode(re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, unsafeSQLIdentificatorNaming(db)))
+                    _ = re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, unsafeSQLIdentificatorNaming(db))
-                    dumpDbPath = os.path.join(conf.dumpPath, "%s-%s" % (_, hashlib.md5(unicodeencode(db)).hexdigest()[:8]))
+                    dumpDbPath = os.path.join(conf.dumpPath, "%s-%s" % (_, hashlib.md5(getBytes(db)).hexdigest()[:8]))
 
                     if not os.path.isdir(dumpDbPath):
                         try:
@@ -456,8 +456,8 @@ class Dump(object):
 
                     _ = re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, normalizeUnicode(unsafeSQLIdentificatorNaming(table)))
                     if len(_) < len(table) or IS_WIN and table.upper() in WINDOWS_RESERVED_NAMES:
-                        _ = unicodeencode(re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, unsafeSQLIdentificatorNaming(table)))
+                        _ = re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, unsafeSQLIdentificatorNaming(table))
-                        dumpFileName = os.path.join(dumpDbPath, "%s-%s.%s" % (_, hashlib.md5(unicodeencode(table)).hexdigest()[:8], conf.dumpFormat.lower()))
+                        dumpFileName = os.path.join(dumpDbPath, "%s-%s.%s" % (_, hashlib.md5(getBytes(table)).hexdigest()[:8], conf.dumpFormat.lower()))
                     else:
                         dumpFileName = os.path.join(dumpDbPath, "%s.%s" % (_, conf.dumpFormat.lower()))
             else:

lib/core/settings.py

@@ -17,7 +17,7 @@ from lib.core.enums import DBMS_DIRECTORY_NAME
 from lib.core.enums import OS
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.3.4.27"
+VERSION = "1.3.4.28"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/request/connect.py

@@ -55,7 +55,6 @@ from lib.core.common import singleTimeLogMessage
 from lib.core.common import singleTimeWarnMessage
 from lib.core.common import stdev
 from lib.core.common import wasLastResponseDelayed
-from lib.core.common import unicodeencode
 from lib.core.common import unsafeVariableNaming
 from lib.core.common import urldecode
 from lib.core.common import urlencode
@@ -416,10 +415,9 @@ class Connect(object):
 
             for key, value in headers.items():
                 del headers[key]
-                value = unicodeencode(value, kb.pageEncoding)
                 for char in (r"\r", r"\n"):
                     value = re.sub(r"(%s)([^ \t])" % char, r"\g<1>\t\g<2>", value)
-                headers[unicodeencode(key, kb.pageEncoding)] = value.strip("\r\n")
+                headers[getBytes(key)] = getBytes(value.strip("\r\n"))
 
             url = getBytes(url)
             post = getBytes(post)
@@ -1134,7 +1132,7 @@ class Connect(object):
 
             while True:
                 try:
-                    compile(unicodeencode(conf.evalCode.replace(';', '\n')), "", "exec")
+                    compile(getBytes(conf.evalCode.replace(';', '\n')), "", "exec")
                 except SyntaxError as ex:
                     if ex.text:
                         original = replacement = ex.text.strip()

lib/techniques/union/use.py

@@ -21,6 +21,8 @@ from lib.core.common import dataToStdout
 from lib.core.common import extractRegexResult
 from lib.core.common import firstNotNone
 from lib.core.common import flattenValue
+from lib.core.common import safeStringFormat
+from lib.core.common import getBytes
 from lib.core.common import getConsoleWidth
 from lib.core.common import getPartRun
 from lib.core.common import getUnicode
@@ -54,7 +56,6 @@ from lib.core.settings import MAX_BUFFERED_PARTIAL_UNION_LENGTH
 from lib.core.settings import NULL
 from lib.core.settings import SQL_SCALAR_REGEX
 from lib.core.settings import TURN_OFF_RESUME_INFO_LIMIT
-from lib.core.settings import UNICODE_ENCODING
 from lib.core.threads import getCurrentThreadData
 from lib.core.threads import runThreads
 from lib.core.unescaper import unescaper
@@ -109,7 +110,7 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
             output = extractRegexResult(r"(?P<result>(<row.+?/>)+)", page)
             if output:
                 try:
-                    root = xml.etree.ElementTree.fromstring("<root>%s</root>" % output.encode(UNICODE_ENCODING))
+                    root = xml.etree.ElementTree.fromstring(safeStringFormat("<root>%s</root>", getBytes(output)))
                     retVal = ""
                     for column in kb.dumpColumns:
                         base64 = True

lib/utils/hash.py

@@ -50,6 +50,7 @@ from lib.core.common import Backend
 from lib.core.common import checkFile
 from lib.core.common import clearConsoleLine
 from lib.core.common import dataToStdout
+from lib.core.common import getBytes
 from lib.core.common import getFileItems
 from lib.core.common import getPublicTypeMembers
 from lib.core.common import getSafeExString
@@ -102,8 +103,7 @@ def mysql_passwd(password, uppercase=True):
     '*00E247AC5F9AF26AE0194B41E1E769DEE1429A29'
     """
 
-    if isinstance(password, six.text_type):
+    password = getBytes(password)
-        password = password.encode(UNICODE_ENCODING)
 
     retVal = "*%s" % sha1(sha1(password).digest()).hexdigest()
 
@@ -143,11 +143,8 @@ def postgres_passwd(password, username, uppercase=False):
     'md599e5ea7a6f7c3269995cba3927fd0093'
     """
 
-    if isinstance(username, six.text_type):
+    username = getBytes(username)
-        username = username.encode(UNICODE_ENCODING)
+    password = getBytes(password)
-
-    if isinstance(password, six.text_type):
-        password = password.encode(UNICODE_ENCODING)
 
     retVal = "md5%s" % md5(password + username).hexdigest()
 
@@ -232,11 +229,8 @@ def oracle_old_passwd(password, username, uppercase=True):  # prior to version '
 
     IV, pad = "\0" * 8, "\0"
 
-    if isinstance(username, six.text_type):
+    username = getBytes(username)
-        username = username.encode(UNICODE_ENCODING)
+    password = getBytes(password)
-
-    if isinstance(password, six.text_type):
-        password = password.encode(UNICODE_ENCODING)
 
     unistr = "".join("\0%s" % c for c in (username + password).upper())
 
@@ -255,8 +249,7 @@ def md5_generic_passwd(password, uppercase=False):
     '179ad45c6ce2cb97cf1029e212046e81'
     """
 
-    if isinstance(password, six.text_type):
+    password = getBytes(password)
-        password = password.encode(UNICODE_ENCODING)
 
     retVal = md5(password).hexdigest()
 
@@ -268,8 +261,7 @@ def sha1_generic_passwd(password, uppercase=False):
     '206c80413b9a96c1312cc346b7d2517b84463edd'
     """
 
-    if isinstance(password, six.text_type):
+    password = getBytes(password)
-        password = password.encode(UNICODE_ENCODING)
 
     retVal = sha1(password).hexdigest()
 
@@ -281,8 +273,7 @@ def apache_sha1_passwd(password, **kwargs):
     '{SHA}IGyAQTualsExLMNGt9JRe4RGPt0='
     """
 
-    if isinstance(password, six.text_type):
+    password = getBytes(password)
-        password = password.encode(UNICODE_ENCODING)
 
     return "{SHA}%s" % base64.b64encode(sha1(password).digest())
 
@@ -292,11 +283,8 @@ def ssha_passwd(password, salt, **kwargs):
     '{SSHA}mU1HPTvnmoXOhE4ROHP6sWfbfoRzYWx0'
     """
 
-    if isinstance(password, six.text_type):
+    password = getBytes(password)
-        password = password.encode(UNICODE_ENCODING)
+    salt = getBytes(salt)
-
-    if isinstance(salt, six.text_type):
-        salt = salt.encode(UNICODE_ENCODING)
 
     return "{SSHA}%s" % base64.b64encode(sha1(password + salt).digest() + salt)
 
@@ -306,11 +294,8 @@ def ssha256_passwd(password, salt, **kwargs):
     '{SSHA256}hhubsLrO/Aje9F/kJrgv5ZLE40UmTrVWvI7Dt6InP99zYWx0'
     """
 
-    if isinstance(password, six.text_type):
+    password = getBytes(password)
-        password = password.encode(UNICODE_ENCODING)
+    salt = getBytes(salt)
-
-    if isinstance(salt, six.text_type):
-        salt = salt.encode(UNICODE_ENCODING)
 
     return "{SSHA256}%s" % base64.b64encode(sha256(password + salt).digest() + salt)
 
@@ -320,11 +305,8 @@ def ssha512_passwd(password, salt, **kwargs):
     '{SSHA512}mCUSLfPMhXCQOJl9WHW/QMn9v9sjq7Ht/Wk7iVau8vLOfh+PeynkGMikqIE8sStFd0khdfcCD8xZmC6UyjTxsHNhbHQ='
     """
 
-    if isinstance(password, six.text_type):
+    password = getBytes(password)
-        password = password.encode(UNICODE_ENCODING)
+    salt = getBytes(salt)
-
-    if isinstance(salt, six.text_type):
-        salt = salt.encode(UNICODE_ENCODING)
 
     return "{SSHA512}%s" % base64.b64encode(sha512(password + salt).digest() + salt)
 
@@ -334,8 +316,7 @@ def sha224_generic_passwd(password, uppercase=False):
     '648db6019764b598f75ab6b7616d2e82563a00eb1531680e19ac4c6f'
     """
 
-    if isinstance(password, six.text_type):
+    password = getBytes(password)
-        password = password.encode(UNICODE_ENCODING)
 
     retVal = sha224(password).hexdigest()
 
@@ -347,8 +328,7 @@ def sha256_generic_passwd(password, uppercase=False):
     '13d249f2cb4127b40cfa757866850278793f814ded3c587fe5889e889a7a9f6c'
     """
 
-    if isinstance(password, six.text_type):
+    password = getBytes(password)
-        password = password.encode(UNICODE_ENCODING)
 
     retVal = sha256(password).hexdigest()
 
@@ -360,8 +340,7 @@ def sha384_generic_passwd(password, uppercase=False):
     '6823546e56adf46849343be991d4b1be9b432e42ed1b4bb90635a0e4b930e49b9ca007bc3e04bf0a4e0df6f1f82769bf'
     """
 
-    if isinstance(password, six.text_type):
+    password = getBytes(password)
-        password = password.encode(UNICODE_ENCODING)
 
     retVal = sha384(password).hexdigest()
 
@@ -373,8 +352,7 @@ def sha512_generic_passwd(password, uppercase=False):
     '78ddc8555bb1677ff5af75ba5fc02cb30bb592b0610277ae15055e189b77fe3fda496e5027a3d99ec85d54941adee1cc174b50438fdc21d82d0a79f85b58cf44'
     """
 
-    if isinstance(password, six.text_type):
+    password = getBytes(password)
-        password = password.encode(UNICODE_ENCODING)
 
     retVal = sha512(password).hexdigest()
 
@@ -392,11 +370,8 @@ def crypt_generic_passwd(password, salt, **kwargs):
     'rl.3StKT.4T8M'
     """
 
-    if isinstance(password, six.text_type):
+    password = getBytes(password)
-        password = password.encode(UNICODE_ENCODING)
+    salt = getBytes(salt)
-
-    if isinstance(salt, six.text_type):
-        salt = salt.encode(UNICODE_ENCODING)
 
     return crypt(password, salt)
 
@@ -419,14 +394,9 @@ def unix_md5_passwd(password, salt, magic="$1$", **kwargs):
 
         return output
 
-    if isinstance(password, six.text_type):
+    password = getBytes(password)
-        password = password.encode(UNICODE_ENCODING)
+    magic = getBytes(magic)
-
+    salt = getBytes(salt)
-    if isinstance(magic, six.text_type):
-        magic = magic.encode(UNICODE_ENCODING)
-
-    if isinstance(salt, six.text_type):
-        salt = salt.encode(UNICODE_ENCODING)
 
     salt = salt[:8]
     ctx = password + magic + salt
@@ -486,11 +456,8 @@ def joomla_passwd(password, salt, **kwargs):
     'e3d5794da74e917637332e0d21b76328:6GGlnaquVXI80b3HRmSyE3K1wEFFaBIf'
     """
 
-    if isinstance(password, six.text_type):
+    password = getBytes(password)
-        password = password.encode(UNICODE_ENCODING)
+    salt = getBytes(salt)
-
-    if isinstance(salt, six.text_type):
-        salt = salt.encode(UNICODE_ENCODING)
 
     return "%s:%s" % (md5("%s%s" % (password, salt)).hexdigest(), salt)
 
@@ -502,11 +469,8 @@ def django_md5_passwd(password, salt, **kwargs):
     'md5$salt$972141bcbcb6a0acc96e92309175b3c5'
     """
 
-    if isinstance(password, six.text_type):
+    password = getBytes(password)
-        password = password.encode(UNICODE_ENCODING)
+    salt = getBytes(salt)
-
-    if isinstance(salt, six.text_type):
-        salt = salt.encode(UNICODE_ENCODING)
 
     return "md5$%s$%s" % (salt, md5("%s%s" % (salt, password)).hexdigest())
 
@@ -518,11 +482,8 @@ def django_sha1_passwd(password, salt, **kwargs):
     'sha1$salt$6ce0e522aba69d8baa873f01420fccd0250fc5b2'
     """
 
-    if isinstance(password, six.text_type):
+    password = getBytes(password)
-        password = password.encode(UNICODE_ENCODING)
+    salt = getBytes(salt)
-
-    if isinstance(salt, six.text_type):
-        salt = salt.encode(UNICODE_ENCODING)
 
     return "sha1$%s$%s" % (salt, sha1("%s%s" % (salt, password)).hexdigest())
 
@@ -534,11 +495,8 @@ def vbulletin_passwd(password, salt, **kwargs):
     '85c4d8ea77ebef2236fb7e9d24ba9482:salt'
     """
 
-    if isinstance(password, six.text_type):
+    password = getBytes(password)
-        password = password.encode(UNICODE_ENCODING)
+    salt = getBytes(salt)
-
-    if isinstance(salt, six.text_type):
-        salt = salt.encode(UNICODE_ENCODING)
 
     return "%s:%s" % (md5("%s%s" % (md5(password).hexdigest(), salt)).hexdigest(), salt)
 
@@ -583,8 +541,7 @@ def wordpress_passwd(password, salt, count, prefix, **kwargs):
 
         return output
 
-    if isinstance(password, six.text_type):
+    password = getBytes(password)
-        password = password.encode(UNICODE_ENCODING)
 
     cipher = md5(salt)
     cipher.update(password)

lib/utils/hashdb.py

@@ -11,6 +11,7 @@ import sqlite3
 import threading
 import time
 
+from lib.core.common import getBytes
 from lib.core.common import getSafeExString
 from lib.core.common import getUnicode
 from lib.core.common import serializeObject
@@ -23,9 +24,9 @@ from lib.core.settings import HASHDB_END_TRANSACTION_RETRIES
 from lib.core.settings import HASHDB_FLUSH_RETRIES
 from lib.core.settings import HASHDB_FLUSH_THRESHOLD
 from lib.core.settings import HASHDB_RETRIEVE_RETRIES
-from lib.core.settings import UNICODE_ENCODING
 from lib.core.threads import getCurrentThreadData
 from lib.core.threads import getCurrentThreadName
+from thirdparty import six
 
 class HashDB(object):
     def __init__(self, filepath):
@@ -67,7 +68,7 @@ class HashDB(object):
 
     @staticmethod
     def hashKey(key):
-        key = key.encode(UNICODE_ENCODING) if isinstance(key, unicode) else repr(key)
+        key = getBytes(key if isinstance(key, six.text_type) else repr(key))
         retVal = int(hashlib.md5(key).hexdigest(), 16) & 0x7fffffffffffffff  # Reference: http://stackoverflow.com/a/4448400
         return retVal