From da15701a5593589b0910882201a4177b05b981ff Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Thu, 18 Apr 2019 16:06:19 +0200 Subject: [PATCH] Minor DREI updates --- lib/controller/checks.py | 3 +- lib/core/common.py | 19 +++---- lib/core/dump.py | 10 ++-- lib/core/settings.py | 2 +- lib/request/connect.py | 6 +- lib/techniques/union/use.py | 5 +- lib/utils/hash.py | 109 +++++++++++------------------------- lib/utils/hashdb.py | 5 +- 8 files changed, 57 insertions(+), 102 deletions(-) diff --git a/lib/controller/checks.py b/lib/controller/checks.py index f8a4e5fc1..34eb41b5f 100644 --- a/lib/controller/checks.py +++ b/lib/controller/checks.py @@ -49,7 +49,6 @@ from lib.core.common import urlencode from lib.core.common import wasLastResponseDBMSError from lib.core.common import wasLastResponseHTTPError from lib.core.compat import xrange -from lib.core.convert import unicodeencode from lib.core.defaults import defaults from lib.core.data import conf from lib.core.data import kb @@ -1611,7 +1610,7 @@ def checkConnection(suppressOutput=False): kb.errorIsNone = True if kb.redirectChoice == REDIRECTION.YES and threadData.lastRedirectURL and threadData.lastRedirectURL[0] == threadData.lastRequestUID: - if (threadData.lastRedirectURL[1] or "").startswith("https://") and unicodeencode(conf.hostname) in threadData.lastRedirectURL[1]: + if (threadData.lastRedirectURL[1] or "").startswith("https://") and conf.hostname in getUnicode(threadData.lastRedirectURL[1]): conf.url = re.sub(r"https?://", "https://", conf.url) match = re.search(r":(\d+)", threadData.lastRedirectURL[1]) port = match.group(1) if match else 443 diff --git a/lib/core/common.py b/lib/core/common.py index eb9f95861..1aaa999ca 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -52,7 +52,6 @@ from lib.core.convert import base64unpickle from lib.core.convert import hexdecode from lib.core.convert import htmlunescape from lib.core.convert import stdoutencode -from lib.core.convert import unicodeencode from lib.core.convert import utf8encode from lib.core.data import conf from lib.core.data import kb @@ -894,14 +893,14 @@ def setColor(message, color=None, bold=False, level=None): retVal = message level = level or extractRegexResult(r"\[(?P%s)\]" % '|'.join(_[0] for _ in getPublicTypeMembers(LOGGING_LEVELS)), message) - if isinstance(level, unicode): - level = unicodeencode(level) - if message and getattr(LOGGER_HANDLER, "is_tty", False): # colorizing handler if bold or color: retVal = colored(message, color=color, on_color=None, attrs=("bold",) if bold else None) elif level: - level = getattr(logging, level, None) if isinstance(level, six.string_types) else level + try: + level = getattr(logging, level, None) + except UnicodeError: + level = None retVal = LOGGER_HANDLER.colorize(message, level) return retVal @@ -989,7 +988,7 @@ def dataToOutFile(filename, data): try: with open(retVal, "w+b") as f: # has to stay as non-codecs because data is raw ASCII encoded data - f.write(unicodeencode(data)) + f.write(getBytes(data)) except UnicodeEncodeError as ex: _ = normalizeUnicode(filename) if filename != _: @@ -2431,7 +2430,7 @@ def getUnicode(value, encoding=None, noneToNull=False): except UnicodeDecodeError: return six.text_type(str(value), errors="ignore") # encoding ignored for non-basestring instances -def getBytes(value): +def getBytes(value, encoding=UNICODE_ENCODING): """ Returns byte representation of provided Unicode value @@ -2446,11 +2445,11 @@ def getBytes(value): for char in xrange(0xF0000, 0xF00FF + 1): value = value.replace(unichr(char), "%s%02x" % (SAFE_HEX_MARKER, char - 0xF0000)) - retVal = value.encode(UNICODE_ENCODING) + retVal = value.encode(encoding) retVal = re.sub(r"%s([0-9a-f]{2})" % SAFE_HEX_MARKER, lambda _: _.group(1).decode("hex"), retVal) else: - retVal = value.encode(UNICODE_ENCODING) + retVal = value.encode(encoding) retVal = re.sub(r"\\x([0-9a-f]{2})", lambda _: _.group(1).decode("hex"), retVal) return retVal @@ -4171,7 +4170,7 @@ def findPageForms(content, url, raise_=False, addToTargets=False): class _(io.BytesIO): def __init__(self, content, url): - io.BytesIO.__init__(self, unicodeencode(content, kb.pageEncoding) if isinstance(content, unicode) else content) + io.BytesIO.__init__(self, getBytes(content, kb.pageEncoding)) self._url = url def geturl(self): diff --git a/lib/core/dump.py b/lib/core/dump.py index 74a6d6c01..e20edd14b 100644 --- a/lib/core/dump.py +++ b/lib/core/dump.py @@ -17,6 +17,7 @@ from lib.core.common import Backend from lib.core.common import checkFile from lib.core.common import dataToDumpFile from lib.core.common import dataToStdout +from lib.core.common import getBytes from lib.core.common import getSafeExString from lib.core.common import getUnicode from lib.core.common import isListLike @@ -26,7 +27,6 @@ from lib.core.common import openFile from lib.core.common import prioritySortColumns from lib.core.common import randomInt from lib.core.common import safeCSValue -from lib.core.common import unicodeencode from lib.core.common import unsafeSQLIdentificatorNaming from lib.core.compat import xrange from lib.core.data import conf @@ -422,8 +422,8 @@ class Dump(object): except: warnFile = True - _ = unicodeencode(re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, unsafeSQLIdentificatorNaming(db))) - dumpDbPath = os.path.join(conf.dumpPath, "%s-%s" % (_, hashlib.md5(unicodeencode(db)).hexdigest()[:8])) + _ = re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, unsafeSQLIdentificatorNaming(db)) + dumpDbPath = os.path.join(conf.dumpPath, "%s-%s" % (_, hashlib.md5(getBytes(db)).hexdigest()[:8])) if not os.path.isdir(dumpDbPath): try: @@ -456,8 +456,8 @@ class Dump(object): _ = re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, normalizeUnicode(unsafeSQLIdentificatorNaming(table))) if len(_) < len(table) or IS_WIN and table.upper() in WINDOWS_RESERVED_NAMES: - _ = unicodeencode(re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, unsafeSQLIdentificatorNaming(table))) - dumpFileName = os.path.join(dumpDbPath, "%s-%s.%s" % (_, hashlib.md5(unicodeencode(table)).hexdigest()[:8], conf.dumpFormat.lower())) + _ = re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, unsafeSQLIdentificatorNaming(table)) + dumpFileName = os.path.join(dumpDbPath, "%s-%s.%s" % (_, hashlib.md5(getBytes(table)).hexdigest()[:8], conf.dumpFormat.lower())) else: dumpFileName = os.path.join(dumpDbPath, "%s.%s" % (_, conf.dumpFormat.lower())) else: diff --git a/lib/core/settings.py b/lib/core/settings.py index b6a7de39b..f3f2ddcc6 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -17,7 +17,7 @@ from lib.core.enums import DBMS_DIRECTORY_NAME from lib.core.enums import OS # sqlmap version (...) -VERSION = "1.3.4.27" +VERSION = "1.3.4.28" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/request/connect.py b/lib/request/connect.py index d74539bed..e89dfafa9 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -55,7 +55,6 @@ from lib.core.common import singleTimeLogMessage from lib.core.common import singleTimeWarnMessage from lib.core.common import stdev from lib.core.common import wasLastResponseDelayed -from lib.core.common import unicodeencode from lib.core.common import unsafeVariableNaming from lib.core.common import urldecode from lib.core.common import urlencode @@ -416,10 +415,9 @@ class Connect(object): for key, value in headers.items(): del headers[key] - value = unicodeencode(value, kb.pageEncoding) for char in (r"\r", r"\n"): value = re.sub(r"(%s)([^ \t])" % char, r"\g<1>\t\g<2>", value) - headers[unicodeencode(key, kb.pageEncoding)] = value.strip("\r\n") + headers[getBytes(key)] = getBytes(value.strip("\r\n")) url = getBytes(url) post = getBytes(post) @@ -1134,7 +1132,7 @@ class Connect(object): while True: try: - compile(unicodeencode(conf.evalCode.replace(';', '\n')), "", "exec") + compile(getBytes(conf.evalCode.replace(';', '\n')), "", "exec") except SyntaxError as ex: if ex.text: original = replacement = ex.text.strip() diff --git a/lib/techniques/union/use.py b/lib/techniques/union/use.py index ffb3085f1..82fcd462d 100644 --- a/lib/techniques/union/use.py +++ b/lib/techniques/union/use.py @@ -21,6 +21,8 @@ from lib.core.common import dataToStdout from lib.core.common import extractRegexResult from lib.core.common import firstNotNone from lib.core.common import flattenValue +from lib.core.common import safeStringFormat +from lib.core.common import getBytes from lib.core.common import getConsoleWidth from lib.core.common import getPartRun from lib.core.common import getUnicode @@ -54,7 +56,6 @@ from lib.core.settings import MAX_BUFFERED_PARTIAL_UNION_LENGTH from lib.core.settings import NULL from lib.core.settings import SQL_SCALAR_REGEX from lib.core.settings import TURN_OFF_RESUME_INFO_LIMIT -from lib.core.settings import UNICODE_ENCODING from lib.core.threads import getCurrentThreadData from lib.core.threads import runThreads from lib.core.unescaper import unescaper @@ -109,7 +110,7 @@ def _oneShotUnionUse(expression, unpack=True, limited=False): output = extractRegexResult(r"(?P()+)", page) if output: try: - root = xml.etree.ElementTree.fromstring("%s" % output.encode(UNICODE_ENCODING)) + root = xml.etree.ElementTree.fromstring(safeStringFormat("%s", getBytes(output))) retVal = "" for column in kb.dumpColumns: base64 = True diff --git a/lib/utils/hash.py b/lib/utils/hash.py index 03c5c4db1..cf52fb562 100644 --- a/lib/utils/hash.py +++ b/lib/utils/hash.py @@ -50,6 +50,7 @@ from lib.core.common import Backend from lib.core.common import checkFile from lib.core.common import clearConsoleLine from lib.core.common import dataToStdout +from lib.core.common import getBytes from lib.core.common import getFileItems from lib.core.common import getPublicTypeMembers from lib.core.common import getSafeExString @@ -102,8 +103,7 @@ def mysql_passwd(password, uppercase=True): '*00E247AC5F9AF26AE0194B41E1E769DEE1429A29' """ - if isinstance(password, six.text_type): - password = password.encode(UNICODE_ENCODING) + password = getBytes(password) retVal = "*%s" % sha1(sha1(password).digest()).hexdigest() @@ -143,11 +143,8 @@ def postgres_passwd(password, username, uppercase=False): 'md599e5ea7a6f7c3269995cba3927fd0093' """ - if isinstance(username, six.text_type): - username = username.encode(UNICODE_ENCODING) - - if isinstance(password, six.text_type): - password = password.encode(UNICODE_ENCODING) + username = getBytes(username) + password = getBytes(password) retVal = "md5%s" % md5(password + username).hexdigest() @@ -232,11 +229,8 @@ def oracle_old_passwd(password, username, uppercase=True): # prior to version ' IV, pad = "\0" * 8, "\0" - if isinstance(username, six.text_type): - username = username.encode(UNICODE_ENCODING) - - if isinstance(password, six.text_type): - password = password.encode(UNICODE_ENCODING) + username = getBytes(username) + password = getBytes(password) unistr = "".join("\0%s" % c for c in (username + password).upper()) @@ -255,8 +249,7 @@ def md5_generic_passwd(password, uppercase=False): '179ad45c6ce2cb97cf1029e212046e81' """ - if isinstance(password, six.text_type): - password = password.encode(UNICODE_ENCODING) + password = getBytes(password) retVal = md5(password).hexdigest() @@ -268,8 +261,7 @@ def sha1_generic_passwd(password, uppercase=False): '206c80413b9a96c1312cc346b7d2517b84463edd' """ - if isinstance(password, six.text_type): - password = password.encode(UNICODE_ENCODING) + password = getBytes(password) retVal = sha1(password).hexdigest() @@ -281,8 +273,7 @@ def apache_sha1_passwd(password, **kwargs): '{SHA}IGyAQTualsExLMNGt9JRe4RGPt0=' """ - if isinstance(password, six.text_type): - password = password.encode(UNICODE_ENCODING) + password = getBytes(password) return "{SHA}%s" % base64.b64encode(sha1(password).digest()) @@ -292,11 +283,8 @@ def ssha_passwd(password, salt, **kwargs): '{SSHA}mU1HPTvnmoXOhE4ROHP6sWfbfoRzYWx0' """ - if isinstance(password, six.text_type): - password = password.encode(UNICODE_ENCODING) - - if isinstance(salt, six.text_type): - salt = salt.encode(UNICODE_ENCODING) + password = getBytes(password) + salt = getBytes(salt) return "{SSHA}%s" % base64.b64encode(sha1(password + salt).digest() + salt) @@ -306,11 +294,8 @@ def ssha256_passwd(password, salt, **kwargs): '{SSHA256}hhubsLrO/Aje9F/kJrgv5ZLE40UmTrVWvI7Dt6InP99zYWx0' """ - if isinstance(password, six.text_type): - password = password.encode(UNICODE_ENCODING) - - if isinstance(salt, six.text_type): - salt = salt.encode(UNICODE_ENCODING) + password = getBytes(password) + salt = getBytes(salt) return "{SSHA256}%s" % base64.b64encode(sha256(password + salt).digest() + salt) @@ -320,11 +305,8 @@ def ssha512_passwd(password, salt, **kwargs): '{SSHA512}mCUSLfPMhXCQOJl9WHW/QMn9v9sjq7Ht/Wk7iVau8vLOfh+PeynkGMikqIE8sStFd0khdfcCD8xZmC6UyjTxsHNhbHQ=' """ - if isinstance(password, six.text_type): - password = password.encode(UNICODE_ENCODING) - - if isinstance(salt, six.text_type): - salt = salt.encode(UNICODE_ENCODING) + password = getBytes(password) + salt = getBytes(salt) return "{SSHA512}%s" % base64.b64encode(sha512(password + salt).digest() + salt) @@ -334,8 +316,7 @@ def sha224_generic_passwd(password, uppercase=False): '648db6019764b598f75ab6b7616d2e82563a00eb1531680e19ac4c6f' """ - if isinstance(password, six.text_type): - password = password.encode(UNICODE_ENCODING) + password = getBytes(password) retVal = sha224(password).hexdigest() @@ -347,8 +328,7 @@ def sha256_generic_passwd(password, uppercase=False): '13d249f2cb4127b40cfa757866850278793f814ded3c587fe5889e889a7a9f6c' """ - if isinstance(password, six.text_type): - password = password.encode(UNICODE_ENCODING) + password = getBytes(password) retVal = sha256(password).hexdigest() @@ -360,8 +340,7 @@ def sha384_generic_passwd(password, uppercase=False): '6823546e56adf46849343be991d4b1be9b432e42ed1b4bb90635a0e4b930e49b9ca007bc3e04bf0a4e0df6f1f82769bf' """ - if isinstance(password, six.text_type): - password = password.encode(UNICODE_ENCODING) + password = getBytes(password) retVal = sha384(password).hexdigest() @@ -373,8 +352,7 @@ def sha512_generic_passwd(password, uppercase=False): '78ddc8555bb1677ff5af75ba5fc02cb30bb592b0610277ae15055e189b77fe3fda496e5027a3d99ec85d54941adee1cc174b50438fdc21d82d0a79f85b58cf44' """ - if isinstance(password, six.text_type): - password = password.encode(UNICODE_ENCODING) + password = getBytes(password) retVal = sha512(password).hexdigest() @@ -392,11 +370,8 @@ def crypt_generic_passwd(password, salt, **kwargs): 'rl.3StKT.4T8M' """ - if isinstance(password, six.text_type): - password = password.encode(UNICODE_ENCODING) - - if isinstance(salt, six.text_type): - salt = salt.encode(UNICODE_ENCODING) + password = getBytes(password) + salt = getBytes(salt) return crypt(password, salt) @@ -419,14 +394,9 @@ def unix_md5_passwd(password, salt, magic="$1$", **kwargs): return output - if isinstance(password, six.text_type): - password = password.encode(UNICODE_ENCODING) - - if isinstance(magic, six.text_type): - magic = magic.encode(UNICODE_ENCODING) - - if isinstance(salt, six.text_type): - salt = salt.encode(UNICODE_ENCODING) + password = getBytes(password) + magic = getBytes(magic) + salt = getBytes(salt) salt = salt[:8] ctx = password + magic + salt @@ -486,11 +456,8 @@ def joomla_passwd(password, salt, **kwargs): 'e3d5794da74e917637332e0d21b76328:6GGlnaquVXI80b3HRmSyE3K1wEFFaBIf' """ - if isinstance(password, six.text_type): - password = password.encode(UNICODE_ENCODING) - - if isinstance(salt, six.text_type): - salt = salt.encode(UNICODE_ENCODING) + password = getBytes(password) + salt = getBytes(salt) return "%s:%s" % (md5("%s%s" % (password, salt)).hexdigest(), salt) @@ -502,11 +469,8 @@ def django_md5_passwd(password, salt, **kwargs): 'md5$salt$972141bcbcb6a0acc96e92309175b3c5' """ - if isinstance(password, six.text_type): - password = password.encode(UNICODE_ENCODING) - - if isinstance(salt, six.text_type): - salt = salt.encode(UNICODE_ENCODING) + password = getBytes(password) + salt = getBytes(salt) return "md5$%s$%s" % (salt, md5("%s%s" % (salt, password)).hexdigest()) @@ -518,11 +482,8 @@ def django_sha1_passwd(password, salt, **kwargs): 'sha1$salt$6ce0e522aba69d8baa873f01420fccd0250fc5b2' """ - if isinstance(password, six.text_type): - password = password.encode(UNICODE_ENCODING) - - if isinstance(salt, six.text_type): - salt = salt.encode(UNICODE_ENCODING) + password = getBytes(password) + salt = getBytes(salt) return "sha1$%s$%s" % (salt, sha1("%s%s" % (salt, password)).hexdigest()) @@ -534,11 +495,8 @@ def vbulletin_passwd(password, salt, **kwargs): '85c4d8ea77ebef2236fb7e9d24ba9482:salt' """ - if isinstance(password, six.text_type): - password = password.encode(UNICODE_ENCODING) - - if isinstance(salt, six.text_type): - salt = salt.encode(UNICODE_ENCODING) + password = getBytes(password) + salt = getBytes(salt) return "%s:%s" % (md5("%s%s" % (md5(password).hexdigest(), salt)).hexdigest(), salt) @@ -583,8 +541,7 @@ def wordpress_passwd(password, salt, count, prefix, **kwargs): return output - if isinstance(password, six.text_type): - password = password.encode(UNICODE_ENCODING) + password = getBytes(password) cipher = md5(salt) cipher.update(password) diff --git a/lib/utils/hashdb.py b/lib/utils/hashdb.py index d0189f725..2f415383a 100644 --- a/lib/utils/hashdb.py +++ b/lib/utils/hashdb.py @@ -11,6 +11,7 @@ import sqlite3 import threading import time +from lib.core.common import getBytes from lib.core.common import getSafeExString from lib.core.common import getUnicode from lib.core.common import serializeObject @@ -23,9 +24,9 @@ from lib.core.settings import HASHDB_END_TRANSACTION_RETRIES from lib.core.settings import HASHDB_FLUSH_RETRIES from lib.core.settings import HASHDB_FLUSH_THRESHOLD from lib.core.settings import HASHDB_RETRIEVE_RETRIES -from lib.core.settings import UNICODE_ENCODING from lib.core.threads import getCurrentThreadData from lib.core.threads import getCurrentThreadName +from thirdparty import six class HashDB(object): def __init__(self, filepath): @@ -67,7 +68,7 @@ class HashDB(object): @staticmethod def hashKey(key): - key = key.encode(UNICODE_ENCODING) if isinstance(key, unicode) else repr(key) + key = getBytes(key if isinstance(key, six.text_type) else repr(key)) retVal = int(hashlib.md5(key).hexdigest(), 16) & 0x7fffffffffffffff # Reference: http://stackoverflow.com/a/4448400 return retVal