From 09aba3b5ce760d9e379be200432254cf79fb3146 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 8 May 2019 12:28:50 +0200 Subject: [PATCH] More DREI updates --- extra/shutils/drei.sh | 3 ++- lib/controller/checks.py | 2 +- lib/core/common.py | 32 ++++++++++++++++------- lib/core/convert.py | 11 ++++++-- lib/core/settings.py | 2 +- lib/core/testing.py | 2 +- lib/parse/cmdline.py | 3 +++ lib/request/basic.py | 2 +- lib/request/connect.py | 3 +++ lib/takeover/metasploit.py | 4 +-- lib/utils/api.py | 2 +- lib/utils/crawler.py | 5 ++-- lib/utils/hash.py | 18 ++++++------- thirdparty/beautifulsoup/beautifulsoup.py | 5 ++-- thirdparty/fcrypt/fcrypt.py | 4 +-- thirdparty/magic/magic.py | 3 ++- 16 files changed, 65 insertions(+), 36 deletions(-) diff --git a/extra/shutils/drei.sh b/extra/shutils/drei.sh index 7bd3ab68a..dfd6da480 100755 --- a/extra/shutils/drei.sh +++ b/extra/shutils/drei.sh @@ -6,7 +6,8 @@ # Stress test against Python3 export SQLMAP_DREI=1 -for i in $(find . -iname "*.py" | grep -v __init__); do python3 -c 'import '`echo $i | cut -d '.' -f 2 | cut -d '/' -f 2- | sed 's/\//./g'`''; done +#for i in $(find . -iname "*.py" | grep -v __init__); do python3 -c 'import '`echo $i | cut -d '.' -f 2 | cut -d '/' -f 2- | sed 's/\//./g'`''; done +for i in $(find . -iname "*.py" | grep -v __init__); do PYTHONWARNINGS=all python3.7 -m compileall $i; done unset SQLMAP_DREI source `dirname "$0"`"/junk.sh" diff --git a/lib/controller/checks.py b/lib/controller/checks.py index 8c56752aa..1fc5e9d32 100644 --- a/lib/controller/checks.py +++ b/lib/controller/checks.py @@ -1614,7 +1614,7 @@ def checkConnection(suppressOutput=False): conf.url = re.sub(r"https?://", "https://", conf.url) match = re.search(r":(\d+)", threadData.lastRedirectURL[1]) port = match.group(1) if match else 443 - conf.url = re.sub(r":\d+(/|\Z)", ":%s\g<1>" % port, conf.url) + conf.url = re.sub(r":\d+(/|\Z)", r":%s\g<1>" % port, conf.url) except SqlmapConnectionException as ex: if conf.ipv6: diff --git a/lib/core/common.py b/lib/core/common.py index 9a2df03f2..a1c1ef56d 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -1635,7 +1635,7 @@ def expandAsteriskForColumns(expression): if expression != conf.sqlQuery: conf.db = db else: - expression = re.sub(r"([^\w])%s" % re.escape(conf.tbl), "\g<1>%s.%s" % (conf.db, conf.tbl), expression) + expression = re.sub(r"([^\w])%s" % re.escape(conf.tbl), r"\g<1>%s.%s" % (conf.db, conf.tbl), expression) else: conf.db = db @@ -1795,12 +1795,24 @@ def getFileType(filePath): >>> getFileType(__file__) 'text' + >>> getFileType(sys.executable) + 'binary' """ try: desc = getUnicode(magic.from_file(filePath) or "") except: - return "unknown" + desc = magic.MAGIC_UNKNOWN_FILETYPE + + if desc == magic.MAGIC_UNKNOWN_FILETYPE: + content = openFile(filePath, "rb", encoding=None).read() + + try: + content.decode() + except: + pass + else: + desc = "ascii" return "text" if any(_ in desc.lower() for _ in ("ascii", "text")) else "binary" @@ -2053,8 +2065,8 @@ def isWindowsDriveLetterPath(filepath): def posixToNtSlashes(filepath): """ - Replaces all occurrences of Posix slashes (/) in provided - filepath with NT ones (\) + Replaces all occurrences of Posix slashes in provided + filepath with NT backslashes >>> posixToNtSlashes('C:/Windows') 'C:\\\\Windows' @@ -2064,8 +2076,8 @@ def posixToNtSlashes(filepath): def ntToPosixSlashes(filepath): """ - Replaces all occurrences of NT slashes (\) in provided - filepath with Posix ones (/) + Replaces all occurrences of NT backslashes in provided + filepath with Posix slashes >>> ntToPosixSlashes('C:\\Windows') 'C:/Windows' @@ -2954,7 +2966,7 @@ def findDynamicContent(firstPage, secondPage): infoMsg = "searching for dynamic content" singleTimeLogMessage(infoMsg) - blocks = SequenceMatcher(None, firstPage, secondPage).get_matching_blocks() + blocks = list(SequenceMatcher(None, firstPage, secondPage).get_matching_blocks()) kb.dynamicMarkings = [] # Removing too small matching blocks @@ -4654,8 +4666,8 @@ def decloakToTemp(filename): content = decloak(filename) - parts = getBytes(os.path.split(filename[:-1])[-1]).split(b'.') - prefix, suffix = parts[0], b".%s" % parts[-1] + parts = os.path.split(filename[:-1])[-1].split('.') + prefix, suffix = parts[0], '.' + parts[-1] handle, filename = tempfile.mkstemp(prefix=prefix, suffix=suffix) os.close(handle) @@ -4692,7 +4704,7 @@ def getRequestHeader(request, name): if request and request.headers and name: _ = name.upper() - retVal = max(value if _ == key.upper() else type(value)() for key, value in request.header_items()) or None + retVal = max(getBytes(value if _ == key.upper() else "") for key, value in request.header_items()) or None return retVal diff --git a/lib/core/convert.py b/lib/core/convert.py index 6ed95c0c0..c46506f78 100644 --- a/lib/core/convert.py +++ b/lib/core/convert.py @@ -11,6 +11,7 @@ except: import pickle import base64 +import binascii import codecs import json import re @@ -160,7 +161,10 @@ def decodeHex(value, binary=True): if value.lower().startswith("0x"): value = value[2:] - retVal = codecs.decode(value, "hex") + try: + retVal = codecs.decode(value, "hex") + except LookupError: + retVal = binascii.unhexlify(value) if not binary: retVal = getText(retVal) @@ -180,7 +184,10 @@ def encodeHex(value, binary=True): if isinstance(value, six.text_type): value = value.encode(UNICODE_ENCODING) - retVal = codecs.encode(value, "hex") + try: + retVal = codecs.encode(value, "hex") + except LookupError: + retVal = binascii.hexlify(value) if not binary: retVal = getText(retVal) diff --git a/lib/core/settings.py b/lib/core/settings.py index 3663ccfc4..6c27234b0 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from lib.core.enums import OS from thirdparty import six # sqlmap version (...) -VERSION = "1.3.5.49" +VERSION = "1.3.5.50" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/core/testing.py b/lib/core/testing.py index 93fd34d42..33c50846f 100644 --- a/lib/core/testing.py +++ b/lib/core/testing.py @@ -410,7 +410,7 @@ def replaceVars(item, vars_): retVal = item if item and vars_: - for var in re.findall("\$\{([^}]+)\}", item): + for var in re.findall(r"\$\{([^}]+)\}", item): if var in vars_: retVal = retVal.replace("${%s}" % var, vars_[var]) diff --git a/lib/parse/cmdline.py b/lib/parse/cmdline.py index b5dae03f2..725e3d84c 100644 --- a/lib/parse/cmdline.py +++ b/lib/parse/cmdline.py @@ -697,6 +697,9 @@ def cmdLineParser(argv=None): parser.add_option("--murphy-rate", dest="murphyRate", type="int", help=SUPPRESS_HELP) + parser.add_option("--debug", dest="debug", action="store_true", + help=SUPPRESS_HELP) + parser.add_option("--disable-precon", dest="disablePrecon", action="store_true", help=SUPPRESS_HELP) diff --git a/lib/request/basic.py b/lib/request/basic.py index 831b278bb..7f4eb6724 100644 --- a/lib/request/basic.py +++ b/lib/request/basic.py @@ -323,7 +323,7 @@ def decodePage(page, contentEncoding, contentType): # e.g. Ãëàâà if b"&#" in page: page = re.sub(b"&#x([0-9a-f]{1,2});", lambda _: decodeHex(_.group(1) if len(_.group(1)) == 2 else "0%s" % _.group(1)), page) - page = re.sub(b"&#(\d{1,3});", lambda _: six.int2byte(int(_.group(1))) if int(_.group(1)) < 256 else _.group(0), page) + page = re.sub(b"&#(\\d{1,3});", lambda _: six.int2byte(int(_.group(1))) if int(_.group(1)) < 256 else _.group(0), page) # e.g. %20%28%29 if b"%" in page: diff --git a/lib/request/connect.py b/lib/request/connect.py index 79d54eea7..9979f31ab 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -674,6 +674,9 @@ class Connect(object): except (_urllib.error.URLError, socket.error, socket.timeout, _http_client.HTTPException, struct.error, binascii.Error, ProxyError, SqlmapCompressionException, WebSocketException, TypeError, ValueError, OverflowError): tbMsg = traceback.format_exc() + if conf.debug: + dataToStdout(tbMsg) + if checking: return None, None, None elif "no host given" in tbMsg: diff --git a/lib/takeover/metasploit.py b/lib/takeover/metasploit.py index c12d678f5..02fe1d793 100644 --- a/lib/takeover/metasploit.py +++ b/lib/takeover/metasploit.py @@ -573,7 +573,7 @@ class Metasploit: timeout = time.time() - start_time > METASPLOIT_SESSION_TIMEOUT if not initialized: - match = re.search(b"Meterpreter session ([\d]+) opened", out) + match = re.search(b"Meterpreter session ([\\d]+) opened", out) if match: self._loadMetExtensions(proc, match.group(1)) @@ -625,7 +625,7 @@ class Metasploit: pollProcess(process) payloadStderr = process.communicate()[1] - match = re.search(b"(Total size:|Length:|succeeded with size|Final size of exe file:) ([\d]+)", payloadStderr) + match = re.search(b"(Total size:|Length:|succeeded with size|Final size of exe file:) ([\\d]+)", payloadStderr) if match: payloadSize = int(match.group(2)) diff --git a/lib/utils/api.py b/lib/utils/api.py index 14c3760e4..41c38907e 100644 --- a/lib/utils/api.py +++ b/lib/utils/api.py @@ -737,7 +737,7 @@ def client(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, username=Non DataStore.password = password dbgMsg = "Example client access from command line:" - dbgMsg += "\n\t$ taskid=$(curl http://%s:%d/task/new 2>1 | grep -o -I '[a-f0-9]\{16\}') && echo $taskid" % (host, port) + dbgMsg += "\n\t$ taskid=$(curl http://%s:%d/task/new 2>1 | grep -o -I '[a-f0-9]\\{16\\}') && echo $taskid" % (host, port) dbgMsg += "\n\t$ curl -H \"Content-Type: application/json\" -X POST -d '{\"url\": \"http://testphp.vulnweb.com/artists.php?artist=1\"}' http://%s:%d/scan/$taskid/start" % (host, port) dbgMsg += "\n\t$ curl http://%s:%d/scan/$taskid/data" % (host, port) dbgMsg += "\n\t$ curl http://%s:%d/scan/$taskid/log" % (host, port) diff --git a/lib/utils/crawler.py b/lib/utils/crawler.py index 8f104928c..898bbbcf6 100644 --- a/lib/utils/crawler.py +++ b/lib/utils/crawler.py @@ -20,6 +20,7 @@ from lib.core.common import readInput from lib.core.common import safeCSValue from lib.core.common import urldecode from lib.core.compat import xrange +from lib.core.convert import htmlunescape from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger @@ -90,7 +91,7 @@ def crawl(target): tags = soup('a') if not tags: - tags = re.finditer(r'(?i)]+href="(?P[^>"]+)"', content) + tags = re.finditer(r'(?i)]+href=["\'](?P[^>"\']+)', content) for tag in tags: href = tag.get("href") if hasattr(tag, "get") else tag.group("href") @@ -98,7 +99,7 @@ def crawl(target): if href: if threadData.lastRedirectURL and threadData.lastRedirectURL[0] == threadData.lastRequestUID: current = threadData.lastRedirectURL[1] - url = _urllib.parse.urljoin(current, href) + url = _urllib.parse.urljoin(current, htmlunescape(href)) # flag to know if we are dealing with the same target host _ = checkSameHost(url, target) diff --git a/lib/utils/hash.py b/lib/utils/hash.py index 5c448c384..c03c81702 100644 --- a/lib/utils/hash.py +++ b/lib/utils/hash.py @@ -162,7 +162,7 @@ def mssql_passwd(password, salt, uppercase=False): """ binsalt = decodeHex(salt) - unistr = b"".join(b"%s\0" % _.encode(UNICODE_ENCODING) if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password) + unistr = b"".join((_.encode(UNICODE_ENCODING) + b"\0") if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password) retVal = "0100%s%s" % (salt, sha1(unistr + binsalt).hexdigest()) @@ -180,7 +180,7 @@ def mssql_old_passwd(password, salt, uppercase=True): # prior to version '2005' """ binsalt = decodeHex(salt) - unistr = b"".join(b"%s\0" % _.encode(UNICODE_ENCODING) if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password) + unistr = b"".join((_.encode(UNICODE_ENCODING) + b"\0") if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password) retVal = "0100%s%s%s" % (salt, sha1(unistr + binsalt).hexdigest(), sha1(unistr.upper() + binsalt).hexdigest()) @@ -196,7 +196,7 @@ def mssql_new_passwd(password, salt, uppercase=False): """ binsalt = decodeHex(salt) - unistr = b"".join(b"%s\0" % _.encode(UNICODE_ENCODING) if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password) + unistr = b"".join((_.encode(UNICODE_ENCODING) + b"\0") if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password) retVal = "0200%s%s" % (salt, sha512(unistr + binsalt).hexdigest()) @@ -231,7 +231,7 @@ def oracle_old_passwd(password, username, uppercase=True): # prior to version ' IV, pad = "\0" * 8, "\0" - unistr = b"".join(b"\0%s" % _.encode(UNICODE_ENCODING) if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in (username + password).upper()) + unistr = b"".join((b"\0" + _.encode(UNICODE_ENCODING)) if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in (username + password).upper()) cipher = des(decodeHex("0123456789ABCDEF"), CBC, IV, pad) encrypted = cipher.encrypt(unistr) @@ -434,7 +434,7 @@ def unix_md5_passwd(password, salt, magic="$1$", **kwargs): hash_ = hash_ + _encode64((int(ord(final[4:5])) << 16) | (int(ord(final[10:11])) << 8) | (int(ord(final[5:6]))), 4) hash_ = hash_ + _encode64((int(ord(final[11:12]))), 2) - return getText(b"%s%s$%s" % (magic, salt, getBytes(hash_))) + return getText(magic + salt + b'$' + getBytes(hash_)) def joomla_passwd(password, salt, **kwargs): """ @@ -444,7 +444,7 @@ def joomla_passwd(password, salt, **kwargs): 'e3d5794da74e917637332e0d21b76328:6GGlnaquVXI80b3HRmSyE3K1wEFFaBIf' """ - return "%s:%s" % (md5(b"%s%s" % (getBytes(password), getBytes(salt))).hexdigest(), salt) + return "%s:%s" % (md5(getBytes(password) + getBytes(salt)).hexdigest(), salt) def django_md5_passwd(password, salt, **kwargs): """ @@ -454,7 +454,7 @@ def django_md5_passwd(password, salt, **kwargs): 'md5$salt$972141bcbcb6a0acc96e92309175b3c5' """ - return "md5$%s$%s" % (salt, md5(b"%s%s" % (getBytes(salt), getBytes(password))).hexdigest()) + return "md5$%s$%s" % (salt, md5(getBytes(salt) + getBytes(password)).hexdigest()) def django_sha1_passwd(password, salt, **kwargs): """ @@ -464,7 +464,7 @@ def django_sha1_passwd(password, salt, **kwargs): 'sha1$salt$6ce0e522aba69d8baa873f01420fccd0250fc5b2' """ - return "sha1$%s$%s" % (salt, sha1(b"%s%s" % (getBytes(salt), getBytes(password))).hexdigest()) + return "sha1$%s$%s" % (salt, sha1(getBytes(salt) + getBytes(password)).hexdigest()) def vbulletin_passwd(password, salt, **kwargs): """ @@ -474,7 +474,7 @@ def vbulletin_passwd(password, salt, **kwargs): '85c4d8ea77ebef2236fb7e9d24ba9482:salt' """ - return "%s:%s" % (md5(b"%s%s" % (binascii.hexlify(md5(getBytes(password)).digest()), getBytes(salt))).hexdigest(), salt) + return "%s:%s" % (md5(binascii.hexlify(md5(getBytes(password)).digest()) + getBytes(salt)).hexdigest(), salt) def wordpress_passwd(password, salt, count, prefix, **kwargs): """ diff --git a/thirdparty/beautifulsoup/beautifulsoup.py b/thirdparty/beautifulsoup/beautifulsoup.py index c02e1c72e..4d7e86be3 100644 --- a/thirdparty/beautifulsoup/beautifulsoup.py +++ b/thirdparty/beautifulsoup/beautifulsoup.py @@ -93,6 +93,7 @@ if sys.version_info >= (3, 0): xrange = range text_type = str binary_type = bytes + basestring = str else: text_type = unicode binary_type = str @@ -673,7 +674,7 @@ class Tag(PageElement): """Calling a tag like a function is the same as calling its findAll() method. Eg. tag('a') returns a list of all the A tags found within this tag.""" - return apply(self.findAll, args, kwargs) + return self.findAll(*args, **kwargs) def __getattr__(self, tag): #print "Getattr %s.%s" % (self.__class__, tag) @@ -1332,7 +1333,7 @@ class BeautifulStoneSoup(Tag, sgmllib.SGMLParser): if (nestingResetTriggers is not None and p.name in nestingResetTriggers) \ or (nestingResetTriggers is None and isResetNesting - and self.RESET_NESTING_TAGS.has_key(p.name)): + and p.name in self.RESET_NESTING_TAGS): #If we encounter one of the nesting reset triggers #peculiar to this tag, or we encounter another tag diff --git a/thirdparty/fcrypt/fcrypt.py b/thirdparty/fcrypt/fcrypt.py index eb49d023b..2f664d81a 100644 --- a/thirdparty/fcrypt/fcrypt.py +++ b/thirdparty/fcrypt/fcrypt.py @@ -455,7 +455,7 @@ def _PERM_OP(a,b,n,m): def _set_key(password): """Generate DES key schedule from ASCII password.""" - c,d = struct.unpack('