From 22484c8599adc489c8a4e9656380e1d58ed759c6 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 9 Nov 2015 13:19:55 +0100 Subject: [PATCH 01/38] Bug fix (-p Host didn't work, while -p host worked) --- lib/core/target.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/core/target.py b/lib/core/target.py index 98101d433..6d5dd9622 100644 --- a/lib/core/target.py +++ b/lib/core/target.py @@ -327,7 +327,7 @@ def _setRequestParams(): if httpHeader == HTTP_HEADER.USER_AGENT: conf.parameters[PLACE.USER_AGENT] = urldecode(headerValue) - condition = any((not conf.testParameter, intersect(conf.testParameter, USER_AGENT_ALIASES))) + condition = any((not conf.testParameter, intersect(conf.testParameter, USER_AGENT_ALIASES, True))) if condition: conf.paramDict[PLACE.USER_AGENT] = {PLACE.USER_AGENT: headerValue} @@ -336,7 +336,7 @@ def _setRequestParams(): elif httpHeader == HTTP_HEADER.REFERER: conf.parameters[PLACE.REFERER] = urldecode(headerValue) - condition = any((not conf.testParameter, intersect(conf.testParameter, REFERER_ALIASES))) + condition = any((not conf.testParameter, intersect(conf.testParameter, REFERER_ALIASES, True))) if condition: conf.paramDict[PLACE.REFERER] = {PLACE.REFERER: headerValue} @@ -345,7 +345,7 @@ def _setRequestParams(): elif httpHeader == HTTP_HEADER.HOST: conf.parameters[PLACE.HOST] = urldecode(headerValue) - condition = any((not conf.testParameter, intersect(conf.testParameter, HOST_ALIASES))) + condition = any((not conf.testParameter, intersect(conf.testParameter, HOST_ALIASES, True))) if condition: conf.paramDict[PLACE.HOST] = {PLACE.HOST: headerValue} From 17350fb4ec2d9ee2b6852f3f4d5365a12890460b Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 9 Nov 2015 14:05:53 +0100 Subject: [PATCH 02/38] Proper fix for #1146 (/ has been escaped with \/ in output) --- lib/core/target.py | 19 +++++++++++++------ lib/request/connect.py | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/lib/core/target.py b/lib/core/target.py index 6d5dd9622..64aeef12f 100644 --- a/lib/core/target.py +++ b/lib/core/target.py @@ -318,13 +318,11 @@ def _setRequestParams(): # Perform checks on header values if conf.httpHeaders: - for httpHeader, headerValue in conf.httpHeaders: + for httpHeader, headerValue in list(conf.httpHeaders): # Url encoding of the header values should be avoided # Reference: http://stackoverflow.com/questions/5085904/is-ok-to-urlencode-the-value-in-headerlocation-value - httpHeader = httpHeader.title() - - if httpHeader == HTTP_HEADER.USER_AGENT: + if httpHeader.title() == HTTP_HEADER.USER_AGENT: conf.parameters[PLACE.USER_AGENT] = urldecode(headerValue) condition = any((not conf.testParameter, intersect(conf.testParameter, USER_AGENT_ALIASES, True))) @@ -333,7 +331,7 @@ def _setRequestParams(): conf.paramDict[PLACE.USER_AGENT] = {PLACE.USER_AGENT: headerValue} testableParameters = True - elif httpHeader == HTTP_HEADER.REFERER: + elif httpHeader.title() == HTTP_HEADER.REFERER: conf.parameters[PLACE.REFERER] = urldecode(headerValue) condition = any((not conf.testParameter, intersect(conf.testParameter, REFERER_ALIASES, True))) @@ -342,7 +340,7 @@ def _setRequestParams(): conf.paramDict[PLACE.REFERER] = {PLACE.REFERER: headerValue} testableParameters = True - elif httpHeader == HTTP_HEADER.HOST: + elif httpHeader.title() == HTTP_HEADER.HOST: conf.parameters[PLACE.HOST] = urldecode(headerValue) condition = any((not conf.testParameter, intersect(conf.testParameter, HOST_ALIASES, True))) @@ -351,6 +349,15 @@ def _setRequestParams(): conf.paramDict[PLACE.HOST] = {PLACE.HOST: headerValue} testableParameters = True + else: + condition = intersect(conf.testParameter, [httpHeader], True) + + if condition: + conf.parameters[PLACE.CUSTOM_HEADER] = str(conf.httpHeaders) + conf.paramDict[PLACE.CUSTOM_HEADER] = {httpHeader: "%s,%s%s" % (httpHeader, headerValue, CUSTOM_INJECTION_MARK_CHAR)} + conf.httpHeaders = [(header, value.replace(CUSTOM_INJECTION_MARK_CHAR, "")) for header, value in conf.httpHeaders] + testableParameters = True + if not conf.parameters: errMsg = "you did not provide any GET, POST and Cookie " errMsg += "parameter, neither an User-Agent, Referer or Host header value" diff --git a/lib/request/connect.py b/lib/request/connect.py index 732af5cd2..6139eac15 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -659,7 +659,7 @@ class Connect(object): if conn and getattr(conn, "redurl", None): _ = urlparse.urlsplit(conn.redurl) _ = ("%s%s" % (_.path or "/", ("?%s" % _.query) if _.query else "")) - requestMsg = re.sub("(\n[A-Z]+ ).+?( HTTP/\d)", "\g<1>%s\g<2>" % re.escape(getUnicode(_)), requestMsg, 1) + requestMsg = re.sub("(\n[A-Z]+ ).+?( HTTP/\d)", "\g<1>%s\g<2>" % getUnicode(_).replace("\\", "\\\\"), requestMsg, 1) if kb.resendPostOnRedirect is False: requestMsg = re.sub("(\[#\d+\]:\n)POST ", "\g<1>GET ", requestMsg) From bc215d1b19a33a2b1400fb8bfb3d20e7c3b00b96 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 9 Nov 2015 14:11:08 +0100 Subject: [PATCH 03/38] I believe that this was a wrong decision. Patching --- lib/request/redirecthandler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/request/redirecthandler.py b/lib/request/redirecthandler.py index 3359f59e7..f75cffed5 100644 --- a/lib/request/redirecthandler.py +++ b/lib/request/redirecthandler.py @@ -38,9 +38,9 @@ class SmartRedirectHandler(urllib2.HTTPRedirectHandler): if headers: if "location" in headers: - retVal = headers.getheaders("location")[0].split("?")[0] + retVal = headers.getheaders("location")[0] elif "uri" in headers: - retVal = headers.getheaders("uri")[0].split("?")[0] + retVal = headers.getheaders("uri")[0] return retVal From 07b14073457aee7538c44457d0d06f5f9431d5f9 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 11 Nov 2015 15:55:28 +0100 Subject: [PATCH 04/38] Patches #1530 --- lib/core/shell.py | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/lib/core/shell.py b/lib/core/shell.py index 7b53bc389..74ae4e452 100644 --- a/lib/core/shell.py +++ b/lib/core/shell.py @@ -7,7 +7,6 @@ See the file 'doc/COPYING' for copying permission import atexit import os -import rlcompleter from lib.core import readlineng as readline from lib.core.data import logger @@ -16,6 +15,29 @@ from lib.core.enums import AUTOCOMPLETE_TYPE from lib.core.enums import OS from lib.core.settings import MAX_HISTORY_LENGTH +try: + import rlcompleter + + class CompleterNG(rlcompleter.Completer): + def global_matches(self, text): + """ + Compute matches when text is a simple name. + Return a list of all names currently defined in self.namespace + that match. + """ + + matches = [] + n = len(text) + + for ns in (self.namespace,): + for word in ns: + if word[:n] == text: + matches.append(word) + + return matches +except: + readline._readline = None + def readlineAvailable(): """ Check if the readline is available. By default @@ -74,24 +96,6 @@ def loadHistory(completion=None): warnMsg = "there was a problem loading the history file '%s' (%s)" % (historyPath, msg) logger.warn(warnMsg) -class CompleterNG(rlcompleter.Completer): - def global_matches(self, text): - """ - Compute matches when text is a simple name. - Return a list of all names currently defined in self.namespace - that match. - """ - - matches = [] - n = len(text) - - for ns in (self.namespace,): - for word in ns: - if word[:n] == text: - matches.append(word) - - return matches - def autoCompletion(completion=None, os=None, commands=None): if not readlineAvailable(): return From d772e7e1d5717967ebe4f7bc0a602d4a10a43ae6 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 11 Nov 2015 16:07:11 +0100 Subject: [PATCH 05/38] Fixes #1529 --- lib/techniques/union/test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/techniques/union/test.py b/lib/techniques/union/test.py index c9e90ce77..fd61975fa 100644 --- a/lib/techniques/union/test.py +++ b/lib/techniques/union/test.py @@ -120,8 +120,10 @@ def _findUnionCharCount(comment, place, parameter, value, prefix, suffix, where= break if not retVal: - ratios.pop(ratios.index(min_)) - ratios.pop(ratios.index(max_)) + if min_ in ratios: + ratios.pop(ratios.index(min_)) + if max_ in ratios: + ratios.pop(ratios.index(max_)) minItem, maxItem = None, None From 8d1e1ea474f5b469e86525cee8ab0e98dcc3d1be Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Thu, 12 Nov 2015 16:58:34 +0100 Subject: [PATCH 06/38] Patch for an Issue #1532 --- sqlmap.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sqlmap.py b/sqlmap.py index 1fd6f47f7..4a0e1ff5a 100755 --- a/sqlmap.py +++ b/sqlmap.py @@ -147,6 +147,13 @@ def main(): logger.error(errMsg) raise SystemExit + elif "bad marshal data (unknown type code)" in excMsg: + match = re.search(r"\s*(.+)\s+ValueError", excMsg) + errMsg = "one of your .pyc files are corrupted%s" % (" ('%s')" % match.group(1) if match else "") + errMsg += ". Please delete .pyc files on your system to fix the problem" + logger.error(errMsg) + raise SystemExit + for match in re.finditer(r'File "(.+?)", line', excMsg): file_ = match.group(1) file_ = os.path.relpath(file_, os.path.dirname(__file__)) From ee7ea68c15e1aaa6108a4192b1892b11076ca39a Mon Sep 17 00:00:00 2001 From: loveshell <82163261@qq.com> Date: Sun, 15 Nov 2015 10:15:05 +0800 Subject: [PATCH 07/38] add Newdefend waf identified --- waf/Newdefend.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 waf/Newdefend.py diff --git a/waf/Newdefend.py b/waf/Newdefend.py new file mode 100644 index 000000000..5cadd8485 --- /dev/null +++ b/waf/Newdefend.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2015 sqlmap developers (http://sqlmap.org/) +See the file 'doc/COPYING' for copying permission +""" + +import re + +from lib.core.enums import HTTP_HEADER +from lib.core.settings import WAF_ATTACK_VECTORS + +__product__ = "Newdefend Web Application Firewall (Newdefend)" + +def detect(get_page): + retval = False + + for vector in WAF_ATTACK_VECTORS: + page, headers, code = get_page(get=vector) + retval = re.search(r"newdefend", headers.get(HTTP_HEADER.SERVER, ""), re.I) is not None + if retval: + break + + return retval From fb2cb25afe6892394a2b991e79b879f8ba558523 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 16 Nov 2015 11:56:06 +0100 Subject: [PATCH 08/38] Bug fix for an Issue #1539 --- lib/core/common.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/core/common.py b/lib/core/common.py index fd1b375fa..f4b4f7b04 100755 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -2943,10 +2943,16 @@ def decodeIntToUnicode(value): if isinstance(value, int): try: if value > 255: + if Backend.isDbms(DBMS.MSSQL): + encoding="UTF-16-LE" if isDBMSVersionAtLeast("2012") else "UTF-16-BE" + else: + encoding = conf.charset + _ = "%x" % value if len(_) % 2 == 1: _ = "0%s" % _ - retVal = getUnicode(hexdecode(_), encoding="UTF-16" if Backend.isDbms(DBMS.MSSQL) else None) + + retVal = getUnicode(hexdecode(_), encoding) else: retVal = getUnicode(chr(value)) except: From 9c69f56a34fd6be7be1676772664439c851abd21 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 16 Nov 2015 11:59:09 +0100 Subject: [PATCH 09/38] Proper patch for an Issue #1539 --- lib/core/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/common.py b/lib/core/common.py index f4b4f7b04..0255cdc14 100755 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -2944,7 +2944,7 @@ def decodeIntToUnicode(value): try: if value > 255: if Backend.isDbms(DBMS.MSSQL): - encoding="UTF-16-LE" if isDBMSVersionAtLeast("2012") else "UTF-16-BE" + encoding = "UTF-16-BE" else: encoding = conf.charset From a212f0c240516175a43fe8e4b7d13f8325f0f5a0 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 16 Nov 2015 12:56:15 +0100 Subject: [PATCH 10/38] Another patch for #1539 --- lib/core/common.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/core/common.py b/lib/core/common.py index 0255cdc14..a15d7b768 100755 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -2943,16 +2943,17 @@ def decodeIntToUnicode(value): if isinstance(value, int): try: if value > 255: - if Backend.isDbms(DBMS.MSSQL): - encoding = "UTF-16-BE" - else: - encoding = conf.charset - _ = "%x" % value if len(_) % 2 == 1: _ = "0%s" % _ + raw = hexdecode(_) - retVal = getUnicode(hexdecode(_), encoding) + if Backend.isDbms(DBMS.MSSQL): + retVal = getUnicode(raw, "UTF-16-BE") + elif Backend.isDbms(DBMS.PGSQL): + retVal = unichr(value) + else: + retVal = getUnicode(raw, conf.charset) else: retVal = getUnicode(chr(value)) except: From ca933fcf1d5630a2d2d9f2c5a9474290c7e488c9 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 16 Nov 2015 14:08:43 +0100 Subject: [PATCH 11/38] Another patch for #1539 --- lib/core/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/common.py b/lib/core/common.py index a15d7b768..1f28b5f25 100755 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -2950,7 +2950,7 @@ def decodeIntToUnicode(value): if Backend.isDbms(DBMS.MSSQL): retVal = getUnicode(raw, "UTF-16-BE") - elif Backend.isDbms(DBMS.PGSQL): + elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.ORACLE): retVal = unichr(value) else: retVal = getUnicode(raw, conf.charset) From 5593bf2fee7c7c2a508d85cee6fd426897295d1a Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 16 Nov 2015 15:02:30 +0100 Subject: [PATCH 12/38] Another patch related to #1539 (simplifying unicode bad chars and preventing double encoding of safe chars) --- extra/safe2bin/safe2bin.py | 17 ++++++++--------- lib/core/common.py | 4 ++++ lib/core/settings.py | 2 +- xml/queries.xml | 2 +- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/extra/safe2bin/safe2bin.py b/extra/safe2bin/safe2bin.py index c91620ec6..ff3f7d8d8 100644 --- a/extra/safe2bin/safe2bin.py +++ b/extra/safe2bin/safe2bin.py @@ -19,15 +19,18 @@ from optparse import OptionParser # Regex used for recognition of hex encoded characters HEX_ENCODED_CHAR_REGEX = r"(?P\\x[0-9A-Fa-f]{2})" -# Regex used for recognition of representation for hex encoded invalid unicode characters -INVALID_UNICODE_CHAR_REGEX = r"(?P\\\?[0-9A-Fa-f]{2})" - # Raw chars that will be safe encoded to their slash (\) representations (e.g. newline to \n) SAFE_ENCODE_SLASH_REPLACEMENTS = "\t\n\r\x0b\x0c" # Characters that don't need to be safe encoded SAFE_CHARS = "".join(filter(lambda x: x not in SAFE_ENCODE_SLASH_REPLACEMENTS, string.printable.replace('\\', ''))) +# Prefix used for hex encoded values +HEX_ENCODED_PREFIX = r"\x" + +# Strings used for temporary marking of hex encoded prefixes (to prevent double encoding) +HEX_ENCODED_PREFIX_MARKER = "__HEX_ENCODED_PREFIX__" + # String used for temporary marking of slash characters SLASH_MARKER = "__SLASH__" @@ -45,6 +48,7 @@ def safecharencode(value): if isinstance(value, basestring): if any(_ not in SAFE_CHARS for _ in value): + retVal = retVal.replace(HEX_ENCODED_PREFIX, HEX_ENCODED_PREFIX_MARKER) retVal = retVal.replace('\\', SLASH_MARKER) for char in SAFE_ENCODE_SLASH_REPLACEMENTS: @@ -53,6 +57,7 @@ def safecharencode(value): retVal = reduce(lambda x, y: x + (y if (y in string.printable or isinstance(value, unicode) and ord(y) >= 160) else '\\x%02x' % ord(y)), retVal, (unicode if isinstance(value, unicode) else str)()) retVal = retVal.replace(SLASH_MARKER, "\\\\") + retVal = retVal.replace(HEX_ENCODED_PREFIX_MARKER, HEX_ENCODED_PREFIX) elif isinstance(value, list): for i in xrange(len(value)): retVal[i] = safecharencode(value[i]) @@ -83,12 +88,6 @@ def safechardecode(value, binary=False): if binary: if isinstance(retVal, unicode): retVal = retVal.encode("utf8") - while True: - match = re.search(INVALID_UNICODE_CHAR_REGEX, retVal) - if match: - retVal = retVal.replace(match.group("result"), chr(ord(binascii.unhexlify(match.group("result").lstrip("\\?"))))) - else: - break elif isinstance(value, (list, tuple)): for i in xrange(len(value)): diff --git a/lib/core/common.py b/lib/core/common.py index 1f28b5f25..a7ddaef35 100755 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -2954,6 +2954,10 @@ def decodeIntToUnicode(value): retVal = unichr(value) else: retVal = getUnicode(raw, conf.charset) + + if Backend.isDbms(DBMS.MYSQL): + import pdb + pdb.set_trace() else: retVal = getUnicode(chr(value)) except: diff --git a/lib/core/settings.py b/lib/core/settings.py index 5e9b6f1a1..d478ae50a 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -587,7 +587,7 @@ EVENTVALIDATION_REGEX = r'(?i)(?P__EVENTVALIDATION[^"]*)[^>]+value="(?P]+>(.+>)?\s*\Z" diff --git a/xml/queries.xml b/xml/queries.xml index 98b79cac7..28fd85bfb 100644 --- a/xml/queries.xml +++ b/xml/queries.xml @@ -3,7 +3,7 @@ - + From 768e5da589da058d4b30d5c41693c941b1d54acd Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 16 Nov 2015 15:04:09 +0100 Subject: [PATCH 13/38] Removing leftover (from 5593bf2fee7c7c2a508d85cee6fd426897295d1a) --- lib/core/common.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/core/common.py b/lib/core/common.py index a7ddaef35..1f28b5f25 100755 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -2954,10 +2954,6 @@ def decodeIntToUnicode(value): retVal = unichr(value) else: retVal = getUnicode(raw, conf.charset) - - if Backend.isDbms(DBMS.MYSQL): - import pdb - pdb.set_trace() else: retVal = getUnicode(chr(value)) except: From c1e3431877d263f2646a886aba8c5a72714ca689 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 16 Nov 2015 15:32:28 +0100 Subject: [PATCH 14/38] Minor patch --- lib/core/enums.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/enums.py b/lib/core/enums.py index 1ba4b8185..785daf95e 100644 --- a/lib/core/enums.py +++ b/lib/core/enums.py @@ -81,7 +81,7 @@ class HTTPMETHOD: POST = "POST" HEAD = "HEAD" PUT = "PUT" - DELETE = "DETELE" + DELETE = "DELETE" TRACE = "TRACE" OPTIONS = "OPTIONS" CONNECT = "CONNECT" From 94639d11a3409f5f102bc39c2a8c9004352a6079 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 16 Nov 2015 15:33:05 +0100 Subject: [PATCH 15/38] Another update related to the #1539 --- lib/core/agent.py | 6 +++++- lib/core/settings.py | 4 ++++ lib/techniques/blind/inference.py | 7 +++++++ plugins/dbms/mysql/syntax.py | 3 ++- plugins/generic/enumeration.py | 1 + xml/queries.xml | 2 +- 6 files changed, 20 insertions(+), 3 deletions(-) diff --git a/lib/core/agent.py b/lib/core/agent.py index 68fc22817..1604f83cd 100644 --- a/lib/core/agent.py +++ b/lib/core/agent.py @@ -37,6 +37,7 @@ from lib.core.settings import BOUNDARY_BACKSLASH_MARKER from lib.core.settings import CUSTOM_INJECTION_MARK_CHAR from lib.core.settings import DEFAULT_COOKIE_DELIMITER from lib.core.settings import DEFAULT_GET_POST_DELIMITER +from lib.core.settings import DEFAULT_MYSQL_CHARACTER_SET from lib.core.settings import GENERIC_SQL_COMMENT from lib.core.settings import PAYLOAD_DELIMITER from lib.core.settings import REPLACEMENT_MARKER @@ -400,7 +401,10 @@ class Agent(object): nulledCastedField = field else: if not (Backend.isDbms(DBMS.SQLITE) and not isDBMSVersionAtLeast('3')): - nulledCastedField = rootQuery.cast.query % field + if Backend.isDbms(DBMS.MYSQL): + nulledCastedField = rootQuery.cast.query.replace(")", " CHARACTER SET %s)") % (field, DEFAULT_MYSQL_CHARACTER_SET) + else: + nulledCastedField = rootQuery.cast.query % field if Backend.getIdentifiedDbms() in (DBMS.ACCESS,): nulledCastedField = rootQuery.isnull.query % (nulledCastedField, nulledCastedField) else: diff --git a/lib/core/settings.py b/lib/core/settings.py index d478ae50a..1e14fd755 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -224,6 +224,10 @@ HOST_ALIASES = ("host",) HSQLDB_DEFAULT_SCHEMA = "PUBLIC" +# Default character set used in MySQL +# Reference: http://pieroxy.net/blog/2013/05/28/mysql_charset_hell.html +DEFAULT_MYSQL_CHARACTER_SET = "latin1" + # Names that can't be used to name files on Windows OS WINDOWS_RESERVED_NAMES = ("CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9") diff --git a/lib/techniques/blind/inference.py b/lib/techniques/blind/inference.py index 11e78ed6c..39fad7e77 100644 --- a/lib/techniques/blind/inference.py +++ b/lib/techniques/blind/inference.py @@ -8,6 +8,7 @@ See the file 'doc/COPYING' for copying permission import threading import time +from extra.safe2bin.safe2bin import safechardecode from extra.safe2bin.safe2bin import safecharencode from lib.core.agent import agent from lib.core.common import Backend @@ -18,6 +19,7 @@ from lib.core.common import decodeIntToUnicode from lib.core.common import filterControlChars from lib.core.common import getCharset from lib.core.common import getCounter +from lib.core.common import getUnicode from lib.core.common import goGoodSamaritan from lib.core.common import getPartRun from lib.core.common import hashDBRetrieve @@ -35,6 +37,7 @@ from lib.core.enums import DBMS from lib.core.enums import PAYLOAD from lib.core.exception import SqlmapThreadException from lib.core.settings import CHAR_INFERENCE_MARK +from lib.core.settings import DEFAULT_MYSQL_CHARACTER_SET from lib.core.settings import INFERENCE_BLANK_BREAK from lib.core.settings import INFERENCE_UNKNOWN_CHAR from lib.core.settings import INFERENCE_GREATER_CHAR @@ -589,6 +592,10 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None raise KeyboardInterrupt _ = finalValue or partialValue + + if Backend.isDbms(DBMS.MYSQL) and safechardecode(_) != _: + _ = getUnicode(safechardecode(_).encode(DEFAULT_MYSQL_CHARACTER_SET)) + return getCounter(kb.technique), safecharencode(_) if kb.safeCharEncode else _ def queryOutputLength(expression, payload): diff --git a/plugins/dbms/mysql/syntax.py b/plugins/dbms/mysql/syntax.py index e593a51fb..312141b55 100644 --- a/plugins/dbms/mysql/syntax.py +++ b/plugins/dbms/mysql/syntax.py @@ -8,6 +8,7 @@ See the file 'doc/COPYING' for copying permission import binascii from lib.core.convert import utf8encode +from lib.core.settings import DEFAULT_MYSQL_CHARACTER_SET from plugins.generic.syntax import Syntax as GenericSyntax class Syntax(GenericSyntax): @@ -26,7 +27,7 @@ class Syntax(GenericSyntax): try: retVal = "0x%s" % binascii.hexlify(value) except UnicodeEncodeError: - retVal = "CONVERT(0x%s USING utf8)" % "".join("%.2x" % ord(_) for _ in utf8encode(value)) + retVal = "CONVERT(0x%s USING %s)" % ("".join("%.2x" % ord(_) for _ in utf8encode(value)), DEFAULT_MYSQL_CHARACTER_SET) return retVal return Syntax._escape(expression, quote, escaper) diff --git a/plugins/generic/enumeration.py b/plugins/generic/enumeration.py index 23826f7e7..4332ea4c4 100644 --- a/plugins/generic/enumeration.py +++ b/plugins/generic/enumeration.py @@ -31,6 +31,7 @@ class Enumeration(Custom, Databases, Entries, Search, Users): kb.data.banner = None kb.data.hostname = "" kb.data.processChar = None + kb.data.characterSet = None Custom.__init__(self) Databases.__init__(self) diff --git a/xml/queries.xml b/xml/queries.xml index 28fd85bfb..98b79cac7 100644 --- a/xml/queries.xml +++ b/xml/queries.xml @@ -3,7 +3,7 @@ - + From 4335ae8330a81d57cef0613fa9542b1a7d108f83 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 16 Nov 2015 16:59:54 +0100 Subject: [PATCH 16/38] Patching previous commit --- lib/core/agent.py | 6 +----- lib/core/settings.py | 4 ---- lib/techniques/blind/inference.py | 4 ---- plugins/dbms/mysql/syntax.py | 3 +-- 4 files changed, 2 insertions(+), 15 deletions(-) diff --git a/lib/core/agent.py b/lib/core/agent.py index 1604f83cd..68fc22817 100644 --- a/lib/core/agent.py +++ b/lib/core/agent.py @@ -37,7 +37,6 @@ from lib.core.settings import BOUNDARY_BACKSLASH_MARKER from lib.core.settings import CUSTOM_INJECTION_MARK_CHAR from lib.core.settings import DEFAULT_COOKIE_DELIMITER from lib.core.settings import DEFAULT_GET_POST_DELIMITER -from lib.core.settings import DEFAULT_MYSQL_CHARACTER_SET from lib.core.settings import GENERIC_SQL_COMMENT from lib.core.settings import PAYLOAD_DELIMITER from lib.core.settings import REPLACEMENT_MARKER @@ -401,10 +400,7 @@ class Agent(object): nulledCastedField = field else: if not (Backend.isDbms(DBMS.SQLITE) and not isDBMSVersionAtLeast('3')): - if Backend.isDbms(DBMS.MYSQL): - nulledCastedField = rootQuery.cast.query.replace(")", " CHARACTER SET %s)") % (field, DEFAULT_MYSQL_CHARACTER_SET) - else: - nulledCastedField = rootQuery.cast.query % field + nulledCastedField = rootQuery.cast.query % field if Backend.getIdentifiedDbms() in (DBMS.ACCESS,): nulledCastedField = rootQuery.isnull.query % (nulledCastedField, nulledCastedField) else: diff --git a/lib/core/settings.py b/lib/core/settings.py index 1e14fd755..d478ae50a 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -224,10 +224,6 @@ HOST_ALIASES = ("host",) HSQLDB_DEFAULT_SCHEMA = "PUBLIC" -# Default character set used in MySQL -# Reference: http://pieroxy.net/blog/2013/05/28/mysql_charset_hell.html -DEFAULT_MYSQL_CHARACTER_SET = "latin1" - # Names that can't be used to name files on Windows OS WINDOWS_RESERVED_NAMES = ("CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9") diff --git a/lib/techniques/blind/inference.py b/lib/techniques/blind/inference.py index 39fad7e77..db6df84a1 100644 --- a/lib/techniques/blind/inference.py +++ b/lib/techniques/blind/inference.py @@ -37,7 +37,6 @@ from lib.core.enums import DBMS from lib.core.enums import PAYLOAD from lib.core.exception import SqlmapThreadException from lib.core.settings import CHAR_INFERENCE_MARK -from lib.core.settings import DEFAULT_MYSQL_CHARACTER_SET from lib.core.settings import INFERENCE_BLANK_BREAK from lib.core.settings import INFERENCE_UNKNOWN_CHAR from lib.core.settings import INFERENCE_GREATER_CHAR @@ -593,9 +592,6 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None _ = finalValue or partialValue - if Backend.isDbms(DBMS.MYSQL) and safechardecode(_) != _: - _ = getUnicode(safechardecode(_).encode(DEFAULT_MYSQL_CHARACTER_SET)) - return getCounter(kb.technique), safecharencode(_) if kb.safeCharEncode else _ def queryOutputLength(expression, payload): diff --git a/plugins/dbms/mysql/syntax.py b/plugins/dbms/mysql/syntax.py index 312141b55..e593a51fb 100644 --- a/plugins/dbms/mysql/syntax.py +++ b/plugins/dbms/mysql/syntax.py @@ -8,7 +8,6 @@ See the file 'doc/COPYING' for copying permission import binascii from lib.core.convert import utf8encode -from lib.core.settings import DEFAULT_MYSQL_CHARACTER_SET from plugins.generic.syntax import Syntax as GenericSyntax class Syntax(GenericSyntax): @@ -27,7 +26,7 @@ class Syntax(GenericSyntax): try: retVal = "0x%s" % binascii.hexlify(value) except UnicodeEncodeError: - retVal = "CONVERT(0x%s USING %s)" % ("".join("%.2x" % ord(_) for _ in utf8encode(value)), DEFAULT_MYSQL_CHARACTER_SET) + retVal = "CONVERT(0x%s USING utf8)" % "".join("%.2x" % ord(_) for _ in utf8encode(value)) return retVal return Syntax._escape(expression, quote, escaper) From 41b8dfab86c41b1bdcb0ebfb1289e69c4587a9b5 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 16 Nov 2015 23:46:10 +0100 Subject: [PATCH 17/38] Implementation for an Issue #1540 --- lib/core/option.py | 40 ++++++++++++++++++++++++++++++++++++++-- lib/core/settings.py | 3 +++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/lib/core/option.py b/lib/core/option.py index 40bcf122a..c253427e6 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -122,6 +122,7 @@ from lib.core.settings import NULL from lib.core.settings import PARAMETER_SPLITTING_REGEX from lib.core.settings import PROBLEMATIC_CUSTOM_INJECTION_PATTERNS from lib.core.settings import SITE +from lib.core.settings import SOCKET_PRE_CONNECT_QUEUE_SIZE from lib.core.settings import SQLMAP_ENVIRONMENT_PREFIX from lib.core.settings import SUPPORTED_DBMS from lib.core.settings import SUPPORTED_OS @@ -1014,10 +1015,44 @@ def _setDNSCache(): kb.cache[args] = socket._getaddrinfo(*args, **kwargs) return kb.cache[args] - if not hasattr(socket, '_getaddrinfo'): + if not hasattr(socket, "_getaddrinfo"): socket._getaddrinfo = socket.getaddrinfo socket.getaddrinfo = _getaddrinfo +def _setSocketPreConnect(): + """ + Makes a pre-connect version of socket.connect + """ + + def _(): + while kb.threadContinue: + for address in socket._ready: + if len(socket._ready[address]) < SOCKET_PRE_CONNECT_QUEUE_SIZE: + s = socket.socket() + s._connect(address) + socket._ready[address].append(s._sock) + time.sleep(0.001) + + def connect(self, address): + found = False + with kb.locks.socket: + if address not in socket._ready: + socket._ready[address] = [] + if len(socket._ready[address]) > 0: + self._sock = socket._ready[address].pop(0) + found = True + if not found: + self._connect(address) + + if not hasattr(socket, "_connect"): + socket._ready = {} + socket.socket._connect = socket.socket.connect + socket.socket.connect = connect + + thread = threading.Thread(target=_) + thread.daemon = True + thread.start() + def _setHTTPHandlers(): """ Check and set the HTTP/SOCKS proxy for all HTTP requests. @@ -1803,7 +1838,7 @@ def _setKnowledgeBaseAttributes(flushAll=True): kb.lastParserStatus = None kb.locks = AttribDict() - for _ in ("cache", "count", "index", "io", "limit", "log", "redirect", "request", "value"): + for _ in ("cache", "count", "index", "io", "limit", "log", "socket", "redirect", "request", "value"): kb.locks[_] = threading.Lock() kb.matchRatio = None @@ -2517,6 +2552,7 @@ def init(): _setHTTPAuthentication() _setHTTPHandlers() _setDNSCache() + _setSocketPreConnect() _setSafeVisit() _doSearch() _setBulkMultipleTargets() diff --git a/lib/core/settings.py b/lib/core/settings.py index d478ae50a..72d7a906f 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -466,6 +466,9 @@ ROTATING_CHARS = ('\\', '|', '|', '/', '-') # Approximate chunk length (in bytes) used by BigArray objects (only last chunk and cached one are held in memory) BIGARRAY_CHUNK_SIZE = 1024 * 1024 +# Maximum number of socket pre-connects +SOCKET_PRE_CONNECT_QUEUE_SIZE = 3 + # Only console display last n table rows TRIM_STDOUT_DUMP_SIZE = 256 From abb1c6a6210ea69654e894a915ff5029022cc183 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 17 Nov 2015 00:12:04 +0100 Subject: [PATCH 18/38] Less intensive loop --- lib/core/option.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/option.py b/lib/core/option.py index c253427e6..9afeed26f 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -1031,7 +1031,7 @@ def _setSocketPreConnect(): s = socket.socket() s._connect(address) socket._ready[address].append(s._sock) - time.sleep(0.001) + time.sleep(0.01) def connect(self, address): found = False From 89abeb02446ee3fbd7969ea2e28edf844eb66f14 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 17 Nov 2015 01:09:57 +0100 Subject: [PATCH 19/38] Patch for 'Exception in thread Thread-1 (most likely raised during interpreter shutdown)' --- lib/core/option.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/core/option.py b/lib/core/option.py index 9afeed26f..b4e374acb 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -1050,7 +1050,6 @@ def _setSocketPreConnect(): socket.socket.connect = connect thread = threading.Thread(target=_) - thread.daemon = True thread.start() def _setHTTPHandlers(): From 5be0a83e947bb30033016d5a91fb7034de8eb281 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 17 Nov 2015 01:38:43 +0100 Subject: [PATCH 20/38] Minor patch --- lib/core/option.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/core/option.py b/lib/core/option.py index b4e374acb..28ea9c4cb 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -1026,12 +1026,16 @@ def _setSocketPreConnect(): def _(): while kb.threadContinue: - for address in socket._ready: - if len(socket._ready[address]) < SOCKET_PRE_CONNECT_QUEUE_SIZE: - s = socket.socket() - s._connect(address) - socket._ready[address].append(s._sock) - time.sleep(0.01) + try: + for address in socket._ready: + if len(socket._ready[address]) < SOCKET_PRE_CONNECT_QUEUE_SIZE: + s = socket.socket() + s._connect(address) + socket._ready[address].append(s._sock) + except socket.error: + pass + finally: + time.sleep(0.01) def connect(self, address): found = False From fd2908336ae6970c23ff36d9968cfff52ea97a25 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 17 Nov 2015 02:35:53 +0100 Subject: [PATCH 21/38] Minor just in case patch --- lib/core/option.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/core/option.py b/lib/core/option.py index 28ea9c4cb..b666c79a6 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -1031,7 +1031,8 @@ def _setSocketPreConnect(): if len(socket._ready[address]) < SOCKET_PRE_CONNECT_QUEUE_SIZE: s = socket.socket() s._connect(address) - socket._ready[address].append(s._sock) + with kb.locks.socket: + socket._ready[address].append(s._sock) except socket.error: pass finally: From 58e049a60d250b881af60091215c75daa3f5c01a Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 17 Nov 2015 02:45:27 +0100 Subject: [PATCH 22/38] More generic approach for number of pre-open sockets (Issue #1540) --- lib/core/option.py | 3 +-- lib/core/settings.py | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/core/option.py b/lib/core/option.py index b666c79a6..d8a9a467d 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -122,7 +122,6 @@ from lib.core.settings import NULL from lib.core.settings import PARAMETER_SPLITTING_REGEX from lib.core.settings import PROBLEMATIC_CUSTOM_INJECTION_PATTERNS from lib.core.settings import SITE -from lib.core.settings import SOCKET_PRE_CONNECT_QUEUE_SIZE from lib.core.settings import SQLMAP_ENVIRONMENT_PREFIX from lib.core.settings import SUPPORTED_DBMS from lib.core.settings import SUPPORTED_OS @@ -1028,7 +1027,7 @@ def _setSocketPreConnect(): while kb.threadContinue: try: for address in socket._ready: - if len(socket._ready[address]) < SOCKET_PRE_CONNECT_QUEUE_SIZE: + if len(socket._ready[address]) < conf.threads: s = socket.socket() s._connect(address) with kb.locks.socket: diff --git a/lib/core/settings.py b/lib/core/settings.py index 72d7a906f..d478ae50a 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -466,9 +466,6 @@ ROTATING_CHARS = ('\\', '|', '|', '/', '-') # Approximate chunk length (in bytes) used by BigArray objects (only last chunk and cached one are held in memory) BIGARRAY_CHUNK_SIZE = 1024 * 1024 -# Maximum number of socket pre-connects -SOCKET_PRE_CONNECT_QUEUE_SIZE = 3 - # Only console display last n table rows TRIM_STDOUT_DUMP_SIZE = 256 From 19f6eb234b7b6fd30e6d62efeff30df7799f3f38 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 17 Nov 2015 08:52:24 +0100 Subject: [PATCH 23/38] Revert of #58e049a60d250b881af60091215c75daa3f5c01a (I can imagine couple of things that could go wrong) --- lib/core/option.py | 3 ++- lib/core/settings.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/core/option.py b/lib/core/option.py index d8a9a467d..b666c79a6 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -122,6 +122,7 @@ from lib.core.settings import NULL from lib.core.settings import PARAMETER_SPLITTING_REGEX from lib.core.settings import PROBLEMATIC_CUSTOM_INJECTION_PATTERNS from lib.core.settings import SITE +from lib.core.settings import SOCKET_PRE_CONNECT_QUEUE_SIZE from lib.core.settings import SQLMAP_ENVIRONMENT_PREFIX from lib.core.settings import SUPPORTED_DBMS from lib.core.settings import SUPPORTED_OS @@ -1027,7 +1028,7 @@ def _setSocketPreConnect(): while kb.threadContinue: try: for address in socket._ready: - if len(socket._ready[address]) < conf.threads: + if len(socket._ready[address]) < SOCKET_PRE_CONNECT_QUEUE_SIZE: s = socket.socket() s._connect(address) with kb.locks.socket: diff --git a/lib/core/settings.py b/lib/core/settings.py index d478ae50a..72d7a906f 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -466,6 +466,9 @@ ROTATING_CHARS = ('\\', '|', '|', '/', '-') # Approximate chunk length (in bytes) used by BigArray objects (only last chunk and cached one are held in memory) BIGARRAY_CHUNK_SIZE = 1024 * 1024 +# Maximum number of socket pre-connects +SOCKET_PRE_CONNECT_QUEUE_SIZE = 3 + # Only console display last n table rows TRIM_STDOUT_DUMP_SIZE = 256 From 39a7b787375ed625ee987143c26c2128f53e90a3 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 17 Nov 2015 09:39:09 +0100 Subject: [PATCH 24/38] Minor fix --- waf/kona.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waf/kona.py b/waf/kona.py index 731cd6f99..db651e970 100644 --- a/waf/kona.py +++ b/waf/kona.py @@ -16,7 +16,7 @@ def detect(get_page): for vector in WAF_ATTACK_VECTORS: page, headers, code = get_page(get=vector) - retval = code == 501 and re.search(r"Reference #[0-9A-Fa-f.]+", page, re.I) is not None + retval = code in (400, 501) and re.search(r"Reference #[0-9A-Fa-f.]+", page, re.I) is not None if retval: break From 2ff4b78dbb313c4aaea0ff0be6556f80010d6c26 Mon Sep 17 00:00:00 2001 From: Wyc0 Date: Wed, 18 Nov 2015 12:31:40 +0800 Subject: [PATCH 25/38] enhance recognition of "baiduyun" in waf/baidu.py --- waf/baidu.py | 1 + 1 file changed, 1 insertion(+) diff --git a/waf/baidu.py b/waf/baidu.py index 7a15feadb..27f1f21ca 100644 --- a/waf/baidu.py +++ b/waf/baidu.py @@ -17,6 +17,7 @@ def detect(get_page): for vector in WAF_ATTACK_VECTORS: page, headers, code = get_page(get=vector) retval = re.search(r"fhl", headers.get("X-Server", ""), re.I) is not None + retval |= re.search(r"yunjiasu-nginx", headers.get("server"), re.I) is not None if retval: break From 69bc875eb30d74e52be5186133702a9fad5767ef Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Wed, 18 Nov 2015 09:04:01 +0100 Subject: [PATCH 26/38] Minor consistency update (with other WAF scripts) for #1543 --- waf/baidu.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/waf/baidu.py b/waf/baidu.py index 27f1f21ca..81f928bd2 100644 --- a/waf/baidu.py +++ b/waf/baidu.py @@ -7,6 +7,7 @@ See the file 'doc/COPYING' for copying permission import re +from lib.core.enums import HTTP_HEADER from lib.core.settings import WAF_ATTACK_VECTORS __product__ = "Yunjiasu Web Application Firewall (Baidu)" @@ -17,7 +18,7 @@ def detect(get_page): for vector in WAF_ATTACK_VECTORS: page, headers, code = get_page(get=vector) retval = re.search(r"fhl", headers.get("X-Server", ""), re.I) is not None - retval |= re.search(r"yunjiasu-nginx", headers.get("server"), re.I) is not None + retval |= re.search(r"yunjiasu-nginx", headers.get(HTTP_HEADER.SERVER), re.I) is not None if retval: break From efe41fbdc791965c2b5ff711382de4a670b9f027 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Fri, 20 Nov 2015 11:32:54 +0100 Subject: [PATCH 27/38] Fixes #1547 --- lib/techniques/error/use.py | 10 +++++++++- lib/techniques/union/use.py | 10 +++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/techniques/error/use.py b/lib/techniques/error/use.py index 8a8009d34..e394b5b58 100644 --- a/lib/techniques/error/use.py +++ b/lib/techniques/error/use.py @@ -37,6 +37,7 @@ from lib.core.dicts import FROM_DUMMY_TABLE from lib.core.enums import DBMS from lib.core.enums import HASHDB_KEYS from lib.core.enums import HTTP_HEADER +from lib.core.exception import SqlmapDataException from lib.core.settings import CHECK_ZERO_COLUMNS_THRESHOLD from lib.core.settings import MIN_ERROR_CHUNK_LENGTH from lib.core.settings import MAX_ERROR_CHUNK_LENGTH @@ -345,7 +346,14 @@ def errorUse(expression, dump=False): numThreads = min(conf.threads, (stopLimit - startLimit)) threadData = getCurrentThreadData() - threadData.shared.limits = iter(xrange(startLimit, stopLimit)) + + try: + threadData.shared.limits = iter(xrange(startLimit, stopLimit)) + except OverflowError: + errMsg = "boundary limits (%d,%d) are too large. Please rerun " % (startLimit, stopLimit) + errMsg += "with switch '--fresh-queries'" + raise SqlmapDataException(errMsg) + threadData.shared.value = BigArray() threadData.shared.buffered = [] threadData.shared.counter = 0 diff --git a/lib/techniques/union/use.py b/lib/techniques/union/use.py index 034223c52..cc4a93aaa 100644 --- a/lib/techniques/union/use.py +++ b/lib/techniques/union/use.py @@ -43,6 +43,7 @@ from lib.core.data import queries from lib.core.dicts import FROM_DUMMY_TABLE from lib.core.enums import DBMS from lib.core.enums import PAYLOAD +from lib.core.exception import SqlmapDataException from lib.core.exception import SqlmapSyntaxException from lib.core.settings import MAX_BUFFERED_PARTIAL_UNION_LENGTH from lib.core.settings import SQL_SCALAR_REGEX @@ -231,7 +232,14 @@ def unionUse(expression, unpack=True, dump=False): return value threadData = getCurrentThreadData() - threadData.shared.limits = iter(xrange(startLimit, stopLimit)) + + try: + threadData.shared.limits = iter(xrange(startLimit, stopLimit)) + except OverflowError: + errMsg = "boundary limits (%d,%d) are too large. Please rerun " % (startLimit, stopLimit) + errMsg += "with switch '--fresh-queries'" + raise SqlmapDataException(errMsg) + numThreads = min(conf.threads, (stopLimit - startLimit)) threadData.shared.value = BigArray() threadData.shared.buffered = [] From 7fa9c8e938e5f41822c56810f69476a6f6d5389c Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Fri, 20 Nov 2015 11:38:26 +0100 Subject: [PATCH 28/38] Patch for an Issue #1546 --- lib/core/agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/agent.py b/lib/core/agent.py index 68fc22817..6327aeeb8 100644 --- a/lib/core/agent.py +++ b/lib/core/agent.py @@ -1053,7 +1053,7 @@ class Agent(object): """ _ = re.escape(PAYLOAD_DELIMITER) - return re.sub("(?s)(%s.*?%s)" % (_, _), ("%s%s%s" % (PAYLOAD_DELIMITER, payload, PAYLOAD_DELIMITER)).replace("\\", r"\\"), value) if value else value + return re.sub("(?s)(%s.*?%s)" % (_, _), ("%s%s%s" % (PAYLOAD_DELIMITER, getUnicode(payload), PAYLOAD_DELIMITER)).replace("\\", r"\\"), value) if value else value def runAsDBMSUser(self, query): if conf.dbmsCred and "Ad Hoc Distributed Queries" not in query: From a5489516ebbf450504db38a801d001d44f0dc402 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Fri, 20 Nov 2015 16:52:59 +0100 Subject: [PATCH 29/38] Fixes #1550 --- lib/core/option.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/core/option.py b/lib/core/option.py index b666c79a6..7c7a39d5e 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -1027,12 +1027,13 @@ def _setSocketPreConnect(): def _(): while kb.threadContinue: try: - for address in socket._ready: - if len(socket._ready[address]) < SOCKET_PRE_CONNECT_QUEUE_SIZE: - s = socket.socket() + for key in socket._ready: + if len(socket._ready[key]) < SOCKET_PRE_CONNECT_QUEUE_SIZE: + family, type, proto, address = key + s = socket.socket(family, type, proto) s._connect(address) with kb.locks.socket: - socket._ready[address].append(s._sock) + socket._ready[key].append(s._sock) except socket.error: pass finally: @@ -1040,11 +1041,12 @@ def _setSocketPreConnect(): def connect(self, address): found = False + key = (self.family, self.type, self.proto, address) with kb.locks.socket: - if address not in socket._ready: - socket._ready[address] = [] - if len(socket._ready[address]) > 0: - self._sock = socket._ready[address].pop(0) + if key not in socket._ready: + socket._ready[key] = [] + if len(socket._ready[key]) > 0: + self._sock = socket._ready[key].pop(0) found = True if not found: self._connect(address) From 763b72a3ed465167f42295fcdf72097dfb0d47ff Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Fri, 20 Nov 2015 17:01:41 +0100 Subject: [PATCH 30/38] Fixes #1551 --- plugins/generic/filesystem.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/generic/filesystem.py b/plugins/generic/filesystem.py index 30daccdc1..3e36be0ae 100644 --- a/plugins/generic/filesystem.py +++ b/plugins/generic/filesystem.py @@ -52,7 +52,12 @@ class Filesystem: lengthQuery = "SELECT DATALENGTH(%s) FROM %s" % (self.tblField, self.fileTblName) - localFileSize = os.path.getsize(localFile) + try: + localFileSize = os.path.getsize(localFile) + except OSError: + warnMsg = "file '%s' is missing" % localFile + logger.warn(warnMsg) + localFileSize = 0 if fileRead and Backend.isDbms(DBMS.PGSQL): logger.info("length of read file '%s' cannot be checked on PostgreSQL" % remoteFile) From 376037123b9a896b721e58bd1f24cf200044b091 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Sun, 22 Nov 2015 15:33:00 +0100 Subject: [PATCH 31/38] Minor fix --- lib/core/option.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/option.py b/lib/core/option.py index 7c7a39d5e..7fe25005d 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -1051,7 +1051,7 @@ def _setSocketPreConnect(): if not found: self._connect(address) - if not hasattr(socket, "_connect"): + if not hasattr(socket.socket, "_connect"): socket._ready = {} socket.socket._connect = socket.socket.connect socket.socket.connect = connect From 4d576928a7a0780c7853603be244369637ae01ec Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Sun, 22 Nov 2015 16:05:48 +0100 Subject: [PATCH 32/38] Fixes #1554 --- lib/request/connect.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/request/connect.py b/lib/request/connect.py index 6139eac15..40e873fe3 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -476,7 +476,7 @@ class Connect(object): status = getUnicode(conn.msg) if extractRegexResult(META_REFRESH_REGEX, page) and not refreshing: - url = extractRegexResult(META_REFRESH_REGEX, page) + refresh = extractRegexResult(META_REFRESH_REGEX, page) debugMsg = "got HTML meta refresh header" logger.debug(debugMsg) @@ -491,13 +491,14 @@ class Connect(object): kb.alwaysRefresh = choice not in ("n", "N") if kb.alwaysRefresh: - if url.lower().startswith('http://'): - kwargs['url'] = url + if re.search(r"\Ahttps?://", refresh, re.I): + url = refresh else: - kwargs['url'] = conf.url[:conf.url.rfind('/') + 1] + url + url = urlparse.urljoin(url, refresh) threadData.lastRedirectMsg = (threadData.lastRequestUID, page) kwargs['refreshing'] = True + kwargs['url'] = url kwargs['get'] = None kwargs['post'] = None From b2dc443835d1d216a83773d78513bcc777a2fe97 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 23 Nov 2015 09:20:35 +0100 Subject: [PATCH 33/38] Fixes #1559 --- plugins/generic/custom.py | 53 +++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/plugins/generic/custom.py b/plugins/generic/custom.py index 8362a4948..65a0b93ed 100644 --- a/plugins/generic/custom.py +++ b/plugins/generic/custom.py @@ -17,6 +17,7 @@ from lib.core.data import conf from lib.core.data import logger from lib.core.dicts import SQL_STATEMENTS from lib.core.enums import AUTOCOMPLETE_TYPE +from lib.core.exception import SqlmapNoneDataException from lib.core.settings import NULL from lib.core.settings import PARAMETER_SPLITTING_REGEX from lib.core.shell import autoCompletion @@ -35,38 +36,42 @@ class Custom: sqlType = None query = query.rstrip(';') - for sqlTitle, sqlStatements in SQL_STATEMENTS.items(): - for sqlStatement in sqlStatements: - if query.lower().startswith(sqlStatement): - sqlType = sqlTitle - break + try: + for sqlTitle, sqlStatements in SQL_STATEMENTS.items(): + for sqlStatement in sqlStatements: + if query.lower().startswith(sqlStatement): + sqlType = sqlTitle + break - if not any(_ in query.upper() for _ in ("OPENROWSET", "INTO")) and (not sqlType or "SELECT" in sqlType): - infoMsg = "fetching %s query output: '%s'" % (sqlType if sqlType is not None else "SQL", query) - logger.info(infoMsg) + if not any(_ in query.upper() for _ in ("OPENROWSET", "INTO")) and (not sqlType or "SELECT" in sqlType): + infoMsg = "fetching %s query output: '%s'" % (sqlType if sqlType is not None else "SQL", query) + logger.info(infoMsg) - output = inject.getValue(query, fromUser=True) + output = inject.getValue(query, fromUser=True) - return output - elif not isStackingAvailable() and not conf.direct: - warnMsg = "execution of custom SQL queries is only " - warnMsg += "available when stacked queries are supported" - logger.warn(warnMsg) + return output + elif not isStackingAvailable() and not conf.direct: + warnMsg = "execution of custom SQL queries is only " + warnMsg += "available when stacked queries are supported" + logger.warn(warnMsg) - return None - else: - if sqlType: - debugMsg = "executing %s query: '%s'" % (sqlType if sqlType is not None else "SQL", query) + return None else: - debugMsg = "executing unknown SQL type query: '%s'" % query - logger.debug(debugMsg) + if sqlType: + debugMsg = "executing %s query: '%s'" % (sqlType if sqlType is not None else "SQL", query) + else: + debugMsg = "executing unknown SQL type query: '%s'" % query + logger.debug(debugMsg) - inject.goStacked(query) + inject.goStacked(query) - debugMsg = "done" - logger.debug(debugMsg) + debugMsg = "done" + logger.debug(debugMsg) - output = NULL + output = NULL + + except SqlmapNoneDataException, ex: + logger.warn(ex) return output From bdb496eaa59f571b48de44c794e0a53016267e1b Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 23 Nov 2015 09:24:30 +0100 Subject: [PATCH 34/38] Fixes #1558 --- lib/utils/purge.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/utils/purge.py b/lib/utils/purge.py index 1447f8061..a27e7f926 100644 --- a/lib/utils/purge.py +++ b/lib/utils/purge.py @@ -11,6 +11,7 @@ import shutil import stat import string +from lib.core.common import getSafeExString from lib.core.data import logger def purge(directory): @@ -79,4 +80,4 @@ def purge(directory): try: shutil.rmtree(directory) except OSError, ex: - logger.error("problem occurred while removing directory '%s' ('%s')" % (directory, unicode(ex))) + logger.error("problem occurred while removing directory '%s' ('%s')" % (directory, getSafeExString(ex))) From 6c083956f41618d1c9dcec3f24912c1b99f16992 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 23 Nov 2015 09:48:43 +0100 Subject: [PATCH 35/38] Patch related to the #1557 --- plugins/dbms/mysql/takeover.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/dbms/mysql/takeover.py b/plugins/dbms/mysql/takeover.py index 8d132fb60..ebd7bdc03 100644 --- a/plugins/dbms/mysql/takeover.py +++ b/plugins/dbms/mysql/takeover.py @@ -60,7 +60,7 @@ class Takeover(GenericTakeover): else: self.__plugindir = "%s/lib/mysql/plugin" % self.__basedir - self.__plugindir = ntToPosixSlashes(normalizePath(self.__plugindir)) + self.__plugindir = ntToPosixSlashes(normalizePath(self.__plugindir)) self.udfRemoteFile = "%s/%s.%s" % (self.__plugindir, self.udfSharedLibName, self.udfSharedLibExt) From 436d87dee1ce9b26ef381c6f6a72e1396a4858bf Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 24 Nov 2015 09:18:46 +0100 Subject: [PATCH 36/38] Fixes #1560 --- sqlmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlmap.py b/sqlmap.py index 4a0e1ff5a..fdbc7afa1 100755 --- a/sqlmap.py +++ b/sqlmap.py @@ -142,7 +142,7 @@ def main(): errMsg = unhandledExceptionMessage() excMsg = traceback.format_exc() - if "No space left" in excMsg: + if any(_ in excMsg for _ in ("No space left", "Disk quota exceeded")): errMsg = "no space left on output device" logger.error(errMsg) raise SystemExit From 527dcce08d1633110989eae79a420b018a3bc32c Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 24 Nov 2015 09:25:11 +0100 Subject: [PATCH 37/38] Better alternative (on Linux getctime() is the time of the last metadata change) --- lib/core/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index 72d7a906f..d942dcdc2 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from lib.core.revision import getRevisionNumber # sqlmap version and site VERSION = "1.0-dev" REVISION = getRevisionNumber() -VERSION_STRING = "sqlmap/%s%s" % (VERSION, "-%s" % REVISION if REVISION else "-nongit-%s" % time.strftime("%Y%m%d", time.gmtime(os.path.getctime(__file__)))) +VERSION_STRING = "sqlmap/%s%s" % (VERSION, "-%s" % REVISION if REVISION else "-nongit-%s" % time.strftime("%Y%m%d", time.gmtime(os.path.getmtime(__file__)))) DESCRIPTION = "automatic SQL injection and database takeover tool" SITE = "http://sqlmap.org" ISSUES_PAGE = "https://github.com/sqlmapproject/sqlmap/issues/new" From 5020269f507d027056312fb63a2a985622e45d59 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 24 Nov 2015 09:38:28 +0100 Subject: [PATCH 38/38] Adding extra mark into non-git checkouts --- lib/core/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/settings.py b/lib/core/settings.py index d942dcdc2..bf5439130 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from lib.core.revision import getRevisionNumber # sqlmap version and site VERSION = "1.0-dev" REVISION = getRevisionNumber() -VERSION_STRING = "sqlmap/%s%s" % (VERSION, "-%s" % REVISION if REVISION else "-nongit-%s" % time.strftime("%Y%m%d", time.gmtime(os.path.getmtime(__file__)))) +VERSION_STRING = "sqlmap/%s%s" % (VERSION, "-%s" % REVISION if REVISION else "-nongit-%s%04x" % (time.strftime("%Y%m%d", time.gmtime(os.path.getmtime(__file__))), os.path.getsize(os.path.join(os.path.dirname(__file__), "common.py")) & 0xffff)) DESCRIPTION = "automatic SQL injection and database takeover tool" SITE = "http://sqlmap.org" ISSUES_PAGE = "https://github.com/sqlmapproject/sqlmap/issues/new"