diff --git a/data/xml/queries.xml b/data/xml/queries.xml index 75f6edf95..deda4364d 100644 --- a/data/xml/queries.xml +++ b/data/xml/queries.xml @@ -357,8 +357,8 @@ - - + + diff --git a/extra/vulnserver/vulnserver.py b/extra/vulnserver/vulnserver.py index f7211e61c..37d7df3c3 100644 --- a/extra/vulnserver/vulnserver.py +++ b/extra/vulnserver/vulnserver.py @@ -44,7 +44,8 @@ SCHEMA = """ CREATE TABLE users ( id INTEGER, name TEXT, - surname TEXT + surname TEXT, + PRIMARY KEY (id) ); INSERT INTO users (id, name, surname) VALUES (1, 'luther', 'blisset'); INSERT INTO users (id, name, surname) VALUES (2, 'fluffy', 'bunny'); diff --git a/lib/core/common.py b/lib/core/common.py index 8f29e086a..bf2006d72 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -1034,7 +1034,10 @@ def dataToStdout(data, forceOutput=False, bold=False, contentType=None, status=C except UnicodeEncodeError: sys.stdout.write(re.sub(r"[^ -~]", '?', clearColors(data))) finally: - sys.stdout.flush() + try: + sys.stdout.flush() + except IOError: + raise SystemExit if multiThreadMode: logging._releaseLock() @@ -1819,7 +1822,7 @@ def expandAsteriskForColumns(expression): the SQL query string (expression) """ - match = re.search(r"(?i)\ASELECT(\s+TOP\s+[\d]+)?\s+\*\s+FROM\s+((`[^`]+`|[^\s]+)+)", expression) + match = re.search(r"(?i)\ASELECT(\s+TOP\s+[\d]+)?\s+\*\s+FROM\s+(([`'\"][^`'\"]+[`'\"]|[\w.]+)+)(\s|\Z)", expression) if match: infoMsg = "you did not provide the fields in your query. " @@ -3399,19 +3402,39 @@ def parseSqliteTableSchema(value): >>> kb.data.cachedColumns = {} >>> parseSqliteTableSchema("CREATE TABLE users(\\n\\t\\tid INTEGER,\\n\\t\\tname TEXT\\n);") True - >>> repr(kb.data.cachedColumns).count(',') == 1 + >>> tuple(kb.data.cachedColumns[conf.db][conf.tbl].items()) == (('id', 'INTEGER'), ('name', 'TEXT')) + True + >>> parseSqliteTableSchema("CREATE TABLE dummy(`foo bar` BIGINT, \\"foo\\" VARCHAR, 'bar' TEXT)"); + True + >>> tuple(kb.data.cachedColumns[conf.db][conf.tbl].items()) == (('foo bar', 'BIGINT'), ('foo', 'VARCHAR'), ('bar', 'TEXT')) + True + >>> parseSqliteTableSchema("CREATE TABLE suppliers(\\n\\tsupplier_id INTEGER PRIMARY KEY DESC,\\n\\tname TEXT NOT NULL\\n);"); + True + >>> tuple(kb.data.cachedColumns[conf.db][conf.tbl].items()) == (('supplier_id', 'INTEGER'), ('name', 'TEXT')) + True + >>> parseSqliteTableSchema("CREATE TABLE country_languages (\\n\\tcountry_id INTEGER NOT NULL,\\n\\tlanguage_id INTEGER NOT NULL,\\n\\tPRIMARY KEY (country_id, language_id),\\n\\tFOREIGN KEY (country_id) REFERENCES countries (country_id) ON DELETE CASCADE ON UPDATE NO ACTION,\\tFOREIGN KEY (language_id) REFERENCES languages (language_id) ON DELETE CASCADE ON UPDATE NO ACTION);"); + True + >>> tuple(kb.data.cachedColumns[conf.db][conf.tbl].items()) == (('country_id', 'INTEGER'), ('language_id', 'INTEGER')) True """ retVal = False + value = extractRegexResult(r"(?s)\((?P.+)\)", value) + if value: table = {} - columns = {} + columns = OrderedDict() - for match in re.finditer(r"[(,]\s*[\"'`]?(\w+)[\"'`]?(?:\s+(INT|INTEGER|TINYINT|SMALLINT|MEDIUMINT|BIGINT|UNSIGNED BIG INT|INT2|INT8|INTEGER|CHARACTER|VARCHAR|VARYING CHARACTER|NCHAR|NATIVE CHARACTER|NVARCHAR|TEXT|CLOB|LONGTEXT|BLOB|NONE|REAL|DOUBLE|DOUBLE PRECISION|FLOAT|REAL|NUMERIC|DECIMAL|BOOLEAN|DATE|DATETIME|NUMERIC)\b)?", decodeStringEscape(value), re.I): + value = re.sub(r"\(.+?\)", "", value).strip() + + for match in re.finditer(r"(?:\A|,)\s*(([\"'`]).+?\2|\w+)(?:\s+(INT|INTEGER|TINYINT|SMALLINT|MEDIUMINT|BIGINT|UNSIGNED BIG INT|INT2|INT8|INTEGER|CHARACTER|VARCHAR|VARYING CHARACTER|NCHAR|NATIVE CHARACTER|NVARCHAR|TEXT|CLOB|LONGTEXT|BLOB|NONE|REAL|DOUBLE|DOUBLE PRECISION|FLOAT|REAL|NUMERIC|DECIMAL|BOOLEAN|DATE|DATETIME|NUMERIC)\b)?", decodeStringEscape(value), re.I): + column = match.group(1).strip(match.group(2) or "") + if re.search(r"(?i)\A(CONSTRAINT|PRIMARY|UNIQUE|CHECK|FOREIGN)\b", column.strip()): + continue retVal = True - columns[match.group(1)] = match.group(2) or "TEXT" + + columns[column] = match.group(3) or "TEXT" table[safeSQLIdentificatorNaming(conf.tbl, True)] = columns kb.data.cachedColumns[conf.db] = table @@ -4010,7 +4033,7 @@ def maskSensitiveData(msg): >>> maskSensitiveData('python sqlmap.py -u "http://www.test.com/vuln.php?id=1" --banner') == 'python sqlmap.py -u *********************************** --banner' True - >>> maskSensitiveData('sqlmap.py -u test.com/index.go?id=index') == 'sqlmap.py -u **************************' + >>> maskSensitiveData('sqlmap.py -u test.com/index.go?id=index --auth-type=basic --auth-creds=foo:bar\\ndummy line') == 'sqlmap.py -u ************************** --auth-type=***** --auth-creds=*******\\ndummy line' True """ @@ -4026,7 +4049,7 @@ def maskSensitiveData(msg): retVal = retVal.replace(value, '*' * len(value)) # Just in case (for problematic parameters regarding user encoding) - for match in re.finditer(r"(?i)[ -]-(u|url|data|cookie|auth-\w+|proxy|host|referer|headers?|H)( |=)(.*?)(?= -?-[a-z]|\Z)", retVal): + for match in re.finditer(r"(?im)[ -]-(u|url|data|cookie|auth-\w+|proxy|host|referer|headers?|H)( |=)(.*?)(?= -?-[a-z]|$)", retVal): retVal = retVal.replace(match.group(3), '*' * len(match.group(3))) # Fail-safe substitutions diff --git a/lib/core/compat.py b/lib/core/compat.py index 7275ea07d..6a7a40769 100644 --- a/lib/core/compat.py +++ b/lib/core/compat.py @@ -12,6 +12,7 @@ import functools import math import os import random +import re import sys import time import uuid @@ -277,8 +278,37 @@ else: xrange = xrange buffer = buffer -try: - from packaging import version - LooseVersion = version.parse -except ImportError: - from distutils.version import LooseVersion +def LooseVersion(version): + """ + >>> LooseVersion("1.0") == LooseVersion("1.0") + True + >>> LooseVersion("1.0.1") > LooseVersion("1.0") + True + >>> LooseVersion("1.0.1-") == LooseVersion("1.0.1") + True + >>> LooseVersion("1.0.11") < LooseVersion("1.0.111") + True + >>> LooseVersion("foobar") > LooseVersion("1.0") + False + >>> LooseVersion("1.0") > LooseVersion("foobar") + False + >>> LooseVersion("3.22-mysql") == LooseVersion("3.22-mysql-ubuntu0.3") + True + >>> LooseVersion("8.0.22-0ubuntu0.20.04.2") + 8.000022 + """ + + match = re.search(r"\A(\d[\d.]*)", version or "") + + if match: + result = 0 + value = match.group(1) + weight = 1.0 + for part in value.strip('.').split('.'): + if part.isdigit(): + result += int(part) * weight + weight *= 1e-3 + else: + result = float("NaN") + + return result diff --git a/lib/core/option.py b/lib/core/option.py index d740e572c..29570dcdb 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -2097,7 +2097,7 @@ def _setKnowledgeBaseAttributes(flushAll=True): kb.lastParserStatus = None kb.locks = AttribDict() - for _ in ("cache", "connError", "count", "handlers", "hint", "index", "io", "limit", "liveCookies", "log", "socket", "redirect", "request", "value"): + for _ in ("cache", "connError", "count", "handlers", "hint", "identYwaf", "index", "io", "limit", "liveCookies", "log", "socket", "redirect", "request", "value"): kb.locks[_] = threading.Lock() kb.matchRatio = None diff --git a/lib/core/settings.py b/lib/core/settings.py index ead5d26f8..ad6ed8453 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from thirdparty import six from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.6.11.10" +VERSION = "1.6.12.11" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/core/testing.py b/lib/core/testing.py index 0514b71e3..96b93ee53 100644 --- a/lib/core/testing.py +++ b/lib/core/testing.py @@ -58,9 +58,9 @@ def vulnTest(): ("-u --flush-session --banner --technique=B --disable-precon --not-string \"no results\"", ("banner: '3.",)), ("-u --flush-session --encoding=gbk --banner --technique=B --first=1 --last=2", ("banner: '3.'",)), ("-u --flush-session --encoding=ascii --forms --crawl=2 --threads=2 --banner", ("total of 2 targets", "might be injectable", "Type: UNION query", "banner: '3.")), - ("-u --flush-session --data=\"{\\\"id\\\": 1}\" --banner", ("might be injectable", "3 columns", "Payload: {\"id\"", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", "banner: '3.")), + ("-u --flush-session --technique=BU --data=\"{\\\"id\\\": 1}\" --banner", ("might be injectable", "3 columns", "Payload: {\"id\"", "Type: boolean-based blind", "Type: UNION query", "banner: '3.")), ("-u --flush-session -H \"Foo: Bar\" -H \"Sna: Fu\" --data=\"\" --union-char=1 --mobile --answers=\"smartphone=3\" --banner --smart -v 5", ("might be injectable", "Payload: --flush-session --method=PUT --data=\"a=1;id=1;b=2\" --param-del=\";\" --skip-static --har= --dump -T users --start=1 --stop=2", ("might be injectable", "Parameter: id (PUT)", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", "2 entries")), + ("-u --flush-session --technique=BU --method=PUT --data=\"a=1;id=1;b=2\" --param-del=\";\" --skip-static --har= --dump -T users --start=1 --stop=2", ("might be injectable", "Parameter: id (PUT)", "Type: boolean-based blind", "Type: UNION query", "2 entries")), ("-u --flush-session -H \"id: 1*\" --tables -t ", ("might be injectable", "Parameter: id #1* ((custom) HEADER)", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", " users ")), ("-u --flush-session --banner --invalid-logical --technique=B --predict-output --test-filter=\"OR boolean\" --tamper=space2dash", ("banner: '3.", " LIKE ")), ("-u --flush-session --cookie=\"PHPSESSID=d41d8cd98f00b204e9800998ecf8427e; id=1*; id2=2\" --tables --union-cols=3", ("might be injectable", "Cookie #1* ((custom) HEADER)", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", " users ")), @@ -69,7 +69,7 @@ def vulnTest(): ("-u --flush-session --parse-errors --test-filter=\"subquery\" --eval=\"import hashlib; id2=2; id3=hashlib.md5(id.encode()).hexdigest()\" --referer=\"localhost\"", ("might be injectable", ": syntax error", "back-end DBMS: SQLite", "WHERE or HAVING clause (subquery")), ("-u --banner --schema --dump -T users --binary-fields=surname --where \"id>3\"", ("banner: '3.", "INTEGER", "TEXT", "id", "name", "surname", "2 entries", "6E616D6569736E756C6C")), ("-u --technique=U --fresh-queries --force-partial --dump -T users --dump-format=HTML --answers=\"crack=n\" -v 3", ("performed 6 queries", "nameisnull", "~using default dictionary", "dumped to HTML file")), - ("-u --flush-session --all", ("5 entries", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", "luther", "blisset", "fluffy", "179ad45c6ce2cb97cf1029e212046e81", "NULL", "nameisnull", "testpass")), + ("-u --flush-session --technique=BU --all", ("5 entries", "Type: boolean-based blind", "Type: UNION query", "luther", "blisset", "fluffy", "179ad45c6ce2cb97cf1029e212046e81", "NULL", "nameisnull", "testpass")), ("-u -z \"tec=B\" --hex --fresh-queries --threads=4 --sql-query=\"SELECT * FROM users\"", ("SELECT * FROM users [5]", "nameisnull")), ("-u \"&echo=foobar*\" --flush-session", ("might be vulnerable to cross-site scripting",)), ("-u \"&query=*\" --flush-session --technique=Q --banner", ("Title: SQLite inline queries", "banner: '3.")), diff --git a/lib/request/basic.py b/lib/request/basic.py index ae3bc2353..d865575e1 100644 --- a/lib/request/basic.py +++ b/lib/request/basic.py @@ -108,7 +108,7 @@ def forgeHeaders(items=None, base=None): if conf.cj: if HTTP_HEADER.COOKIE in headers: for cookie in conf.cj: - if cookie.domain_specified and not (conf.hostname or "").endswith(cookie.domain): + if cookie is None or cookie.domain_specified and not (conf.hostname or "").endswith(cookie.domain): continue if ("%s=" % getUnicode(cookie.name)) in getUnicode(headers[HTTP_HEADER.COOKIE]): @@ -401,13 +401,14 @@ def processResponse(page, responseHeaders, code=None, status=None): if not conf.skipWaf and kb.processResponseCounter < IDENTYWAF_PARSE_LIMIT: rawResponse = "%s %s %s\n%s\n%s" % (_http_client.HTTPConnection._http_vsn_str, code or "", status or "", "".join(getUnicode(responseHeaders.headers if responseHeaders else [])), page[:HEURISTIC_PAGE_SIZE_THRESHOLD]) - identYwaf.non_blind.clear() - if identYwaf.non_blind_check(rawResponse, silent=True): - for waf in identYwaf.non_blind: - if waf not in kb.identifiedWafs: - kb.identifiedWafs.add(waf) - errMsg = "WAF/IPS identified as '%s'" % identYwaf.format_name(waf) - singleTimeLogMessage(errMsg, logging.CRITICAL) + with kb.locks.identYwaf: + identYwaf.non_blind.clear() + if identYwaf.non_blind_check(rawResponse, silent=True): + for waf in set(identYwaf.non_blind): + if waf not in kb.identifiedWafs: + kb.identifiedWafs.add(waf) + errMsg = "WAF/IPS identified as '%s'" % identYwaf.format_name(waf) + singleTimeLogMessage(errMsg, logging.CRITICAL) if kb.originalPage is None: for regex in (EVENTVALIDATION_REGEX, VIEWSTATE_REGEX): diff --git a/lib/request/connect.py b/lib/request/connect.py index 84ec25e4d..5c0a207aa 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -308,7 +308,7 @@ class Connect(object): threadData.lastRequestUID = kb.requestCounter if conf.proxyFreq: - if kb.requestCounter % conf.proxyFreq == 1: + if kb.requestCounter % conf.proxyFreq == 0: conf.proxy = None warnMsg = "changing proxy" diff --git a/lib/utils/api.py b/lib/utils/api.py index 363b5bea2..6fa8c0ab2 100644 --- a/lib/utils/api.py +++ b/lib/utils/api.py @@ -17,6 +17,7 @@ import socket import sqlite3 import sys import tempfile +import threading import time from lib.core.common import dataToStdout @@ -88,6 +89,7 @@ class Database(object): def connect(self, who="server"): self.connection = sqlite3.connect(self.database, timeout=3, isolation_level=None, check_same_thread=False) self.cursor = self.connection.cursor() + self.lock = threading.Lock() logger.debug("REST-JSON API %s connected to IPC database" % who) def disconnect(self): @@ -101,17 +103,20 @@ class Database(object): self.connection.commit() def execute(self, statement, arguments=None): - while True: - try: - if arguments: - self.cursor.execute(statement, arguments) + with self.lock: + while True: + try: + if arguments: + self.cursor.execute(statement, arguments) + else: + self.cursor.execute(statement) + except sqlite3.OperationalError as ex: + if "locked" not in getSafeExString(ex): + raise + else: + time.sleep(1) else: - self.cursor.execute(statement) - except sqlite3.OperationalError as ex: - if "locked" not in getSafeExString(ex): - raise - else: - break + break if statement.lstrip().upper().startswith("SELECT"): return self.cursor.fetchall() diff --git a/sqlmap.py b/sqlmap.py index 7c7e14ebd..93bc145aa 100755 --- a/sqlmap.py +++ b/sqlmap.py @@ -250,7 +250,10 @@ def main(): raise SystemExit except KeyboardInterrupt: - print() + try: + print() + except IOError: + pass except EOFError: print() @@ -495,6 +498,11 @@ def main(): logger.critical(errMsg) raise SystemExit + elif "database disk image is malformed" in excMsg: + errMsg = "local session file seems to be malformed. Please rerun with '--flush-session'" + logger.critical(errMsg) + raise SystemExit + elif "AttributeError: 'module' object has no attribute 'F_GETFD'" in excMsg: errMsg = "invalid runtime (\"%s\") " % excMsg.split("Error: ")[-1].strip() errMsg += "(Reference: 'https://stackoverflow.com/a/38841364' & 'https://bugs.python.org/issue24944#msg249231')"