diff --git a/lib/core/bigarray.py b/lib/core/bigarray.py index a6f6ac24d..ea6338697 100644 --- a/lib/core/bigarray.py +++ b/lib/core/bigarray.py @@ -51,6 +51,11 @@ class Cache(object): class BigArray(list): """ List-like class used for storing large amounts of data (disk cached) + + >>> _ = BigArray(xrange(100000)) + >>> _[20] = 0 + >>> _[100] + 100 """ def __init__(self, items=None): diff --git a/lib/core/dicts.py b/lib/core/dicts.py index e1a21fa4a..5fb35af9e 100644 --- a/lib/core/dicts.py +++ b/lib/core/dicts.py @@ -334,3 +334,260 @@ PART_RUN_CONTENT_TYPES = { "osCmd": CONTENT_TYPE.OS_CMD, "regRead": CONTENT_TYPE.REG_READ } + +# Reference: http://www.w3.org/TR/1999/REC-html401-19991224/sgml/entities.html + +HTML_ENTITIES = { + "quot": 34, + "amp": 38, + "lt": 60, + "gt": 62, + "nbsp": 160, + "iexcl": 161, + "cent": 162, + "pound": 163, + "curren": 164, + "yen": 165, + "brvbar": 166, + "sect": 167, + "uml": 168, + "copy": 169, + "ordf": 170, + "laquo": 171, + "not": 172, + "shy": 173, + "reg": 174, + "macr": 175, + "deg": 176, + "plusmn": 177, + "sup2": 178, + "sup3": 179, + "acute": 180, + "micro": 181, + "para": 182, + "middot": 183, + "cedil": 184, + "sup1": 185, + "ordm": 186, + "raquo": 187, + "frac14": 188, + "frac12": 189, + "frac34": 190, + "iquest": 191, + "Agrave": 192, + "Aacute": 193, + "Acirc": 194, + "Atilde": 195, + "Auml": 196, + "Aring": 197, + "AElig": 198, + "Ccedil": 199, + "Egrave": 200, + "Eacute": 201, + "Ecirc": 202, + "Euml": 203, + "Igrave": 204, + "Iacute": 205, + "Icirc": 206, + "Iuml": 207, + "ETH": 208, + "Ntilde": 209, + "Ograve": 210, + "Oacute": 211, + "Ocirc": 212, + "Otilde": 213, + "Ouml": 214, + "times": 215, + "Oslash": 216, + "Ugrave": 217, + "Uacute": 218, + "Ucirc": 219, + "Uuml": 220, + "Yacute": 221, + "THORN": 222, + "szlig": 223, + "agrave": 224, + "aacute": 225, + "acirc": 226, + "atilde": 227, + "auml": 228, + "aring": 229, + "aelig": 230, + "ccedil": 231, + "egrave": 232, + "eacute": 233, + "ecirc": 234, + "euml": 235, + "igrave": 236, + "iacute": 237, + "icirc": 238, + "iuml": 239, + "eth": 240, + "ntilde": 241, + "ograve": 242, + "oacute": 243, + "ocirc": 244, + "otilde": 245, + "ouml": 246, + "divide": 247, + "oslash": 248, + "ugrave": 249, + "uacute": 250, + "ucirc": 251, + "uuml": 252, + "yacute": 253, + "thorn": 254, + "yuml": 255, + "OElig": 338, + "oelig": 339, + "Scaron": 352, + "fnof": 402, + "scaron": 353, + "Yuml": 376, + "circ": 710, + "tilde": 732, + "Alpha": 913, + "Beta": 914, + "Gamma": 915, + "Delta": 916, + "Epsilon": 917, + "Zeta": 918, + "Eta": 919, + "Theta": 920, + "Iota": 921, + "Kappa": 922, + "Lambda": 923, + "Mu": 924, + "Nu": 925, + "Xi": 926, + "Omicron": 927, + "Pi": 928, + "Rho": 929, + "Sigma": 931, + "Tau": 932, + "Upsilon": 933, + "Phi": 934, + "Chi": 935, + "Psi": 936, + "Omega": 937, + "alpha": 945, + "beta": 946, + "gamma": 947, + "delta": 948, + "epsilon": 949, + "zeta": 950, + "eta": 951, + "theta": 952, + "iota": 953, + "kappa": 954, + "lambda": 955, + "mu": 956, + "nu": 957, + "xi": 958, + "omicron": 959, + "pi": 960, + "rho": 961, + "sigmaf": 962, + "sigma": 963, + "tau": 964, + "upsilon": 965, + "phi": 966, + "chi": 967, + "psi": 968, + "omega": 969, + "thetasym": 977, + "upsih": 978, + "piv": 982, + "bull": 8226, + "hellip": 8230, + "prime": 8242, + "Prime": 8243, + "oline": 8254, + "frasl": 8260, + "ensp": 8194, + "emsp": 8195, + "thinsp": 8201, + "zwnj": 8204, + "zwj": 8205, + "lrm": 8206, + "rlm": 8207, + "ndash": 8211, + "mdash": 8212, + "lsquo": 8216, + "rsquo": 8217, + "sbquo": 8218, + "ldquo": 8220, + "rdquo": 8221, + "bdquo": 8222, + "dagger": 8224, + "Dagger": 8225, + "permil": 8240, + "lsaquo": 8249, + "rsaquo": 8250, + "euro": 8364, + "weierp": 8472, + "image": 8465, + "real": 8476, + "trade": 8482, + "alefsym": 8501, + "larr": 8592, + "uarr": 8593, + "rarr": 8594, + "darr": 8595, + "harr": 8596, + "crarr": 8629, + "lArr": 8656, + "uArr": 8657, + "rArr": 8658, + "dArr": 8659, + "hArr": 8660, + "forall": 8704, + "part": 8706, + "exist": 8707, + "empty": 8709, + "nabla": 8711, + "isin": 8712, + "notin": 8713, + "ni": 8715, + "prod": 8719, + "sum": 8721, + "minus": 8722, + "lowast": 8727, + "radic": 8730, + "prop": 8733, + "infin": 8734, + "ang": 8736, + "and": 8743, + "or": 8744, + "cap": 8745, + "cup": 8746, + "int": 8747, + "there4": 8756, + "sim": 8764, + "cong": 8773, + "asymp": 8776, + "ne": 8800, + "equiv": 8801, + "le": 8804, + "ge": 8805, + "sub": 8834, + "sup": 8835, + "nsub": 8836, + "sube": 8838, + "supe": 8839, + "oplus": 8853, + "otimes": 8855, + "perp": 8869, + "sdot": 8901, + "lceil": 8968, + "rceil": 8969, + "lfloor": 8970, + "rfloor": 8971, + "lang": 9001, + "rang": 9002, + "loz": 9674, + "spades": 9824, + "clubs": 9827, + "hearts": 9829, + "diams": 9830 +} diff --git a/lib/core/settings.py b/lib/core/settings.py index 82575e1d2..6cf837db9 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from lib.core.enums import OS from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.3.7.2" +VERSION = "1.3.7.3" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/core/wordlist.py b/lib/core/wordlist.py index 89ded46ff..1b4d5b048 100644 --- a/lib/core/wordlist.py +++ b/lib/core/wordlist.py @@ -18,7 +18,9 @@ class Wordlist(six.Iterator): Iterator for looping over a large dictionaries >>> from lib.core.option import paths - >>> isinstance(next(Wordlist(paths.SMALL_DICT)), six.string_types) + >>> isinstance(next(Wordlist(paths.SMALL_DICT)), six.binary_type) + True + >>> isinstance(next(Wordlist(paths.WORDLIST)), six.binary_type) True """ @@ -58,7 +60,7 @@ class Wordlist(six.Iterator): raise SqlmapDataException(errMsg) self.fp = _.open(_.namelist()[0]) else: - self.fp = open(self.current, 'r') + self.fp = open(self.current, "rb") self.iter = iter(self.fp) self.index += 1 diff --git a/lib/request/basic.py b/lib/request/basic.py index 0ea5614b8..34a9d55f3 100644 --- a/lib/request/basic.py +++ b/lib/request/basic.py @@ -34,6 +34,7 @@ from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger from lib.core.decorators import cachedmethod +from lib.core.dicts import HTML_ENTITIES from lib.core.enums import DBMS from lib.core.enums import HTTP_HEADER from lib.core.enums import PLACE @@ -49,7 +50,6 @@ from lib.core.settings import SELECT_FROM_TABLE_REGEX from lib.core.settings import VIEWSTATE_REGEX from lib.parse.headers import headersParser from lib.parse.html import htmlParser -from lib.utils.htmlentities import htmlEntities from thirdparty import six from thirdparty.chardet import detect from thirdparty.identywaf import identYwaf @@ -341,7 +341,7 @@ def decodePage(page, contentEncoding, contentType): page = re.sub(b"%([0-9a-fA-F]{2})", lambda _: decodeHex(_.group(1)), page) # e.g. & - page = re.sub(b"&([^;]+);", lambda _: six.int2byte(htmlEntities[getText(_.group(1))]) if htmlEntities.get(getText(_.group(1)), 256) < 256 else _.group(0), page) + page = re.sub(b"&([^;]+);", lambda _: six.int2byte(HTML_ENTITIES[getText(_.group(1))]) if HTML_ENTITIES.get(getText(_.group(1)), 256) < 256 else _.group(0), page) kb.pageEncoding = kb.pageEncoding or checkCharEncoding(getHeuristicCharEncoding(page)) @@ -364,7 +364,7 @@ def decodePage(page, contentEncoding, contentType): page = re.sub(r"&#(\d+);", _, page) # e.g. ζ - page = re.sub(r"&([^;]+);", lambda _: _unichr(htmlEntities[_.group(1)]) if htmlEntities.get(_.group(1), 0) > 255 else _.group(0), page) + page = re.sub(r"&([^;]+);", lambda _: _unichr(HTML_ENTITIES[_.group(1)]) if HTML_ENTITIES.get(_.group(1), 0) > 255 else _.group(0), page) return page diff --git a/lib/utils/htmlentities.py b/lib/utils/htmlentities.py deleted file mode 100644 index a97320ec0..000000000 --- a/lib/utils/htmlentities.py +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/env python - -""" -Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/) -See the file 'LICENSE' for copying permission -""" - -# Reference: http://www.w3.org/TR/1999/REC-html401-19991224/sgml/entities.html - -htmlEntities = { - "quot": 34, - "amp": 38, - "lt": 60, - "gt": 62, - "nbsp": 160, - "iexcl": 161, - "cent": 162, - "pound": 163, - "curren": 164, - "yen": 165, - "brvbar": 166, - "sect": 167, - "uml": 168, - "copy": 169, - "ordf": 170, - "laquo": 171, - "not": 172, - "shy": 173, - "reg": 174, - "macr": 175, - "deg": 176, - "plusmn": 177, - "sup2": 178, - "sup3": 179, - "acute": 180, - "micro": 181, - "para": 182, - "middot": 183, - "cedil": 184, - "sup1": 185, - "ordm": 186, - "raquo": 187, - "frac14": 188, - "frac12": 189, - "frac34": 190, - "iquest": 191, - "Agrave": 192, - "Aacute": 193, - "Acirc": 194, - "Atilde": 195, - "Auml": 196, - "Aring": 197, - "AElig": 198, - "Ccedil": 199, - "Egrave": 200, - "Eacute": 201, - "Ecirc": 202, - "Euml": 203, - "Igrave": 204, - "Iacute": 205, - "Icirc": 206, - "Iuml": 207, - "ETH": 208, - "Ntilde": 209, - "Ograve": 210, - "Oacute": 211, - "Ocirc": 212, - "Otilde": 213, - "Ouml": 214, - "times": 215, - "Oslash": 216, - "Ugrave": 217, - "Uacute": 218, - "Ucirc": 219, - "Uuml": 220, - "Yacute": 221, - "THORN": 222, - "szlig": 223, - "agrave": 224, - "aacute": 225, - "acirc": 226, - "atilde": 227, - "auml": 228, - "aring": 229, - "aelig": 230, - "ccedil": 231, - "egrave": 232, - "eacute": 233, - "ecirc": 234, - "euml": 235, - "igrave": 236, - "iacute": 237, - "icirc": 238, - "iuml": 239, - "eth": 240, - "ntilde": 241, - "ograve": 242, - "oacute": 243, - "ocirc": 244, - "otilde": 245, - "ouml": 246, - "divide": 247, - "oslash": 248, - "ugrave": 249, - "uacute": 250, - "ucirc": 251, - "uuml": 252, - "yacute": 253, - "thorn": 254, - "yuml": 255, - "OElig": 338, - "oelig": 339, - "Scaron": 352, - "fnof": 402, - "scaron": 353, - "Yuml": 376, - "circ": 710, - "tilde": 732, - "Alpha": 913, - "Beta": 914, - "Gamma": 915, - "Delta": 916, - "Epsilon": 917, - "Zeta": 918, - "Eta": 919, - "Theta": 920, - "Iota": 921, - "Kappa": 922, - "Lambda": 923, - "Mu": 924, - "Nu": 925, - "Xi": 926, - "Omicron": 927, - "Pi": 928, - "Rho": 929, - "Sigma": 931, - "Tau": 932, - "Upsilon": 933, - "Phi": 934, - "Chi": 935, - "Psi": 936, - "Omega": 937, - "alpha": 945, - "beta": 946, - "gamma": 947, - "delta": 948, - "epsilon": 949, - "zeta": 950, - "eta": 951, - "theta": 952, - "iota": 953, - "kappa": 954, - "lambda": 955, - "mu": 956, - "nu": 957, - "xi": 958, - "omicron": 959, - "pi": 960, - "rho": 961, - "sigmaf": 962, - "sigma": 963, - "tau": 964, - "upsilon": 965, - "phi": 966, - "chi": 967, - "psi": 968, - "omega": 969, - "thetasym": 977, - "upsih": 978, - "piv": 982, - "bull": 8226, - "hellip": 8230, - "prime": 8242, - "Prime": 8243, - "oline": 8254, - "frasl": 8260, - "ensp": 8194, - "emsp": 8195, - "thinsp": 8201, - "zwnj": 8204, - "zwj": 8205, - "lrm": 8206, - "rlm": 8207, - "ndash": 8211, - "mdash": 8212, - "lsquo": 8216, - "rsquo": 8217, - "sbquo": 8218, - "ldquo": 8220, - "rdquo": 8221, - "bdquo": 8222, - "dagger": 8224, - "Dagger": 8225, - "permil": 8240, - "lsaquo": 8249, - "rsaquo": 8250, - "euro": 8364, - "weierp": 8472, - "image": 8465, - "real": 8476, - "trade": 8482, - "alefsym": 8501, - "larr": 8592, - "uarr": 8593, - "rarr": 8594, - "darr": 8595, - "harr": 8596, - "crarr": 8629, - "lArr": 8656, - "uArr": 8657, - "rArr": 8658, - "dArr": 8659, - "hArr": 8660, - "forall": 8704, - "part": 8706, - "exist": 8707, - "empty": 8709, - "nabla": 8711, - "isin": 8712, - "notin": 8713, - "ni": 8715, - "prod": 8719, - "sum": 8721, - "minus": 8722, - "lowast": 8727, - "radic": 8730, - "prop": 8733, - "infin": 8734, - "ang": 8736, - "and": 8743, - "or": 8744, - "cap": 8745, - "cup": 8746, - "int": 8747, - "there4": 8756, - "sim": 8764, - "cong": 8773, - "asymp": 8776, - "ne": 8800, - "equiv": 8801, - "le": 8804, - "ge": 8805, - "sub": 8834, - "sup": 8835, - "nsub": 8836, - "sube": 8838, - "supe": 8839, - "oplus": 8853, - "otimes": 8855, - "perp": 8869, - "sdot": 8901, - "lceil": 8968, - "rceil": 8969, - "lfloor": 8970, - "rfloor": 8971, - "lang": 9001, - "rang": 9002, - "loz": 9674, - "spades": 9824, - "clubs": 9827, - "hearts": 9829, - "diams": 9830, -}