diff --git a/lib/core/common.py b/lib/core/common.py index 0cc4b3134..3b10135d2 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -3497,6 +3497,28 @@ def getLatestRevision(): return retVal +def fetchRandomAgent(): + """ + Returns random HTTP User-Agent header value + + >>> '(' in fetchRandomAgent() + True + """ + + if not kb.userAgents: + debugMsg = "loading random HTTP User-Agent header(s) from " + debugMsg += "file '%s'" % paths.USER_AGENTS + logger.debug(debugMsg) + + try: + kb.userAgents = getFileItems(paths.USER_AGENTS) + except IOError: + errMsg = "unable to read HTTP User-Agent header " + errMsg += "file '%s'" % paths.USER_AGENTS + raise SqlmapSystemException(errMsg) + + return random.sample(kb.userAgents, 1)[0] + def createGithubIssue(errMsg, excMsg): """ Automatically create a Github issue with unhandled exception information @@ -3550,9 +3572,9 @@ def createGithubIssue(errMsg, excMsg): except: pass - userAgent = "curl/7.{curl_minor}.{curl_revision} (x86_64-pc-linux-gnu) libcurl/7.{curl_minor}.{curl_revision} OpenSSL/0.9.8{openssl_revision} zlib/1.2.{zlib_revision}".format(curl_minor=random.randint(8, 22), curl_revision=random.randint(1, 9), openssl_revision=random.choice(string.ascii_lowercase), zlib_revision=random.randint(2, 6)) + data = {"title": "Unhandled exception (#%s)" % key, "body": "```%s\n```\n```\n%s```" % (errMsg, excMsg)} - req = _urllib.request.Request(url="https://api.github.com/repos/sqlmapproject/sqlmap/issues", data=getBytes(json.dumps(data)), headers={HTTP_HEADER.AUTHORIZATION: "token %s" % decodeBase64(GITHUB_REPORT_OAUTH_TOKEN, binary=False), HTTP_HEADER.USER_AGENT: userAgent}) + req = _urllib.request.Request(url="https://api.github.com/repos/sqlmapproject/sqlmap/issues", data=getBytes(json.dumps(data)), headers={HTTP_HEADER.AUTHORIZATION: "token %s" % decodeBase64(GITHUB_REPORT_OAUTH_TOKEN, binary=False), HTTP_HEADER.USER_AGENT: fetchRandomAgent()}) try: content = _urllib.request.urlopen(req).read() diff --git a/lib/core/option.py b/lib/core/option.py index 423b72cfa..76f369fcb 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -25,6 +25,7 @@ from lib.core.common import dataToStdout from lib.core.common import decodeStringEscape from lib.core.common import getPublicTypeMembers from lib.core.common import getSafeExString +from lib.core.common import fetchRandomAgent from lib.core.common import filterNone from lib.core.common import findLocalPort from lib.core.common import findPageForms @@ -137,7 +138,6 @@ from lib.request.httpshandler import HTTPSHandler from lib.request.pkihandler import HTTPSPKIAuthHandler from lib.request.rangehandler import HTTPRangeHandler from lib.request.redirecthandler import SmartRedirectHandler -from lib.request.templates import getPageTemplate from lib.utils.har import HTTPCollectorFactory from lib.utils.crawler import crawl from lib.utils.deps import checkDependencies @@ -1400,22 +1400,7 @@ def _setHTTPUserAgent(): conf.httpHeaders.append((HTTP_HEADER.USER_AGENT, DEFAULT_USER_AGENT)) else: - if not kb.userAgents: - debugMsg = "loading random HTTP User-Agent header(s) from " - debugMsg += "file '%s'" % paths.USER_AGENTS - logger.debug(debugMsg) - - try: - kb.userAgents = getFileItems(paths.USER_AGENTS) - except IOError: - warnMsg = "unable to read HTTP User-Agent header " - warnMsg += "file '%s'" % paths.USER_AGENTS - logger.warn(warnMsg) - - conf.httpHeaders.append((HTTP_HEADER.USER_AGENT, DEFAULT_USER_AGENT)) - return - - userAgent = random.sample(kb.userAgents or [DEFAULT_USER_AGENT], 1)[0] + userAgent = fetchRandomAgent() infoMsg = "fetched random HTTP User-Agent header value '%s' from " % userAgent infoMsg += "file '%s'" % paths.USER_AGENTS diff --git a/lib/core/settings.py b/lib/core/settings.py index bb34a56dc..e49cf6eed 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from lib.core.enums import OS from thirdparty import six # sqlmap version (...) -VERSION = "1.3.5.24" +VERSION = "1.3.5.25" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/txt/user-agents.txt b/txt/user-agents.txt index 2e0b12bf7..488a09d47 100644 --- a/txt/user-agents.txt +++ b/txt/user-agents.txt @@ -285,7 +285,6 @@ Opera/9.20 (X11; Linux i686; U; es-es) Opera/9.20 (X11; Linux i686; U; pl) Opera/9.20 (X11; Linux i686; U; ru) Opera/9.20 (X11; Linux i686; U; tr) -Opera/9.20 (X11; Linux ppc; U; en) Opera/9.20 (X11; Linux x86_64; U; en) Opera/9.21 (Macintosh; Intel Mac OS X; U; en) Opera/9.21 (Macintosh; PPC Mac OS X; U; en) @@ -364,8 +363,8 @@ Opera/9.27 (Windows NT 5.1; U; ja) Opera/9.27 (Windows NT 5.2; U; en) Opera/9.27 (X11; Linux i686; U; en) Opera/9.27 (X11; Linux i686; U; fr) -Opera 9.4 (Windows NT 5.3; U; en) -Opera 9.4 (Windows NT 6.1; U; en) +Opera/9.4 (Windows NT 5.3; U; en) +Opera/9.4 (Windows NT 6.1; U; en) Opera/9.50 (Macintosh; Intel Mac OS X; U; de) Opera/9.50 (Macintosh; Intel Mac OS X; U; en) Opera/9.50 (Windows NT 5.1; U; es-ES) @@ -375,7 +374,6 @@ Opera/9.50 (Windows NT 5.1; U; nn) Opera/9.50 (Windows NT 5.1; U; ru) Opera/9.50 (Windows NT 5.2; U; it) Opera/9.50 (X11; Linux i686; U; es-ES) -Opera/9.50 (X11; Linux ppc; U; en) Opera/9.50 (X11; Linux x86_64; U; nb) Opera/9.50 (X11; Linux x86_64; U; pl) Opera/9.51 (Macintosh; Intel Mac OS X; U; en) @@ -406,7 +404,6 @@ Opera/9.52 (Windows NT 6.0; U; Opera/9.52 (X11; Linux x86_64; U); en) Opera/9.52 (X11; Linux i686; U; cs) Opera/9.52 (X11; Linux i686; U; en) Opera/9.52 (X11; Linux i686; U; fr) -Opera/9.52 (X11; Linux ppc; U; de) Opera/9.52 (X11; Linux x86_64; U) Opera/9.52 (X11; Linux x86_64; U; en) Opera/9.52 (X11; Linux x86_64; U; ru) @@ -616,7 +613,6 @@ Opera/12.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.02 # Mozilla Firefox -mozilla/3.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/5.0.1 Mozilla/4.0 (compatible; Intel Mac OS X 10.6; rv:2.0b8) Gecko/20100101 Firefox/4.0b8) Mozilla/4.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.2.2) Gecko/2010324480 Firefox/3.5.4 Mozilla/4.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.7) Gecko/2008398325 Firefox/3.1.4 @@ -1125,7 +1121,7 @@ Mozilla/5.0 (Windows; U; Windows NT 5.2; nl; rv:1.9b5) Gecko/2008032620 Firefox/ Mozilla/5.0 (Windows; U; Windows NT 5.2; ru; rv:1.9.2.11) Gecko/20101012 Firefox/3.6.11 Mozilla/5.0 (Windows; U; Windows NT 5.2; rv:1.7.3) Gecko/20041001 Firefox/0.10.1 Mozilla/5.0 (Windows; U; Windows NT 5.2; rv:1.9.2.11) Gecko/20101012 Firefox/3.6.11 -Mozilla/5.0(Windows; U; Windows NT 5.2; rv:1.9.2) Gecko/20100101 Firefox/3.6 +Mozilla/5.0 (Windows; U; Windows NT 5.2; rv:1.9.2) Gecko/20100101 Firefox/3.6 Mozilla/5.0 (Windows; U; Windows NT 5.2; sk; rv:1.8.1.15) Gecko/20080623 Firefox/2.0.0.15 Mozilla/5.0 (Windows; U; Windows NT 5.2 x64; en-US; rv:1.9a1) Gecko/20060214 Firefox/1.6a1 Mozilla/5.0 (Windows; U; Windows NT 5.2; zh-CN; rv:1.9.1.5) Gecko/Firefox/3.5.5 @@ -1355,7 +1351,7 @@ Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-CN; rv:1.9.2.14) Gecko/20110218 Fire Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-CN; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3 (.NET CLR 3.5.30729) Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-CN; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8 Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-TW; rv:1.9.2.4) Gecko/20100611 Firefox/3.6.4 (.NET CLR 3.5.30729) -Mozilla/5.0(Windows; U; Windows NT 7.0; rv:1.9.2) Gecko/20100101 Firefox/3.6 +Mozilla/5.0 (Windows; U; Windows NT 7.0; rv:1.9.2) Gecko/20100101 Firefox/3.6 Mozilla/5.0 (Windows; U; WinNT4.0; de-DE; rv:1.7.5) Gecko/20041108 Firefox/1.0 Mozilla/5.0 (Windows; U; WinNT4.0; de-DE; rv:1.7.6) Gecko/20050226 Firefox/1.0.1 Mozilla/5.0 (Windows; U; WinNT4.0; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0 @@ -1385,7 +1381,6 @@ Mozilla/5.0 (X11; Linux i686; rv:21.0) Gecko/20100101 Firefox/21.0 Mozilla/5.0 (X11; Linux i686; rv:6.0) Gecko/20100101 Firefox/6.0 Mozilla/5.0 (X11; Linux i686; U; en; rv:1.8.0) Gecko/20060728 Firefox/1.5.0 Mozilla/5.0 (X11; Linux i686; U; pl; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 -Mozilla/5.0 (X11; Linux ppc; rv:5.0) Gecko/20100101 Firefox/5.0 Mozilla/5.0 (X11; Linux x86_64) Gecko Firefox/5.0 Mozilla/5.0 (X11; Linux x86_64; rv:2.0.1) Gecko/20110506 Firefox/4.0.1 Mozilla/5.0 (X11; Linux x86_64; rv:2.0b4) Gecko/20100818 Firefox/4.0b4 @@ -2209,13 +2204,6 @@ Mozilla/5.0 (X11; U; Linux i686; zh-TW; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 Mozilla/5.0 (X11; U; Linux i686; zh-TW; rv:1.9.0.7) Gecko/2009030422 Ubuntu/8.04 (hardy) Firefox/3.0.7 Mozilla/5.0 (X11; U; Linux ia64; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3 Mozilla/5.0 (X11; U; Linux MIPS32 1074Kf CPS QuadCore; en-US; rv:1.9.2.13) Gecko/20110103 Fedora/3.6.13-1.fc14 Firefox/3.6.13 -Mozilla/5.0 (X11; U; Linux ppc64; en-US; rv:1.8.1.14) Gecko/20080418 Ubuntu/7.10 (gutsy) Firefox/2.0.0.14 -Mozilla/5.0 (X11; U; Linux ppc; da-DK; rv:1.7.12) Gecko/20051010 Firefox/1.0.7 (Ubuntu package 1.0.7) -Mozilla/5.0 (X11; U; Linux ppc; en-GB; rv:1.9.0.12) Gecko/2009070818 Ubuntu/8.10 (intrepid) Firefox/3.0.12 -Mozilla/5.0 (X11; U; Linux ppc; en-US; rv:1.7.12) Gecko/20051222 Firefox/1.0.7 -Mozilla/5.0 (X11; U; Linux ppc; en-US; rv:1.8.1.3) Gecko/20070310 Firefox/2.0.0.3 (Debian-2.0.0.3-1) -Mozilla/5.0 (X11; U; Linux ppc; en-US; rv:1.9.0.4) Gecko/2008111317 Ubuntu/8.04 (hardy) Firefox/3.0.4 -Mozilla/5.0 (X11; U; Linux ppc; fr; rv:1.9.2.12) Gecko/20101027 Ubuntu/10.10 (maverick) Firefox/3.6.12 Mozilla/5.0 (X11; U; Linux sparc64; en-US; rv:1.8.1.17) Gecko/20081108 Firefox/2.0.0.17 Mozilla/5.0 (X11; U; Linux x64_64; es-AR; rv:1.9.0.3) Gecko/2008092515 Ubuntu/8.10 (intrepid) Firefox/3.0.3 Mozilla/5.0 (X11; U; Linux x86_64; cs-CZ; rv:1.9.0.4) Gecko/2008111318 Ubuntu/8.04 (hardy) Firefox/3.0.4 @@ -2547,7 +2535,6 @@ Mozilla/5.0 (X11; U; OpenBSD i386; en-US; rv:1.8.1.6) Gecko/20070819 Firefox/2.0 Mozilla/5.0 (X11; U; OpenBSD i386; en-US; rv:1.8.1.7) Gecko/20070930 Firefox/2.0.0.7 Mozilla/5.0 (X11; U; OpenBSD i386; en-US; rv:1.9.2.20) Gecko/20110803 Firefox/3.6.20 Mozilla/5.0 (X11; U; OpenBSD i386; en-US; rv:1.9.2.8) Gecko/20101230 Firefox/3.6.8 -Mozilla/5.0 (X11; U; OpenBSD ppc; en-US; rv:1.8.0.10) Gecko/20070223 Firefox/1.5.0.10 Mozilla/5.0 (X11; U; OpenBSD sparc64; en-AU; rv:1.8.1.6) Gecko/20071225 Firefox/2.0.0.6 Mozilla/5.0 (X11; U; OpenBSD sparc64; en-CA; rv:1.8.0.2) Gecko/20060429 Firefox/1.5.0.2 Mozilla/5.0 (X11; U; OpenBSD sparc64; en-US; rv:1.8.1.6) Gecko/20070816 Firefox/2.0.0.6 @@ -3452,16 +3439,6 @@ Mozilla/4.0 (compatible; MSIE 4.01; Windows 98; DigExt) Mozilla/4.0 (compatible; MSIE 4.01; Windows 98; Hotbar 3.0) Mozilla/4.0 (compatible; MSIE 4.01; Windows CE) Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; PPC) -Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; PPC; 240x320; PPC) -Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; PPC; 240x320; Sprint:PPC-6700; PPC; 240x320) -Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; Smartphone; 176x220) -Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; Sprint;PPC-i830; PPC; 240x320) -Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; Sprint:PPC-i830; PPC; 240x320) -Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; Sprint:SCH-i320; Smartphone; 176x220) -Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; Sprint; SCH-i830; PPC; 240x320) -Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; Sprint:SCH-i830; PPC; 240x320) -Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; Sprint:SPH-ip320; Smartphone; 176x220) -Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; Sprint:SPH-ip830w; PPC; 240x320) Mozilla/4.0 (compatible; MSIE 4.01; Windows NT) Mozilla/4.0 (compatible; MSIE 4.01; Windows NT 5.0) Mozilla/4.0 (compatible; MSIE 4.0; Windows 95) @@ -3597,7 +3574,6 @@ Mozilla/4.0 (Mozilla/4.0; MSIE 7.0; Windows NT 5.1; FDM; SV1) Mozilla/4.0 (Mozilla/4.0; MSIE 7.0; Windows NT 5.1; FDM; SV1; .NET CLR 3.0.04506.30) Mozilla/4.0 (MSIE 6.0; Windows NT 5.0) Mozilla/4.0 (MSIE 6.0; Windows NT 5.1) -Mozilla/4.0 PPC (compatible; MSIE 4.01; Windows CE; PPC; 240x320; Sprint:PPC-6700; PPC; 240x320) Mozilla/4.0 WebTV/2.6 (compatible; MSIE 4.0) Mozilla/4.0 (Windows; MSIE 6.0; Windows NT 5.0) Mozilla/4.0 (Windows; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727) @@ -3605,8 +3581,6 @@ Mozilla/4.0 (Windows; MSIE 6.0; Windows NT 5.2) Mozilla/4.0 (Windows; MSIE 6.0; Windows NT 6.0) Mozilla/4.0 (Windows; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727) Mozilla/4.0 (X11; MSIE 6.0; i686; .NET CLR 1.1.4322; .NET CLR 2.0.50727; FDM) -Mozilla/45.0 (compatible; MSIE 6.0; Windows NT 5.1) -Mozilla/4.79 [en] (compatible; MSIE 7.0; Windows NT 5.0; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 1.1.4322; .NET CLR 3.0.04506.30; .NET CLR 3.0.04506.648) Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0) Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/4.0; InfoPath.2; SV1; .NET CLR 2.0.50727; WOW64) Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0) @@ -3809,7 +3783,6 @@ Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; sv-se) AppleWebKit/525.18 (KHTM Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; sv-se) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1 Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; tr) AppleWebKit/528.4+ (KHTML, like Gecko) Version/4.0dp1 Safari/526.11.2 Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_2; en) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.1 Safari/525.18 -Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_2; en-gb) AppleWebKit/526+ (KHTML, like Gecko) Version/3.1 iPhone Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_2; en-gb) AppleWebKit/526+ (KHTML, like Gecko) Version/3.1 Safari/525.9 Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_3; en) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.1 Safari/525.20 Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_3; en-us) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.1 Safari/525.20 @@ -4209,4 +4182,4 @@ Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-CN) AppleWebKit/533+ (KHTML, like Ge Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-HK) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5 Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-TW) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10 Mozilla/5.0 (X11; U; Linux x86_64; en-ca) AppleWebKit/531.2+ (KHTML, like Gecko) Version/5.0 Safari/531.2+ -Mozilla/5.0 (X11; U; Linux x86_64; en-us) AppleWebKit/531.2+ (KHTML, like Gecko) Version/5.0 Safari/531.2+ \ No newline at end of file +Mozilla/5.0 (X11; U; Linux x86_64; en-us) AppleWebKit/531.2+ (KHTML, like Gecko) Version/5.0 Safari/531.2+