diff --git a/lib/core/option.py b/lib/core/option.py index 15caebca6..be9700fb9 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -133,7 +133,6 @@ from lib.core.update import update from lib.parse.configfile import configFileParser from lib.parse.payloads import loadBoundaries from lib.parse.payloads import loadPayloads -from lib.parse.sitemap import parseSitemap from lib.request.basic import checkCharEncoding from lib.request.basicauthhandler import SmartHTTPBasicAuthHandler from lib.request.chunkedhandler import ChunkedHandler @@ -338,25 +337,6 @@ def _setCrawler(): if not conf.bulkFile: crawl(conf.url) - else: - targets = getFileItems(conf.bulkFile) - - for i in xrange(len(targets)): - try: - target = targets[i] - - if not re.search(r"(?i)\Ahttp[s]*://", target): - target = "http://%s" % target - - crawl(target) - - if conf.verbose in (1, 2): - status = "%d/%d links visited (%d%%)" % (i + 1, len(targets), round(100.0 * (i + 1) / len(targets))) - dataToStdout("\r[%s] [INFO] %s" % (time.strftime("%X"), status), True) - except Exception as ex: - if not isinstance(ex, SqlmapUserQuitException): - errMsg = "problem occurred while crawling at '%s' ('%s')" % (target, getSafeExString(ex)) - logger.error(errMsg) def _doSearch(): """ @@ -1939,7 +1919,6 @@ def _setKnowledgeBaseAttributes(flushAll=True): kb.mergeCookies = None kb.multipleCtrlC = False kb.negativeLogic = False - kb.normalizeCrawlingChoice = None kb.nullConnection = None kb.oldMsf = None kb.orderByColumns = None @@ -1993,7 +1972,6 @@ def _setKnowledgeBaseAttributes(flushAll=True): kb.reduceTests = None kb.tlsSNI = {} kb.stickyDBMS = False - kb.storeCrawlingChoice = None kb.storeHashesChoice = None kb.suppressResumeInfo = False kb.tableFrom = None @@ -2013,11 +1991,14 @@ def _setKnowledgeBaseAttributes(flushAll=True): kb.xpCmdshellAvailable = False if flushAll: + kb.checkSitemap = None kb.headerPaths = {} kb.keywords = set(getFileItems(paths.SQL_KEYWORDS)) + kb.normalizeCrawlingChoice = None kb.passwordMgr = None kb.preprocessFunctions = [] kb.skipVulnHost = None + kb.storeCrawlingChoice = None kb.tamperFunctions = [] kb.targets = OrderedSet() kb.testedParams = set() diff --git a/lib/core/settings.py b/lib/core/settings.py index be57ca47c..fb252adb9 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -18,7 +18,7 @@ from lib.core.enums import OS from thirdparty.six import unichr as _unichr # sqlmap version (...) -VERSION = "1.3.11.4" +VERSION = "1.3.11.5" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/utils/crawler.py b/lib/utils/crawler.py index 2ca95da71..35932ea7e 100644 --- a/lib/utils/crawler.py +++ b/lib/utils/crawler.py @@ -133,10 +133,12 @@ def crawl(target): threadData.shared.deeper = set() threadData.shared.unprocessed = set([target]) - message = "do you want to check for the existence of " - message += "site's sitemap(.xml) [y/N] " + if kb.checkSitemap is None: + message = "do you want to check for the existence of " + message += "site's sitemap(.xml) [y/N] " + kb.checkSitemap = readInput(message, default='N', boolean=True) - if readInput(message, default='N', boolean=True): + if kb.checkSitemap: found = True items = None url = _urllib.parse.urljoin(target, "/sitemap.xml") @@ -158,10 +160,9 @@ def crawl(target): threadData.shared.unprocessed.update(items) logger.info("%s links found" % ("no" if not items else len(items))) - infoMsg = "starting crawler" - if conf.bulkFile: - infoMsg += " for target URL '%s'" % target - logger.info(infoMsg) + if not conf.bulkFile: + infoMsg = "starting crawler for target URL '%s'" % target + logger.info(infoMsg) for i in xrange(conf.crawlDepth): threadData.shared.count = 0 diff --git a/sqlmap.py b/sqlmap.py index f3b7744bc..de58153f1 100755 --- a/sqlmap.py +++ b/sqlmap.py @@ -45,6 +45,7 @@ try: from lib.core.common import dataToStdout from lib.core.common import filterNone from lib.core.common import getDaysFromLastUpdate + from lib.core.common import getFileItems from lib.core.common import getSafeExString from lib.core.common import maskSensitiveData from lib.core.common import openFile @@ -57,6 +58,7 @@ try: from lib.core.common import MKSTEMP_PREFIX from lib.core.common import setColor from lib.core.common import unhandledExceptionMessage + from lib.core.compat import xrange from lib.core.exception import SqlmapBaseException from lib.core.exception import SqlmapShellQuitException from lib.core.exception import SqlmapSilentQuitException @@ -73,6 +75,7 @@ try: from lib.core.settings import UNICODE_ENCODING from lib.core.settings import VERSION from lib.parse.cmdline import cmdLineParser + from lib.utils.crawler import crawl from thirdparty import six except KeyboardInterrupt: errMsg = "user aborted" @@ -177,7 +180,30 @@ def main(): profile() else: try: - start() + if conf.crawlDepth and conf.bulkFile: + targets = getFileItems(conf.bulkFile) + + for i in xrange(len(targets)): + try: + kb.targets.clear() + target = targets[i] + + if not re.search(r"(?i)\Ahttp[s]*://", target): + target = "http://%s" % target + + infoMsg = "starting crawler for target URL '%s' (%d/%d)" % (target, i + 1, len(targets)) + logger.info(infoMsg) + + crawl(target) + except Exception as ex: + if not isinstance(ex, SqlmapUserQuitException): + errMsg = "problem occurred while crawling '%s' ('%s')" % (target, getSafeExString(ex)) + logger.error(errMsg) + else: + if kb.targets: + start() + else: + start() except Exception as ex: os._exitcode = 1