From b8ffcf949520682c5c8e8fe977ae91da71d78877 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 4 Jul 2011 19:58:41 +0000 Subject: [PATCH] few fixes here and there and multi-core processing for dictionary based hash attack --- extra/odict/odict.py | 7 +- lib/controller/controller.py | 27 ++-- lib/core/option.py | 4 +- lib/core/settings.py | 8 +- lib/core/threads.py | 13 +- lib/techniques/brute/use.py | 22 +++- lib/utils/hash.py | 243 ++++++++++++++++++++++++----------- plugins/generic/misc.py | 2 +- 8 files changed, 225 insertions(+), 101 deletions(-) diff --git a/extra/odict/odict.py b/extra/odict/odict.py index 18bf9c295..9a712b048 100644 --- a/extra/odict/odict.py +++ b/extra/odict/odict.py @@ -33,7 +33,7 @@ if INTP_VER < (2, 2): import types, warnings -class OrderedDict(dict): +class _OrderedDict(dict): """ A class of dictionary that keeps the insertion order of keys. @@ -869,6 +869,11 @@ class OrderedDict(dict): """ self._sequence.sort(*args, **kwargs) +if INTP_VER >= (2, 7): + from collections import OrderedDict +else: + OrderedDict = _OrderedDict + class Keys(object): # FIXME: should this object be a subclass of list? """ diff --git a/lib/controller/controller.py b/lib/controller/controller.py index 8714a69a6..61af51089 100644 --- a/lib/controller/controller.py +++ b/lib/controller/controller.py @@ -60,17 +60,22 @@ def __selectInjection(): Selection function for injection place, parameters and type. """ - points = [] + points = {} - for i in xrange(0, len(kb.injections)): - place = kb.injections[i].place - parameter = kb.injections[i].parameter - ptype = kb.injections[i].ptype + for injection in kb.injections: + place = injection.place + parameter = injection.parameter + ptype = injection.ptype point = (place, parameter, ptype) if point not in points: - points.append(point) + points[point] = injection + else: + for key in points[point].keys(): + if key != 'data': + points[point][key] = points[point][key] or injection[key] + points[point]['data'].update(injection['data']) if len(points) == 1: kb.injection = kb.injections[0] @@ -126,19 +131,11 @@ def __formatInjection(inj): def __showInjections(): header = "sqlmap identified the following injection points with " header += "a total of %d HTTP(s) requests" % kb.testQueryCount - data = "" - for inj in kb.injections: - data += __formatInjection(inj) - - data = data.rstrip("\n") + data = "".join(set(map(lambda x: __formatInjection(x), kb.injections))).rstrip("\n") conf.dumper.technic(header, data) - if inj.place in (HTTPMETHOD.GET, HTTPMETHOD.POST): - debugMsg = "usage of %s payloads requires manual url-encoding" % inj.place - logger.debug(debugMsg) - if conf.tamper: infoMsg = "changes made by tampering scripts are not " infoMsg += "included in shown payload content(s)" diff --git a/lib/core/option.py b/lib/core/option.py index beb1fa386..a0383d6b6 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -1453,7 +1453,6 @@ def __setKnowledgeBaseAttributes(flushAll=True): kb.testQueryCount = 0 kb.threadContinue = True kb.threadException = False - kb.threadData = {} kb.uChar = "NULL" kb.xpCmdshellAvailable = False @@ -1650,6 +1649,9 @@ def __mergeOptions(inputOptions, overrideOptions): conf[key] = value def __setTrafficOutputFP(): + infoMsg = "setting file for logging HTTP traffic" + logger.info(infoMsg) + if conf.trafficFile: conf.trafficFP = openFile(conf.trafficFile, "w+") diff --git a/lib/core/settings.py b/lib/core/settings.py index 263d23034..ae301ede1 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -300,7 +300,7 @@ MYSQL_ERROR_CHUNK_LENGTH = 50 MSSQL_ERROR_CHUNK_LENGTH = 100 # Do not unescape the injected statement if it contains any of the following SQL words -EXCLUDE_UNESCAPE = ("WAITFOR DELAY ", " INTO DUMPFILE ", " INTO OUTFILE ", "CREATE ", "BULK ", "EXEC ", "RECONFIGURE ", "DECLARE ", CHAR_INFERENCE_MARK) +EXCLUDE_UNESCAPE = ("WAITFOR DELAY ", " INTO DUMPFILE ", " INTO OUTFILE ", "CREATE ", "BULK ", "EXEC ", "RECONFIGURE ", "DECLARE ", "'%s'" % CHAR_INFERENCE_MARK) # Mark used for replacement of reflected values REFLECTED_VALUE_MARKER = '__REFLECTED_VALUE__' @@ -364,3 +364,9 @@ DUMMY_SQL_INJECTION_CHARS = ";()\"'" # Extensions skipped by crawler CRAWL_EXCLUDE_EXTENSIONS = ("gif","jpg","jar","tif","bmp","war","ear","mpg","wmv","mpeg","scm","iso","dmp","dll","cab","so","avi","bin","exe","iso","tar","png","pdf","ps","mp3","zip","rar","gz") + +# Template used for common table existence check +BRUTE_TABLE_EXISTS_TEMPLATE = "EXISTS(SELECT %d FROM %s)" + +# Template used for common column existence check +BRUTE_COLUMN_EXISTS_TEMPLATE = "EXISTS(SELECT %s FROM %s)" diff --git a/lib/core/threads.py b/lib/core/threads.py index 4151ef259..0266c14b5 100644 --- a/lib/core/threads.py +++ b/lib/core/threads.py @@ -25,7 +25,7 @@ from lib.core.settings import PYVERSION shared = advancedDict() -class ThreadData(): +class _ThreadData(threading.local): """ Represents thread independent data """ @@ -44,6 +44,8 @@ class ThreadData(): self.shared = shared self.valueStack = [] +ThreadData = _ThreadData() + def getCurrentThreadUID(): return hash(threading.currentThread()) @@ -52,13 +54,12 @@ def readInput(message, default=None): def getCurrentThreadData(): """ - Returns current thread's dependent data + Returns current thread's local data """ - threadUID = getCurrentThreadUID() - if threadUID not in kb.threadData: - kb.threadData[threadUID] = ThreadData() - return kb.threadData[threadUID] + global ThreadData + + return ThreadData def exceptionHandledFunction(threadFunction): try: diff --git a/lib/techniques/brute/use.py b/lib/techniques/brute/use.py index f056438ef..f87ef0d0a 100644 --- a/lib/techniques/brute/use.py +++ b/lib/techniques/brute/use.py @@ -20,6 +20,7 @@ from lib.core.common import getPageWordSet from lib.core.common import popValue from lib.core.common import pushValue from lib.core.common import randomInt +from lib.core.common import randomStr from lib.core.common import readInput from lib.core.common import safeStringFormat from lib.core.common import safeSQLIdentificatorNaming @@ -27,10 +28,13 @@ from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger from lib.core.enums import DBMS +from lib.core.exception import sqlmapDataException from lib.core.exception import sqlmapMissingMandatoryOptionException from lib.core.exception import sqlmapThreadException from lib.core.settings import MAX_NUMBER_OF_THREADS from lib.core.settings import METADB_SUFFIX +from lib.core.settings import BRUTE_COLUMN_EXISTS_TEMPLATE +from lib.core.settings import BRUTE_TABLE_EXISTS_TEMPLATE from lib.core.session import safeFormatString from lib.core.threads import getCurrentThreadData from lib.core.threads import runThreads @@ -52,6 +56,13 @@ def __addPageTextWords(): return wordsList def tableExists(tableFile, regex=None): + result = inject.checkBooleanExpression("%s" % safeStringFormat(BRUTE_TABLE_EXISTS_TEMPLATE, (randomInt(1), randomStr()))) + if result: + errMsg = "can't use table existence check because of detected invalid results " + errMsg += "(most probably caused by inability of the used injection " + errMsg += "to distinguish errornous results)" + raise sqlmapDataException, errMsg + tables = getFileItems(tableFile, lowercase=Backend.getIdentifiedDbms() in (DBMS.ACCESS), unique=True) infoMsg = "checking table existence using items from '%s'" % tableFile @@ -84,7 +95,7 @@ def tableExists(tableFile, regex=None): else: fullTableName = table - result = inject.checkBooleanExpression("%s" % safeStringFormat("EXISTS(SELECT %d FROM %s)", (randomInt(1), fullTableName))) + result = inject.checkBooleanExpression("%s" % safeStringFormat(BRUTE_TABLE_EXISTS_TEMPLATE, (randomInt(1), fullTableName))) kb.locks.ioLock.acquire() @@ -135,6 +146,13 @@ def columnExists(columnFile, regex=None): errMsg = "missing table parameter" raise sqlmapMissingMandatoryOptionException, errMsg + result = inject.checkBooleanExpression(safeStringFormat(BRUTE_COLUMN_EXISTS_TEMPLATE, (randomStr(), randomStr()))) + if result: + errMsg = "can't use column existence check because of detected invalid results " + errMsg += "(most probably caused by inability of the used injection " + errMsg += "to distinguish errornous results)" + raise sqlmapDataException, errMsg + infoMsg = "checking column existence using items from '%s'" % columnFile logger.info(infoMsg) @@ -169,7 +187,7 @@ def columnExists(columnFile, regex=None): kb.locks.countLock.release() break - result = inject.checkBooleanExpression("%s" % safeStringFormat("EXISTS(SELECT %s FROM %s)", (column, table))) + result = inject.checkBooleanExpression(safeStringFormat(BRUTE_COLUMN_EXISTS_TEMPLATE, (column, table))) kb.locks.ioLock.acquire() diff --git a/lib/utils/hash.py b/lib/utils/hash.py index 57f1577b1..e3de4c8ae 100644 --- a/lib/utils/hash.py +++ b/lib/utils/hash.py @@ -19,6 +19,7 @@ import time from hashlib import md5 from hashlib import sha1 +from Queue import Queue from zipfile import ZipFile from extra.pydes.pyDes import des @@ -35,6 +36,7 @@ from lib.core.common import normalizeUnicode from lib.core.common import paths from lib.core.common import readInput from lib.core.common import singleTimeLogMessage +from lib.core.common import singleTimeWarnMessage from lib.core.common import Wordlist from lib.core.convert import hexdecode from lib.core.convert import hexencode @@ -50,9 +52,13 @@ from lib.core.settings import DUMMY_USER_PREFIX from lib.core.settings import GENERAL_IP_ADDRESS_REGEX from lib.core.settings import HASH_MOD_ITEM_DISPLAY from lib.core.settings import IS_WIN +from lib.core.settings import PYVERSION from lib.core.settings import ML from lib.core.settings import UNICODE_ENCODING +if PYVERSION >= "2.6": + import multiprocessing + def mysql_passwd(password, uppercase=True): """ Reference(s): @@ -320,6 +326,7 @@ def dictionaryAttack(attack_dict): suffix_list = [""] hash_regexes = [] results = [] + processException = False for (_, hashes) in attack_dict.items(): for hash_ in hashes: @@ -421,10 +428,8 @@ def dictionaryAttack(attack_dict): kb.wordlist.append(normalizeUnicode(user)) if hash_regex in (HASH.MYSQL, HASH.MYSQL_OLD, HASH.MD5_GENERIC, HASH.SHA1_GENERIC): - count = 0 - for suffix in suffix_list: - if not attack_info: + if not attack_info or processException: break if suffix: @@ -434,53 +439,91 @@ def dictionaryAttack(attack_dict): kb.wordlist.rewind() - for word in kb.wordlist: - if not attack_info: - break - - count += 1 - - if not isinstance(word, basestring): - continue - - if suffix: - word = word + suffix + def bruteProcess(attack_info, hash_regex, wordlist, suffix, retVal, proc_id, proc_count): + count = 0 try: - current = __functions__[hash_regex](password = word, uppercase = False) + for word in kb.wordlist: + if not attack_info: + break - for item in attack_info: - ((user, hash_), _) = item + count += 1 - if hash_ == current: - results.append((user, hash_, word)) - clearConsoleLine() + if not isinstance(word, basestring): + continue - infoMsg = "[%s] [INFO] found: '%s'" % (time.strftime("%X"), word) + if suffix: + word = word + suffix - if user and not user.startswith(DUMMY_USER_PREFIX): - infoMsg += " for user '%s'\n" % user - else: - infoMsg += " for hash '%s'\n" % hash_ + try: + current = __functions__[hash_regex](password = word, uppercase = False) - dataToStdout(infoMsg, True) + for item in attack_info: + ((user, hash_), _) = item - attack_info.remove(item) + if hash_ == current: + retVal.put((user, hash_, word)) - elif count % HASH_MOD_ITEM_DISPLAY == 0 or hash_regex in (HASH.ORACLE_OLD) or hash_regex == HASH.CRYPT_GENERIC and IS_WIN: - status = 'current status: %d%s (%s...)' % (kb.wordlist.percentage(), '%', word.ljust(5)[:8]) - dataToStdout("\r[%s] [INFO] %s" % (time.strftime("%X"), status)) + clearConsoleLine() + + infoMsg = "[%s] [INFO] found: '%s'" % (time.strftime("%X"), word) + + if user and not user.startswith(DUMMY_USER_PREFIX): + infoMsg += " for user '%s'\n" % user + else: + infoMsg += " for hash '%s'\n" % hash_ + + dataToStdout(infoMsg, True) + + attack_info.remove(item) + + elif proc_id == 0 and count % HASH_MOD_ITEM_DISPLAY == 0 or hash_regex in (HASH.ORACLE_OLD) or hash_regex == HASH.CRYPT_GENERIC and IS_WIN: + status = 'current status: %d%s (%s...)' % (proc_count * kb.wordlist.percentage(), '%', word.ljust(5)[:5]) + dataToStdout("\r[%s] [INFO] %s" % (time.strftime("%X"), status)) + + except KeyboardInterrupt: + raise + + except: + warnMsg = "there was a problem while hashing entry: %s. " % repr(word) + warnMsg += "Please report by e-mail to %s." % ML + logger.critical(warnMsg) except KeyboardInterrupt: - print - warnMsg = "user aborted during dictionary attack phase" - logger.warn(warnMsg) - return results + pass - except: - warnMsg = "there was a problem while hashing entry: %s. " % repr(word) - warnMsg += "Please report by e-mail to %s." % ML - logger.critical(warnMsg) + retVal = None + + try: + if PYVERSION >= "2.6": + infoMsg = "starting %d hash attack processes " % multiprocessing.cpu_count() + singleTimeLogMessage(infoMsg) + + processes = [] + retVal = multiprocessing.Queue() + for i in xrange(multiprocessing.cpu_count()): + p = multiprocessing.Process(target=bruteProcess, args=(attack_info, hash_regex, kb.wordlist, suffix, retVal, i, multiprocessing.cpu_count())) + p.start() + processes.append(p) + + for p in processes: + p.join() + + else: + warnMsg = "multiprocessing not supported on current version of " + warnMsg += "Python (%s < 2.6)" % PYVERSION + singleTimeWarnMessage(warnMsg) + + retVal = Queue() + bruteProcess(attack_info, hash_regex, kb.wordlist, suffix, retVal, 0, 1) + + except KeyboardInterrupt: + print + processException = True + warnMsg = "user aborted during dictionary attack phase" + logger.warn(warnMsg) + + results = [retVal.get() for i in xrange(retVal.qsize())] if retVal else [] clearConsoleLine() @@ -490,7 +533,7 @@ def dictionaryAttack(attack_dict): found = False for suffix in suffix_list: - if found: + if found or processException: break if suffix: @@ -500,50 +543,102 @@ def dictionaryAttack(attack_dict): kb.wordlist.rewind() - for word in kb.wordlist: - current = __functions__[hash_regex](password = word, uppercase = False, **kwargs) - count += 1 - - if not isinstance(word, basestring): - continue - - if suffix: - word = word + suffix + def bruteProcess(user, hash_, kwargs, hash_regex, wordlist, suffix, retVal, found, proc_id, proc_count): + count = 0 try: - if hash_ == current: - if regex == HASH.ORACLE_OLD: #only for cosmetic purposes - word = word.upper() - results.append((user, hash_, word)) - clearConsoleLine() + for word in kb.wordlist: - infoMsg = "[%s] [INFO] found: '%s'" % (time.strftime("%X"), word) + current = __functions__[hash_regex](password = word, uppercase = False, **kwargs) + count += 1 - if user and not user.startswith(DUMMY_USER_PREFIX): - infoMsg += " for user '%s'\n" % user - else: - infoMsg += " for hash '%s'\n" % hash_ + if not isinstance(word, basestring): + continue - dataToStdout(infoMsg, True) + if suffix: + word = word + suffix - found = True - break - elif count % HASH_MOD_ITEM_DISPLAY == 0 or hash_regex in (HASH.ORACLE_OLD) or hash_regex == HASH.CRYPT_GENERIC and IS_WIN: - status = 'current status: %d%s (%s...)' % (kb.wordlist.percentage(), '%', word.ljust(5)[:5]) - if not user.startswith(DUMMY_USER_PREFIX): - status += ' (user: %s)' % user - dataToStdout("\r[%s] [INFO] %s" % (time.strftime("%X"), status)) + try: + if hash_ == current: + if regex == HASH.ORACLE_OLD: #only for cosmetic purposes + word = word.upper() + + retVal.put((user, hash_, word)) + + clearConsoleLine() + + infoMsg = "[%s] [INFO] found: '%s'" % (time.strftime("%X"), word) + + if user and not user.startswith(DUMMY_USER_PREFIX): + infoMsg += " for user '%s'\n" % user + else: + infoMsg += " for hash '%s'\n" % hash_ + + dataToStdout(infoMsg, True) + + found.value = True + break + elif proc_id == 0 and count % HASH_MOD_ITEM_DISPLAY == 0 or hash_regex in (HASH.ORACLE_OLD) or hash_regex == HASH.CRYPT_GENERIC and IS_WIN: + status = 'current status: %d%s (%s...)' % (proc_count * kb.wordlist.percentage(), '%', word.ljust(5)[:5]) + if not user.startswith(DUMMY_USER_PREFIX): + status += ' (user: %s)' % user + dataToStdout("\r[%s] [INFO] %s" % (time.strftime("%X"), status)) + + except KeyboardInterrupt: + raise + + except: + warnMsg = "there was a problem while hashing entry: %s. " % repr(word) + warnMsg += "Please report by e-mail to %s." % ML + logger.critical(warnMsg) except KeyboardInterrupt: - print - warnMsg = "user aborted during dictionary attack phase" - logger.warn(warnMsg) - return results + pass - except: - warnMsg = "there was a problem while hashing entry: %s. " % repr(word) - warnMsg += "Please report by e-mail to %s." % ML - logger.critical(warnMsg) + retVal = None + + try: + if PYVERSION >= "2.6": + infoMsg = "starting %d hash attack processes " % multiprocessing.cpu_count() + singleTimeLogMessage(infoMsg) + + processes = [] + retVal = multiprocessing.Queue() + found_ = multiprocessing.Value('i', False) + + for i in xrange(multiprocessing.cpu_count()): + p = multiprocessing.Process(target=bruteProcess, args=(user, hash_, kwargs, hash_regex, kb.wordlist, suffix, retVal, found_, i, multiprocessing.cpu_count())) + p.start() + processes.append(p) + + for p in processes: + p.join() + + found = found_.value != 0 + + else: + warnMsg = "multiprocessing not supported on current version of " + warnMsg += "Python (%s < 2.6)" % PYVERSION + singleTimeWarnMessage(warnMsg) + + class Value(): + pass + + retVal = Queue() + found_ = Value() + found_.value = False + + bruteProcess(user, hash_, kwargs, hash_regex, kb.wordlist, suffix, retVal, found_, 0, 1) + + found = found_.value + + except KeyboardInterrupt: + print + processException = True + warnMsg = "user aborted during dictionary attack phase" + logger.warn(warnMsg) + + results = [retVal.get() for i in xrange(retVal.qsize())] if retVal else [] clearConsoleLine() diff --git a/plugins/generic/misc.py b/plugins/generic/misc.py index c8c6edc64..b1480d503 100644 --- a/plugins/generic/misc.py +++ b/plugins/generic/misc.py @@ -164,7 +164,7 @@ class Miscellaneous: if not choice or choice == "1": choice = "1" condParam = " LIKE '%%%s%%'" - elif choice.isdigit() and choice == "2": + elif choice == "2": condParam = "='%s'" else: errMsg = "invalid value"