few fixes here and there and multi-core processing for dictionary based hash attack

extra/odict/odict.py

@@ -33,7 +33,7 @@ if INTP_VER < (2, 2):
 
 import types, warnings
 
-class OrderedDict(dict):
+class _OrderedDict(dict):
     """
     A class of dictionary that keeps the insertion order of keys.
 
@@ -869,6 +869,11 @@ class OrderedDict(dict):
         """
         self._sequence.sort(*args, **kwargs)
 
+if INTP_VER >= (2, 7):
+    from collections import OrderedDict
+else:
+    OrderedDict = _OrderedDict
+
 class Keys(object):
     # FIXME: should this object be a subclass of list?
     """

lib/controller/controller.py

@@ -60,17 +60,22 @@ def __selectInjection():
     Selection function for injection place, parameters and type.
     """
 
-    points = []
+    points = {}
 
-    for i in xrange(0, len(kb.injections)):
+    for injection in kb.injections:
-        place = kb.injections[i].place
+        place = injection.place
-        parameter = kb.injections[i].parameter
+        parameter = injection.parameter
-        ptype = kb.injections[i].ptype
+        ptype = injection.ptype
 
         point = (place, parameter, ptype)
 
         if point not in points:
-            points.append(point)
+            points[point] = injection
+        else:
+            for key in points[point].keys():
+                if key != 'data':
+                    points[point][key] = points[point][key] or injection[key]
+            points[point]['data'].update(injection['data'])
 
     if len(points) == 1:
         kb.injection = kb.injections[0]
@@ -126,19 +131,11 @@ def __formatInjection(inj):
 def __showInjections():
     header = "sqlmap identified the following injection points with "
     header += "a total of %d HTTP(s) requests" % kb.testQueryCount
-    data = ""
 
-    for inj in kb.injections:
+    data = "".join(set(map(lambda x: __formatInjection(x), kb.injections))).rstrip("\n")
-        data += __formatInjection(inj)
-
-    data = data.rstrip("\n")
 
     conf.dumper.technic(header, data)
 
-    if inj.place in (HTTPMETHOD.GET, HTTPMETHOD.POST):
-        debugMsg = "usage of %s payloads requires manual url-encoding" % inj.place
-        logger.debug(debugMsg)
-
     if conf.tamper:
         infoMsg = "changes made by tampering scripts are not "
         infoMsg += "included in shown payload content(s)"

lib/core/option.py

@@ -1453,7 +1453,6 @@ def __setKnowledgeBaseAttributes(flushAll=True):
     kb.testQueryCount = 0
     kb.threadContinue = True
     kb.threadException = False
-    kb.threadData = {}
     kb.uChar = "NULL"
     kb.xpCmdshellAvailable = False
 
@@ -1650,6 +1649,9 @@ def __mergeOptions(inputOptions, overrideOptions):
             conf[key] = value
 
 def __setTrafficOutputFP():
+    infoMsg = "setting file for logging HTTP traffic"
+    logger.info(infoMsg)
+
     if conf.trafficFile:
         conf.trafficFP = openFile(conf.trafficFile, "w+")
 

lib/core/settings.py

@@ -300,7 +300,7 @@ MYSQL_ERROR_CHUNK_LENGTH = 50
 MSSQL_ERROR_CHUNK_LENGTH = 100
 
 # Do not unescape the injected statement if it contains any of the following SQL words
-EXCLUDE_UNESCAPE = ("WAITFOR DELAY ", " INTO DUMPFILE ", " INTO OUTFILE ", "CREATE ", "BULK ", "EXEC ", "RECONFIGURE ", "DECLARE ", CHAR_INFERENCE_MARK)
+EXCLUDE_UNESCAPE = ("WAITFOR DELAY ", " INTO DUMPFILE ", " INTO OUTFILE ", "CREATE ", "BULK ", "EXEC ", "RECONFIGURE ", "DECLARE ", "'%s'" % CHAR_INFERENCE_MARK)
 
 # Mark used for replacement of reflected values
 REFLECTED_VALUE_MARKER = '__REFLECTED_VALUE__'
@@ -364,3 +364,9 @@ DUMMY_SQL_INJECTION_CHARS = ";()\"'"
 
 # Extensions skipped by crawler
 CRAWL_EXCLUDE_EXTENSIONS = ("gif","jpg","jar","tif","bmp","war","ear","mpg","wmv","mpeg","scm","iso","dmp","dll","cab","so","avi","bin","exe","iso","tar","png","pdf","ps","mp3","zip","rar","gz")
+
+# Template used for common table existence check
+BRUTE_TABLE_EXISTS_TEMPLATE = "EXISTS(SELECT %d FROM %s)"
+
+# Template used for common column existence check
+BRUTE_COLUMN_EXISTS_TEMPLATE = "EXISTS(SELECT %s FROM %s)"

lib/core/threads.py

@@ -25,7 +25,7 @@ from lib.core.settings import PYVERSION
 
 shared = advancedDict()
 
-class ThreadData():
+class _ThreadData(threading.local):
     """
     Represents thread independent data
     """
@@ -44,6 +44,8 @@ class ThreadData():
         self.shared = shared
         self.valueStack = []
 
+ThreadData = _ThreadData()
+
 def getCurrentThreadUID():
     return hash(threading.currentThread())
 
@@ -52,13 +54,12 @@ def readInput(message, default=None):
 
 def getCurrentThreadData():
     """
-    Returns current thread's dependent data
+    Returns current thread's local data
     """
 
-    threadUID = getCurrentThreadUID()
+    global ThreadData
-    if threadUID not in kb.threadData:
+
-        kb.threadData[threadUID] = ThreadData()
+    return ThreadData
-    return kb.threadData[threadUID]
 
 def exceptionHandledFunction(threadFunction):
     try:

lib/techniques/brute/use.py

@@ -20,6 +20,7 @@ from lib.core.common import getPageWordSet
 from lib.core.common import popValue
 from lib.core.common import pushValue
 from lib.core.common import randomInt
+from lib.core.common import randomStr
 from lib.core.common import readInput
 from lib.core.common import safeStringFormat
 from lib.core.common import safeSQLIdentificatorNaming
@@ -27,10 +28,13 @@ from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.enums import DBMS
+from lib.core.exception import sqlmapDataException
 from lib.core.exception import sqlmapMissingMandatoryOptionException
 from lib.core.exception import sqlmapThreadException
 from lib.core.settings import MAX_NUMBER_OF_THREADS
 from lib.core.settings import METADB_SUFFIX
+from lib.core.settings import BRUTE_COLUMN_EXISTS_TEMPLATE
+from lib.core.settings import BRUTE_TABLE_EXISTS_TEMPLATE
 from lib.core.session import safeFormatString
 from lib.core.threads import getCurrentThreadData
 from lib.core.threads import runThreads
@@ -52,6 +56,13 @@ def __addPageTextWords():
     return wordsList
 
 def tableExists(tableFile, regex=None):
+    result = inject.checkBooleanExpression("%s" % safeStringFormat(BRUTE_TABLE_EXISTS_TEMPLATE, (randomInt(1), randomStr())))
+    if result:
+        errMsg = "can't use table existence check because of detected invalid results "
+        errMsg += "(most probably caused by inability of the used injection "
+        errMsg += "to distinguish errornous results)"
+        raise sqlmapDataException, errMsg
+
     tables = getFileItems(tableFile, lowercase=Backend.getIdentifiedDbms() in (DBMS.ACCESS), unique=True)
 
     infoMsg = "checking table existence using items from '%s'" % tableFile
@@ -84,7 +95,7 @@ def tableExists(tableFile, regex=None):
             else:
                 fullTableName = table
 
-            result = inject.checkBooleanExpression("%s" % safeStringFormat("EXISTS(SELECT %d FROM %s)", (randomInt(1), fullTableName)))
+            result = inject.checkBooleanExpression("%s" % safeStringFormat(BRUTE_TABLE_EXISTS_TEMPLATE, (randomInt(1), fullTableName)))
 
             kb.locks.ioLock.acquire()
 
@@ -135,6 +146,13 @@ def columnExists(columnFile, regex=None):
         errMsg = "missing table parameter"
         raise sqlmapMissingMandatoryOptionException, errMsg
 
+    result = inject.checkBooleanExpression(safeStringFormat(BRUTE_COLUMN_EXISTS_TEMPLATE, (randomStr(), randomStr())))
+    if result:
+        errMsg = "can't use column existence check because of detected invalid results "
+        errMsg += "(most probably caused by inability of the used injection "
+        errMsg += "to distinguish errornous results)"
+        raise sqlmapDataException, errMsg
+
     infoMsg = "checking column existence using items from '%s'" % columnFile
     logger.info(infoMsg)
 
@@ -169,7 +187,7 @@ def columnExists(columnFile, regex=None):
                 kb.locks.countLock.release()
                 break
 
-            result = inject.checkBooleanExpression("%s" % safeStringFormat("EXISTS(SELECT %s FROM %s)", (column, table)))
+            result = inject.checkBooleanExpression(safeStringFormat(BRUTE_COLUMN_EXISTS_TEMPLATE, (column, table)))
 
             kb.locks.ioLock.acquire()
 

lib/utils/hash.py

@@ -19,6 +19,7 @@ import time
 
 from hashlib import md5
 from hashlib import sha1
+from Queue import Queue
 from zipfile import ZipFile
 
 from extra.pydes.pyDes import des
@@ -35,6 +36,7 @@ from lib.core.common import normalizeUnicode
 from lib.core.common import paths
 from lib.core.common import readInput
 from lib.core.common import singleTimeLogMessage
+from lib.core.common import singleTimeWarnMessage
 from lib.core.common import Wordlist
 from lib.core.convert import hexdecode
 from lib.core.convert import hexencode
@@ -50,9 +52,13 @@ from lib.core.settings import DUMMY_USER_PREFIX
 from lib.core.settings import GENERAL_IP_ADDRESS_REGEX
 from lib.core.settings import HASH_MOD_ITEM_DISPLAY
 from lib.core.settings import IS_WIN
+from lib.core.settings import PYVERSION
 from lib.core.settings import ML
 from lib.core.settings import UNICODE_ENCODING
 
+if PYVERSION >= "2.6":
+    import multiprocessing
+
 def mysql_passwd(password, uppercase=True):
     """
     Reference(s):
@@ -320,6 +326,7 @@ def dictionaryAttack(attack_dict):
     suffix_list = [""]
     hash_regexes = []
     results = []
+    processException = False
 
     for (_, hashes) in attack_dict.items():
         for hash_ in hashes:
@@ -421,10 +428,8 @@ def dictionaryAttack(attack_dict):
                 kb.wordlist.append(normalizeUnicode(user))
 
         if hash_regex in (HASH.MYSQL, HASH.MYSQL_OLD, HASH.MD5_GENERIC, HASH.SHA1_GENERIC):
-            count = 0
-
             for suffix in suffix_list:
-                if not attack_info:
+                if not attack_info or processException:
                     break
 
                 if suffix:
@@ -434,53 +439,91 @@ def dictionaryAttack(attack_dict):
 
                 kb.wordlist.rewind()
 
-                for word in kb.wordlist:
+                def bruteProcess(attack_info, hash_regex, wordlist, suffix, retVal, proc_id, proc_count):
-                    if not attack_info:
+                    count = 0
-                        break
-
-                    count += 1
-
-                    if not isinstance(word, basestring):
-                        continue
-
-                    if suffix:
-                        word = word + suffix
 
                     try:
-                        current = __functions__[hash_regex](password = word, uppercase = False)
+                        for word in kb.wordlist:
+                            if not attack_info:
+                                break
 
-                        for item in attack_info:
+                            count += 1
-                            ((user, hash_), _) = item
 
-                            if hash_ == current:
+                            if not isinstance(word, basestring):
-                                results.append((user, hash_, word))
+                                continue
-                                clearConsoleLine()
 
-                                infoMsg = "[%s] [INFO] found: '%s'" % (time.strftime("%X"), word)
+                            if suffix:
+                                word = word + suffix
 
-                                if user and not user.startswith(DUMMY_USER_PREFIX):
+                            try:
-                                    infoMsg += " for user '%s'\n" % user
+                                current = __functions__[hash_regex](password = word, uppercase = False)
-                                else:
-                                    infoMsg += " for hash '%s'\n" % hash_
 
-                                dataToStdout(infoMsg, True)
+                                for item in attack_info:
+                                    ((user, hash_), _) = item
 
-                                attack_info.remove(item)
+                                    if hash_ == current:
+                                        retVal.put((user, hash_, word))
 
-                            elif count % HASH_MOD_ITEM_DISPLAY == 0 or hash_regex in (HASH.ORACLE_OLD) or hash_regex == HASH.CRYPT_GENERIC and IS_WIN:
+                                        clearConsoleLine()
-                                status = 'current status: %d%s (%s...)' % (kb.wordlist.percentage(), '%', word.ljust(5)[:8])
+
-                                dataToStdout("\r[%s] [INFO] %s" % (time.strftime("%X"), status))
+                                        infoMsg = "[%s] [INFO] found: '%s'" % (time.strftime("%X"), word)
+
+                                        if user and not user.startswith(DUMMY_USER_PREFIX):
+                                            infoMsg += " for user '%s'\n" % user
+                                        else:
+                                            infoMsg += " for hash '%s'\n" % hash_
+
+                                        dataToStdout(infoMsg, True)
+
+                                        attack_info.remove(item)
+
+                                    elif proc_id == 0 and count % HASH_MOD_ITEM_DISPLAY == 0 or hash_regex in (HASH.ORACLE_OLD) or hash_regex == HASH.CRYPT_GENERIC and IS_WIN:
+                                        status = 'current status: %d%s (%s...)' % (proc_count * kb.wordlist.percentage(), '%', word.ljust(5)[:5])
+                                        dataToStdout("\r[%s] [INFO] %s" % (time.strftime("%X"), status))
+
+                            except KeyboardInterrupt:
+                                raise
+
+                            except:
+                                warnMsg = "there was a problem while hashing entry: %s. " % repr(word)
+                                warnMsg += "Please report by e-mail to %s." % ML
+                                logger.critical(warnMsg)
 
                     except KeyboardInterrupt:
-                        print
+                        pass
-                        warnMsg = "user aborted during dictionary attack phase"
-                        logger.warn(warnMsg)
-                        return results
 
-                    except:
+                retVal = None
-                        warnMsg = "there was a problem while hashing entry: %s. " % repr(word)
+
-                        warnMsg += "Please report by e-mail to %s." % ML
+                try:
-                        logger.critical(warnMsg)
+                    if PYVERSION >= "2.6":
+                        infoMsg = "starting %d hash attack processes " % multiprocessing.cpu_count()
+                        singleTimeLogMessage(infoMsg)
+
+                        processes = []
+                        retVal = multiprocessing.Queue()
+                        for i in xrange(multiprocessing.cpu_count()):
+                            p = multiprocessing.Process(target=bruteProcess, args=(attack_info, hash_regex, kb.wordlist, suffix, retVal, i, multiprocessing.cpu_count()))
+                            p.start()
+                            processes.append(p)
+
+                        for p in processes:
+                            p.join()
+
+                    else:
+                        warnMsg = "multiprocessing not supported on current version of "
+                        warnMsg += "Python (%s < 2.6)" % PYVERSION
+                        singleTimeWarnMessage(warnMsg)
+
+                        retVal = Queue()
+                        bruteProcess(attack_info, hash_regex, kb.wordlist, suffix, retVal, 0, 1)
+
+                except KeyboardInterrupt:
+                    print
+                    processException = True
+                    warnMsg = "user aborted during dictionary attack phase"
+                    logger.warn(warnMsg)
+
+                results = [retVal.get() for i in xrange(retVal.qsize())] if retVal else []
 
             clearConsoleLine()
 
@@ -490,7 +533,7 @@ def dictionaryAttack(attack_dict):
                 found = False
 
                 for suffix in suffix_list:
-                    if found:
+                    if found or processException:
                         break
 
                     if suffix:
@@ -500,50 +543,102 @@ def dictionaryAttack(attack_dict):
 
                     kb.wordlist.rewind()
 
-                    for word in kb.wordlist:
+                    def bruteProcess(user, hash_, kwargs, hash_regex, wordlist, suffix, retVal, found, proc_id, proc_count):
-                        current = __functions__[hash_regex](password = word, uppercase = False, **kwargs)
+                        count = 0
-                        count += 1
-
-                        if not isinstance(word, basestring):
-                            continue
-
-                        if suffix:
-                            word = word + suffix
 
                         try:
-                            if hash_ == current:
+                            for word in kb.wordlist:
-                                if regex == HASH.ORACLE_OLD: #only for cosmetic purposes
-                                    word = word.upper()
-                                results.append((user, hash_, word))
-                                clearConsoleLine()
 
-                                infoMsg = "[%s] [INFO] found: '%s'" % (time.strftime("%X"), word)
+                                current = __functions__[hash_regex](password = word, uppercase = False, **kwargs)
+                                count += 1
 
-                                if user and not user.startswith(DUMMY_USER_PREFIX):
+                                if not isinstance(word, basestring):
-                                    infoMsg += " for user '%s'\n" % user
+                                    continue
-                                else:
-                                    infoMsg += " for hash '%s'\n" % hash_
 
-                                dataToStdout(infoMsg, True)
+                                if suffix:
+                                    word = word + suffix
 
-                                found = True
+                                try:
-                                break
+                                    if hash_ == current:
-                            elif count % HASH_MOD_ITEM_DISPLAY == 0 or hash_regex in (HASH.ORACLE_OLD) or hash_regex == HASH.CRYPT_GENERIC and IS_WIN:
+                                        if regex == HASH.ORACLE_OLD: #only for cosmetic purposes
-                                status = 'current status: %d%s (%s...)' % (kb.wordlist.percentage(), '%', word.ljust(5)[:5])
+                                            word = word.upper()
-                                if not user.startswith(DUMMY_USER_PREFIX):
+
-                                    status += ' (user: %s)' % user
+                                        retVal.put((user, hash_, word))
-                                dataToStdout("\r[%s] [INFO] %s" % (time.strftime("%X"), status))
+
+                                        clearConsoleLine()
+
+                                        infoMsg = "[%s] [INFO] found: '%s'" % (time.strftime("%X"), word)
+
+                                        if user and not user.startswith(DUMMY_USER_PREFIX):
+                                            infoMsg += " for user '%s'\n" % user
+                                        else:
+                                            infoMsg += " for hash '%s'\n" % hash_
+
+                                        dataToStdout(infoMsg, True)
+
+                                        found.value = True
+                                        break
+                                    elif proc_id == 0 and count % HASH_MOD_ITEM_DISPLAY == 0 or hash_regex in (HASH.ORACLE_OLD) or hash_regex == HASH.CRYPT_GENERIC and IS_WIN:
+                                        status = 'current status: %d%s (%s...)' % (proc_count * kb.wordlist.percentage(), '%', word.ljust(5)[:5])
+                                        if not user.startswith(DUMMY_USER_PREFIX):
+                                            status += ' (user: %s)' % user
+                                        dataToStdout("\r[%s] [INFO] %s" % (time.strftime("%X"), status))
+
+                                except KeyboardInterrupt:
+                                    raise
+
+                                except:
+                                    warnMsg = "there was a problem while hashing entry: %s. " % repr(word)
+                                    warnMsg += "Please report by e-mail to %s." % ML
+                                    logger.critical(warnMsg)
 
                         except KeyboardInterrupt:
-                            print
+                            pass
-                            warnMsg = "user aborted during dictionary attack phase"
-                            logger.warn(warnMsg)
-                            return results
 
-                        except:
+                    retVal = None
-                            warnMsg = "there was a problem while hashing entry: %s. " % repr(word)
+
-                            warnMsg += "Please report by e-mail to %s." % ML
+                    try:
-                            logger.critical(warnMsg)
+                        if PYVERSION >= "2.6":
+                            infoMsg = "starting %d hash attack processes " % multiprocessing.cpu_count()
+                            singleTimeLogMessage(infoMsg)
+
+                            processes = []
+                            retVal = multiprocessing.Queue()
+                            found_ = multiprocessing.Value('i', False)
+
+                            for i in xrange(multiprocessing.cpu_count()):
+                                p = multiprocessing.Process(target=bruteProcess, args=(user, hash_, kwargs, hash_regex, kb.wordlist, suffix, retVal, found_, i, multiprocessing.cpu_count()))
+                                p.start()
+                                processes.append(p)
+
+                            for p in processes:
+                                p.join()
+
+                            found = found_.value != 0
+
+                        else:
+                            warnMsg = "multiprocessing not supported on current version of "
+                            warnMsg += "Python (%s < 2.6)" % PYVERSION
+                            singleTimeWarnMessage(warnMsg)
+
+                            class Value():
+                                pass
+
+                            retVal = Queue()
+                            found_ = Value()
+                            found_.value = False
+
+                            bruteProcess(user, hash_, kwargs, hash_regex, kb.wordlist, suffix, retVal, found_, 0, 1)
+
+                            found = found_.value
+
+                    except KeyboardInterrupt:
+                        print
+                        processException = True
+                        warnMsg = "user aborted during dictionary attack phase"
+                        logger.warn(warnMsg)
+
+                    results = [retVal.get() for i in xrange(retVal.qsize())] if retVal else []
 
                 clearConsoleLine()
 

plugins/generic/misc.py

@@ -164,7 +164,7 @@ class Miscellaneous:
         if not choice or choice == "1":
             choice = "1"
             condParam = " LIKE '%%%s%%'"
-        elif choice.isdigit() and choice == "2":
+        elif choice == "2":
             condParam = "='%s'"
         else:
             errMsg = "invalid value"