massive (like really massive) dictionary support

lib/core/common.py

@@ -132,6 +132,62 @@ class UnicodeRawConfigParser(RawConfigParser):
 
             fp.write("\n")
 
+class Wordlist:
+    """
+    Iterator for looping over a large dictionaries
+    """
+
+    def __init__(self, filenames):
+        self.filenames = filenames
+        self.fp = None
+        self.index = 0
+        self.iter = None
+        self.cursize = 0
+        self.custom = []
+        self.adjust()
+
+    def __iter__(self):
+        return self
+
+    def adjust(self):
+        self.closeFP()
+        if self.index > len(self.filenames):
+            raise StopIteration
+        elif self.index == len(self.filenames):
+            self.iter = iter(self.custom)
+        else:
+            current = self.filenames[self.index]
+            infoMsg = "loading dictionary from: '%s'" % current
+            singleTimeLogMessage(infoMsg)
+            self.fp = open(current, "r")
+            self.cursize = os.path.getsize(current)
+            self.iter = self.fp.xreadlines()
+        self.index += 1
+
+    def append(self, value):
+        self.custom.append(value)
+        
+    def closeFP(self):
+        if self.fp:
+            self.fp.close()
+
+    def next(self):
+        try:
+            return self.iter.next().rstrip()
+        except StopIteration:
+            self.adjust()
+            return self.iter.next().rstrip()
+
+    def percentage(self):
+        retVal = 0
+        if self.fp:
+            retVal = round(100.0 * self.fp.tell() / self.cursize)
+        return retVal
+
+    def rewind(self):
+        self.index = 0
+        self.adjust()
+
 class DynamicContentItem:
     """
     Represents line in content page with dynamic properties (candidate

lib/utils/hash.py

@@ -13,6 +13,7 @@ try:
 except ImportError, _:
     from extra.fcrypt.fcrypt import crypt
 
+import os
 import re
 import time
 
@@ -33,6 +34,8 @@ from lib.core.common import getPublicTypeMembers
 from lib.core.common import normalizeUnicode
 from lib.core.common import paths
 from lib.core.common import readInput
+from lib.core.common import singleTimeLogMessage
+from lib.core.common import Wordlist
 from lib.core.convert import hexdecode
 from lib.core.convert import hexencode
 from lib.core.convert import utf8encode
@@ -363,7 +366,6 @@ def dictionaryAttack(attack_dict):
             continue
 
         if not kb.wordlist:
-
             while not kb.wordlist:
                 message = "what dictionary do you want to use?\n"
                 message += "[1] Default (Press Enter)\n"
@@ -386,15 +388,10 @@ def dictionaryAttack(attack_dict):
                         else:
                             dictPaths = [paths.WORDLIST]
 
-                    kb.wordlist = []
-
                     for dictPath in dictPaths:
                         checkFile(dictPath)
 
-                        infoMsg = "loading dictionary from: '%s'" % dictPath
-                        logger.info(infoMsg)
-
-                        kb.wordlist.extend(getFileItems(dictPath, None, False))
+                    kb.wordlist = Wordlist(dictPaths)
 
                 except sqlmapFilePathException, msg:
                     warnMsg = "there was a problem while loading dictionaries"
@@ -416,8 +413,6 @@ def dictionaryAttack(attack_dict):
             if user:
                 kb.wordlist.append(normalizeUnicode(user))
 
-        length = len(kb.wordlist) * len(suffix_list)
-
         if hash_regex in (HASH.MYSQL, HASH.MYSQL_OLD, HASH.MD5_GENERIC, HASH.SHA1_GENERIC):
             count = 0
 
@@ -425,6 +420,13 @@ def dictionaryAttack(attack_dict):
                 if not attack_info:
                     break
 
+                if suffix:
+                    clearConsoleLine()
+                    infoMsg = "using suffix: '%s'" % suffix
+                    logger.info(infoMsg)
+
+                kb.wordlist.rewind()
+
                 for word in kb.wordlist:
                     if not attack_info:
                         break
@@ -458,8 +460,8 @@ def dictionaryAttack(attack_dict):
 
                                 attack_info.remove(item)
 
-                            elif count % HASH_MOD_ITEM_DISPLAY == 0 or count == length or hash_regex in (HASH.ORACLE_OLD) or hash_regex == HASH.CRYPT_GENERIC and IS_WIN:
-                                status = '%d/%d words (%d%s)' % (count, length, round(100.0*count/length), '%')
+                            elif count % HASH_MOD_ITEM_DISPLAY == 0 or hash_regex in (HASH.ORACLE_OLD) or hash_regex == HASH.CRYPT_GENERIC and IS_WIN:
+                                status = 'current status: %d%s (%s...)' % (kb.wordlist.percentage(), '%', word.ljust(5)[:5])
                                 dataToStdout("\r[%s] [INFO] %s" % (time.strftime("%X"), status))
 
                     except KeyboardInterrupt:
@@ -484,6 +486,13 @@ def dictionaryAttack(attack_dict):
                     if found:
                         break
 
+                    if suffix:
+                        clearConsoleLine()
+                        infoMsg = "using suffix: '%s'" % suffix
+                        logger.info(infoMsg)
+
+                    kb.wordlist.rewind()
+
                     for word in kb.wordlist:
                         current = __functions__[hash_regex](password = word, uppercase = False, **kwargs)
                         count += 1
@@ -512,8 +521,8 @@ def dictionaryAttack(attack_dict):
 
                                 found = True
                                 break
-                            elif count % HASH_MOD_ITEM_DISPLAY == 0 or count == length or hash_regex in (HASH.ORACLE_OLD) or hash_regex == HASH.CRYPT_GENERIC and IS_WIN:
-                                status = '%d/%d words (%d%s)' % (count, length, round(100.0*count/length), '%')
+                            elif count % HASH_MOD_ITEM_DISPLAY == 0 or hash_regex in (HASH.ORACLE_OLD) or hash_regex == HASH.CRYPT_GENERIC and IS_WIN:
+                                status = 'current status: %d%s (%s...)' % (kb.wordlist.percentage(), '%', word.ljust(5)[:5])
                                 if not user.startswith(DUMMY_USER_PREFIX):
                                     status += ' (user: %s)' % user
                                 dataToStdout("\r[%s] [INFO] %s" % (time.strftime("%X"), status))