doing proper big table support for partial union too

lib/core/common.py

@@ -1334,49 +1334,53 @@ def parseUnionPage(output, expression, partial=False, condition=None, sort=True)
 
     data = BigArray()
 
-    outCond1 = ( output.startswith(kb.misc.start) and output.endswith(kb.misc.stop) )
+    if isinstance(output, list):
-    outCond2 = ( output.startswith(DUMP_START_MARKER) and output.endswith(DUMP_STOP_MARKER) )
-
-    if outCond1 or outCond2:
-        if outCond1:
-            regExpr = '%s(.*?)%s' % (kb.misc.start, kb.misc.stop)
-        elif outCond2:
-            regExpr = '%s(.*?)%s' % (DUMP_START_MARKER, DUMP_STOP_MARKER)
-
-        output = re.findall(regExpr, output, re.DOTALL | re.IGNORECASE)
-        if condition is None:
-            condition = (
-                          kb.resumedQueries and conf.url in kb.resumedQueries.keys()
-                          and expression in kb.resumedQueries[conf.url].keys()
-                        )
-
-        if partial or not condition:
-            logOutput = "".join(["%s%s%s" % (DUMP_START_MARKER, replaceNewlineTabs(value), DUMP_STOP_MARKER) for value in output])
-            dataToSessionFile("[%s][%s][%s][%s][%s]\n" % (conf.url, kb.injection.place, conf.parameters[kb.injection.place], expression, logOutput))
-
-        if sort:
-            dict_ = {}
-            for entry in output:
-                dict_[entry.lower()] = entry
-            output = dict_.values()
-
         for entry in output:
-            info = []
+            data.append(entry[0] if len(entry) == 1 else entry)            
-
-            if DUMP_DEL_MARKER in entry:
-                entry = entry.split(DUMP_DEL_MARKER)
-            else:
-                entry = entry.split(kb.misc.delimiter)
-
-            if len(entry) == 1:
-                data.append(entry[0])
-            else:
-                for value in entry:
-                    info.append(value)
-
-                data.append(info)
     else:
-        data = output
+        outCond1 = ( output.startswith(kb.misc.start) and output.endswith(kb.misc.stop) )
+        outCond2 = ( output.startswith(DUMP_START_MARKER) and output.endswith(DUMP_STOP_MARKER) )
+
+        if outCond1 or outCond2:
+            if outCond1:
+                regExpr = '%s(.*?)%s' % (kb.misc.start, kb.misc.stop)
+            elif outCond2:
+                regExpr = '%s(.*?)%s' % (DUMP_START_MARKER, DUMP_STOP_MARKER)
+
+            output = re.findall(regExpr, output, re.DOTALL | re.IGNORECASE)
+            if condition is None:
+                condition = (
+                            kb.resumedQueries and conf.url in kb.resumedQueries.keys()
+                            and expression in kb.resumedQueries[conf.url].keys()
+                            )
+
+            if partial or not condition:
+                logOutput = "".join(["%s%s%s" % (DUMP_START_MARKER, replaceNewlineTabs(value), DUMP_STOP_MARKER) for value in output])
+                dataToSessionFile("[%s][%s][%s][%s][%s]\n" % (conf.url, kb.injection.place, conf.parameters[kb.injection.place], expression, logOutput))
+
+            if sort:
+                dict_ = {}
+                for entry in output:
+                    dict_[entry.lower()] = entry
+                output = dict_.values()
+
+            for entry in output:
+                info = []
+
+                if DUMP_DEL_MARKER in entry:
+                    entry = entry.split(DUMP_DEL_MARKER)
+                else:
+                    entry = entry.split(kb.misc.delimiter)
+
+                if len(entry) == 1:
+                    data.append(entry[0])
+                else:
+                    for value in entry:
+                        info.append(value)
+
+                    data.append(info)
+        else:
+            data = output
 
     if len(data) == 1 and isinstance(data[0], basestring):
         data = data[0]

lib/techniques/union/test.py

@@ -19,7 +19,6 @@ from lib.core.common import dataToStdout
 from lib.core.common import extractRegexResult
 from lib.core.common import getUnicode
 from lib.core.common import listToStrValue
-from lib.core.common import parseUnionPage
 from lib.core.common import popValue
 from lib.core.common import pushValue
 from lib.core.common import randomStr

lib/techniques/union/use.py

@@ -13,6 +13,7 @@ import time
 
 from lib.core.agent import agent
 from lib.core.common import Backend
+from lib.core.common import BigArray
 from lib.core.common import calculateDeltaSeconds
 from lib.core.common import clearConsoleLine
 from lib.core.common import dataToStdout
@@ -262,7 +263,7 @@ def unionUse(expression, unpack=True, dump=False):
             threadData = getCurrentThreadData()
             threadData.shared.limits = range(startLimit, stopLimit)
             numThreads = min(conf.threads, len(threadData.shared.limits))
-            threadData.shared.value = ""
+            threadData.shared.value = BigArray()
 
             if stopLimit > TURN_OFF_RESUME_INFO_LIMIT:
                 kb.suppressResumeInfo = True
@@ -305,16 +306,15 @@ def unionUse(expression, unpack=True, dump=False):
                             break
 
                         if output:
-                            kb.locks.value.acquire()
+                            if all(map(lambda x: x in output, [kb.misc.start, kb.misc.stop])):
-                            threadData.shared.value += output
+                                items = extractRegexResult(r'%s(?P<result>.*?)%s' % (kb.misc.start, kb.misc.stop), output, re.DOTALL | re.IGNORECASE).split(kb.misc.delimiter)
-                            kb.locks.value.release()
+                                kb.locks.value.acquire()
+                                threadData.shared.value.append(items)
+                                kb.locks.value.release()
+                            else:
+                                items = output.replace(kb.misc.start, "").replace(kb.misc.stop, "").split(kb.misc.delimiter)
 
                             if conf.verbose == 1:
-                                if all(map(lambda x: x in output, [kb.misc.start, kb.misc.stop])):
-                                    items = extractRegexResult(r'%s(?P<result>.*?)%s' % (kb.misc.start, kb.misc.stop), output, re.DOTALL | re.IGNORECASE).split(kb.misc.delimiter)
-                                else:
-                                    items = output.replace(kb.misc.start, "").replace(kb.misc.stop, "").split(kb.misc.delimiter)
-
                                 status = "[%s] [INFO] retrieved: %s\r\n" % (time.strftime("%X"), safecharencode(",".join(map(lambda x: "\"%s\"" % x, items))))
 
                                 if len(status) > width: