From 094129a6560eab602f6a4b0982ea07cc2f5780da Mon Sep 17 00:00:00 2001
From: Miroslav Stampar <miroslav.stampar@gmail.com>
Date: Thu, 22 Dec 2011 15:42:21 +0000
Subject: [PATCH] minor optimization

---
 lib/core/common.py   | 7 +++++--
 lib/core/settings.py | 3 +++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/lib/core/common.py b/lib/core/common.py
index 9920e1478..5e01444cd 100644
--- a/lib/core/common.py
+++ b/lib/core/common.py
@@ -112,6 +112,7 @@ from lib.core.settings import DYNAMICITY_MARK_LENGTH
 from lib.core.settings import REFLECTIVE_MISS_THRESHOLD
 from lib.core.settings import SENSITIVE_DATA_REGEX
 from lib.core.settings import SUPPORTED_OS
+from lib.core.settings import UNION_UNIQUE_FIFO_LENGTH
 from lib.core.settings import URI_INJECTION_MARK_CHAR
 from lib.core.settings import URI_QUESTION_MARKER
 from lib.core.threads import getCurrentThreadData
@@ -1336,7 +1337,7 @@ def parseUnionPage(output, unique=True):
     if output.startswith(kb.chars.start) and output.endswith(kb.chars.stop):
         regExpr = '%s(.*?)%s' % (kb.chars.start, kb.chars.stop)
         output = re.finditer(regExpr, output, re.DOTALL | re.IGNORECASE)
-        _ = set()
+        _ = []
 
         for entry in output:
             entry = entry.group(1)
@@ -1344,7 +1345,9 @@ def parseUnionPage(output, unique=True):
             if unique:
                 key = entry.lower()
                 if key not in _:
-                    _.add(key)
+                    _.append(key)
+                    if len(_) > UNION_UNIQUE_FIFO_LENGTH:
+                        _.pop(0)
                 else:
                     continue
 
diff --git a/lib/core/settings.py b/lib/core/settings.py
index 563825058..a1377bb50 100644
--- a/lib/core/settings.py
+++ b/lib/core/settings.py
@@ -115,6 +115,9 @@ UNKNOWN_DBMS_VERSION = "Unknown"
 # dynamicity mark length used in dynamicity removal engine
 DYNAMICITY_MARK_LENGTH = 32
 
+# length of FIFO buffer for removing possible duplicates in union/inband data retrieval
+UNION_UNIQUE_FIFO_LENGTH = 10
+
 # dummy user prefix used in dictionary attack
 DUMMY_USER_PREFIX = "__dummy__"