memory optimization for reflective removal mechanism (there was no need for \n\r in the first place as there was no re.S flag used - also, one re.sub "flags <-> count" bug fixed)

This commit is contained in:
Miroslav Stampar 2012-03-28 19:27:12 +00:00
parent 7d131d1fb1
commit 9433bbe26d
2 changed files with 10 additions and 10 deletions

View File

@ -116,7 +116,7 @@ from lib.core.settings import LARGE_OUTPUT_THRESHOLD
from lib.core.settings import ML
from lib.core.settings import MIN_TIME_RESPONSES
from lib.core.settings import PAYLOAD_DELIMITER
from lib.core.settings import REFLECTED_NON_ALPHA_NUM_REGEX
from lib.core.settings import REFLECTED_REPLACEMENT_REGEX
from lib.core.settings import REFLECTED_MAX_REGEX_PARTS
from lib.core.settings import REFLECTED_VALUE_MARKER
from lib.core.settings import TIME_STDEV_COEFF
@ -2627,17 +2627,17 @@ def removeReflectiveValues(content, payload, suppressWarning=False):
if all([content, payload]) and isinstance(content, unicode) and kb.reflectiveMechanism:
payload = getUnicode(urldecode(payload.replace(PAYLOAD_DELIMITER, '')))
regex = r"\b%s\b" % filterStringValue(payload, r'[A-Za-z0-9]', REFLECTED_NON_ALPHA_NUM_REGEX.encode("string-escape"))
regex = filterStringValue(payload, r'[A-Za-z0-9]', REFLECTED_REPLACEMENT_REGEX.encode("string-escape"))
while 2 * REFLECTED_NON_ALPHA_NUM_REGEX in regex:
regex = regex.replace(2 * REFLECTED_NON_ALPHA_NUM_REGEX, REFLECTED_NON_ALPHA_NUM_REGEX)
while 2 * REFLECTED_REPLACEMENT_REGEX in regex:
regex = regex.replace(2 * REFLECTED_REPLACEMENT_REGEX, REFLECTED_REPLACEMENT_REGEX)
if all(part.lower() in content.lower() for part in regex.strip('\\b').split(REFLECTED_NON_ALPHA_NUM_REGEX)): # fast optimization check
parts = regex.split(REFLECTED_NON_ALPHA_NUM_REGEX)
if all(part.lower() in content.lower() for part in regex.split(REFLECTED_REPLACEMENT_REGEX)): # fast optimization check
parts = regex.split(REFLECTED_REPLACEMENT_REGEX)
if len(parts) > REFLECTED_MAX_REGEX_PARTS: # preventing CPU hogs
regex = "%s.+?%s" % (REFLECTED_NON_ALPHA_NUM_REGEX.join(parts[:REFLECTED_MAX_REGEX_PARTS / 2]), REFLECTED_NON_ALPHA_NUM_REGEX.join(parts[-REFLECTED_MAX_REGEX_PARTS / 2:]))
regex = "%s.+?%s" % (REFLECTED_REPLACEMENT_REGEX.join(parts[:REFLECTED_MAX_REGEX_PARTS / 2]), REFLECTED_REPLACEMENT_REGEX.join(parts[-REFLECTED_MAX_REGEX_PARTS / 2:]))
retVal = re.sub(regex, REFLECTED_VALUE_MARKER, content, re.I)
retVal = re.sub(r"(?i)\b%s\b" % regex, REFLECTED_VALUE_MARKER, content)
if retVal != content:
kb.reflectiveCounters[REFLECTIVE_COUNTER.HIT] += 1

View File

@ -329,8 +329,8 @@ EXCLUDE_UNESCAPE = ("WAITFOR DELAY ", " INTO DUMPFILE ", " INTO OUTFILE ", "CREA
# Mark used for replacement of reflected values
REFLECTED_VALUE_MARKER = '__REFLECTED_VALUE__'
# Regular expression used for marking non-alphanum characters
REFLECTED_NON_ALPHA_NUM_REGEX = r'[^\r\n]+?'
# Regular expression used for replacing non-alphanum characters
REFLECTED_REPLACEMENT_REGEX = r'.+'
# Maximum number of alpha-numerical parts in reflected regex (for speed purposes)
REFLECTED_MAX_REGEX_PARTS = 10