optimization of reflective removal mechanism

This commit is contained in:
Miroslav Stampar 2011-07-12 22:28:19 +00:00
parent 4cb9988243
commit 9933edc718

View File

@ -2595,22 +2595,22 @@ def removeReflectiveValues(content, payload, suppressWarning=False):
while 2 * REFLECTED_NON_ALPHA_NUM_REGEX in regex:
regex = regex.replace(2 * REFLECTED_NON_ALPHA_NUM_REGEX, REFLECTED_NON_ALPHA_NUM_REGEX)
if reduce(lambda x,y: x if x else y, regex.split(REFLECTED_NON_ALPHA_NUM_REGEX)).lower() in content.lower(): # fast optimization check
if all([part.lower() in content.lower() for part in regex.split(REFLECTED_NON_ALPHA_NUM_REGEX)]): # fast optimization check
retVal = re.sub(regex, REFLECTED_VALUE_MARKER, content, re.I)
if retVal != content:
kb.reflectiveCounters[REFLECTIVE_COUNTER.HIT] += 1
if not suppressWarning:
debugMsg = "reflective value found and filtered out"
logger.debug(debugMsg)
if retVal != content:
kb.reflectiveCounters[REFLECTIVE_COUNTER.HIT] += 1
if not suppressWarning:
debugMsg = "reflective value found and filtered out"
logger.debug(debugMsg)
elif not kb.testMode and not kb.reflectiveCounters[REFLECTIVE_COUNTER.HIT]:
kb.reflectiveCounters[REFLECTIVE_COUNTER.MISS] += 1
if kb.reflectiveCounters[REFLECTIVE_COUNTER.MISS] > REFLECTIVE_MISS_THRESHOLD:
kb.reflectiveMechanism = False
if not suppressWarning:
debugMsg = "turning off reflection removal mechanism (for optimization purposes)"
logger.debug(debugMsg)
elif not kb.testMode and not kb.reflectiveCounters[REFLECTIVE_COUNTER.HIT]:
kb.reflectiveCounters[REFLECTIVE_COUNTER.MISS] += 1
if kb.reflectiveCounters[REFLECTIVE_COUNTER.MISS] > REFLECTIVE_MISS_THRESHOLD:
kb.reflectiveMechanism = False
if not suppressWarning:
debugMsg = "turning off reflection removal mechanism (for optimization purposes)"
logger.debug(debugMsg)
return retVal