optimization of reflective removal mechanism

This commit is contained in:
Miroslav Stampar 2011-07-12 22:28:19 +00:00
parent 4cb9988243
commit 9933edc718

View File

@ -2595,22 +2595,22 @@ def removeReflectiveValues(content, payload, suppressWarning=False):
while 2 * REFLECTED_NON_ALPHA_NUM_REGEX in regex: while 2 * REFLECTED_NON_ALPHA_NUM_REGEX in regex:
regex = regex.replace(2 * REFLECTED_NON_ALPHA_NUM_REGEX, REFLECTED_NON_ALPHA_NUM_REGEX) regex = regex.replace(2 * REFLECTED_NON_ALPHA_NUM_REGEX, REFLECTED_NON_ALPHA_NUM_REGEX)
if reduce(lambda x,y: x if x else y, regex.split(REFLECTED_NON_ALPHA_NUM_REGEX)).lower() in content.lower(): # fast optimization check if all([part.lower() in content.lower() for part in regex.split(REFLECTED_NON_ALPHA_NUM_REGEX)]): # fast optimization check
retVal = re.sub(regex, REFLECTED_VALUE_MARKER, content, re.I) retVal = re.sub(regex, REFLECTED_VALUE_MARKER, content, re.I)
if retVal != content: if retVal != content:
kb.reflectiveCounters[REFLECTIVE_COUNTER.HIT] += 1 kb.reflectiveCounters[REFLECTIVE_COUNTER.HIT] += 1
if not suppressWarning: if not suppressWarning:
debugMsg = "reflective value found and filtered out" debugMsg = "reflective value found and filtered out"
logger.debug(debugMsg) logger.debug(debugMsg)
elif not kb.testMode and not kb.reflectiveCounters[REFLECTIVE_COUNTER.HIT]: elif not kb.testMode and not kb.reflectiveCounters[REFLECTIVE_COUNTER.HIT]:
kb.reflectiveCounters[REFLECTIVE_COUNTER.MISS] += 1 kb.reflectiveCounters[REFLECTIVE_COUNTER.MISS] += 1
if kb.reflectiveCounters[REFLECTIVE_COUNTER.MISS] > REFLECTIVE_MISS_THRESHOLD: if kb.reflectiveCounters[REFLECTIVE_COUNTER.MISS] > REFLECTIVE_MISS_THRESHOLD:
kb.reflectiveMechanism = False kb.reflectiveMechanism = False
if not suppressWarning: if not suppressWarning:
debugMsg = "turning off reflection removal mechanism (for optimization purposes)" debugMsg = "turning off reflection removal mechanism (for optimization purposes)"
logger.debug(debugMsg) logger.debug(debugMsg)
return retVal return retVal