speedup of UNION dumping

This commit is contained in:
Miroslav Stampar 2011-12-22 10:44:14 +00:00
parent 1ae413a206
commit 4a1a0773b7
3 changed files with 7 additions and 5 deletions

View File

@ -1328,7 +1328,7 @@ def getRange(count, dump=False, plusOne=False):
return indexRange return indexRange
def parseUnionPage(output, expression, partial=False, unique=True): def parseUnionPage(output, unique=True):
if output is None: if output is None:
return None return None
@ -1336,10 +1336,12 @@ def parseUnionPage(output, expression, partial=False, unique=True):
if output.startswith(kb.chars.start) and output.endswith(kb.chars.stop): if output.startswith(kb.chars.start) and output.endswith(kb.chars.stop):
regExpr = '%s(.*?)%s' % (kb.chars.start, kb.chars.stop) regExpr = '%s(.*?)%s' % (kb.chars.start, kb.chars.stop)
output = re.findall(regExpr, output, re.DOTALL | re.IGNORECASE) output = re.finditer(regExpr, output, re.DOTALL | re.IGNORECASE)
_ = set() _ = set()
for entry in output: for entry in output:
entry = entry.group(1)
if unique: if unique:
key = entry.lower() key = entry.lower()
if key not in _: if key not in _:

View File

@ -384,7 +384,7 @@ def __goInband(expression, expected=None, unique=True, resumeValue=True, unpack=
if isinstance(output, list): if isinstance(output, list):
data = output data = output
else: else:
data = parseUnionPage(output, expression, partial, unique) data = parseUnionPage(output, unique)
return data return data

View File

@ -233,13 +233,13 @@ def unionUse(expression, unpack=True, dump=False):
countedExpression = countedExpression[:untilOrderChar] countedExpression = countedExpression[:untilOrderChar]
count = resume(countedExpression, None) count = resume(countedExpression, None)
count = parseUnionPage(count, countedExpression) count = parseUnionPage(count)
if not count or not count.isdigit(): if not count or not count.isdigit():
output = __oneShotUnionUse(countedExpression, unpack) output = __oneShotUnionUse(countedExpression, unpack)
if output: if output:
count = parseUnionPage(output, countedExpression) count = parseUnionPage(output)
if isNumPosStrValue(count): if isNumPosStrValue(count):
if isinstance(stopLimit, int) and stopLimit > 0: if isinstance(stopLimit, int) and stopLimit > 0: