Implementation for an Issue #307

This commit is contained in:
Miroslav Stampar 2012-12-18 16:03:35 +01:00
parent eb23b1b1a5
commit 88d8494b5a
3 changed files with 40 additions and 6 deletions

View File

@ -40,6 +40,10 @@ class BigArray(list):
self.chunks[-1] = filename
self.chunks.append([])
def extend(self, value):
for _ in value:
self.append(_)
def pop(self):
if len(self.chunks[-1]) < 1:
self.chunks.pop()

View File

@ -357,6 +357,8 @@ def errorUse(expression, dump=False):
threadData.shared.limits = iter(xrange(startLimit, stopLimit))
numThreads = min(conf.threads, (stopLimit - startLimit))
threadData.shared.outputs = BigArray()
threadData.shared.buffered = []
threadData.shared.lastFlushed = startLimit - 1
if kb.dumpTable and (len(expressionFieldsList) < (stopLimit - startLimit) > CHECK_ZERO_COLUMNS_THRESHOLD):
for field in expressionFieldsList:
@ -392,7 +394,15 @@ def errorUse(expression, dump=False):
output = output[0]
with kb.locks.outputs:
threadData.shared.outputs.append(output)
index = None
for index in xrange(len(threadData.shared.buffered)):
if threadData.shared.buffered[index][0] >= num:
break
threadData.shared.buffered.insert(index or 0, (num, output))
while threadData.shared.buffered and threadData.shared.lastFlushed + 1 == threadData.shared.buffered[0][0]:
threadData.shared.lastFlushed += 1
threadData.shared.outputs.append(threadData.shared.buffered[0][1])
del threadData.shared.buffered[0]
runThreads(numThreads, errorThread)
@ -403,6 +413,7 @@ def errorUse(expression, dump=False):
logger.warn(warnMsg)
finally:
threadData.shared.outputs.extend(_[1] for _ in sorted(threadData.shared.buffered))
outputs = threadData.shared.outputs
kb.suppressResumeInfo = False

View File

@ -274,6 +274,8 @@ def unionUse(expression, unpack=True, dump=False):
threadData.shared.limits = iter(xrange(startLimit, stopLimit))
numThreads = min(conf.threads, (stopLimit - startLimit))
threadData.shared.value = BigArray()
threadData.shared.buffered = []
threadData.shared.lastFlushed = startLimit - 1
if stopLimit > TURN_OFF_RESUME_INFO_LIMIT:
kb.suppressResumeInfo = True
@ -306,14 +308,28 @@ def unionUse(expression, unpack=True, dump=False):
break
if output:
if all(map(lambda x: x in output, [kb.chars.start, kb.chars.stop])):
if all(map(lambda _: _ in output, (kb.chars.start, kb.chars.stop))):
items = parseUnionPage(output)
if isNoneValue(items):
continue
with kb.locks.value:
for item in arrayizeValue(items):
threadData.shared.value.append(item)
index = None
for index in xrange(len(threadData.shared.buffered)):
if threadData.shared.buffered[index][0] >= num:
break
threadData.shared.buffered.insert(index or 0, (num, items))
while threadData.shared.buffered and threadData.shared.lastFlushed + 1 == threadData.shared.buffered[0][0]:
threadData.shared.lastFlushed += 1
_ = threadData.shared.buffered[0][1]
if not isNoneValue(_):
threadData.shared.value.extend(arrayizeValue(_))
del threadData.shared.buffered[0]
else:
with kb.locks.value:
index = None
for index in xrange(len(threadData.shared.buffered)):
if threadData.shared.buffered[index][0] >= num:
break
threadData.shared.buffered.insert(index or 0, (num, None))
items = output.replace(kb.chars.start, "").replace(kb.chars.stop, "").split(kb.chars.delimiter)
if conf.verbose == 1 and not (threadData.resumed and kb.suppressResumeInfo):
@ -337,6 +353,9 @@ def unionUse(expression, unpack=True, dump=False):
logger.warn(warnMsg)
finally:
for _ in sorted(threadData.shared.buffered):
if not isNoneValue(_[1]):
threadData.shared.value.extend(arrayizeValue(_[1]))
value = threadData.shared.value
kb.suppressResumeInfo = False