Partial implementation for an Issue #189 (error-based; still partial union left)

This commit is contained in:
stamparm 2013-05-09 16:23:57 +02:00
parent 9fe5a8832f
commit 3873805dab
3 changed files with 23 additions and 11 deletions

View File

@ -69,17 +69,17 @@ class ProgressBar(object):
percentString = getUnicode(percentDone) + "%"
self._progBar = "%s %s" % (percentString, self._progBar)
def progress(self, deltaTime, newAmount, threads=1):
def progress(self, deltaTime, newAmount):
"""
This method saves item delta time and shows updated progress bar with calculated eta
"""
if len(self._times) <= ((self._max * 3) / 100) or newAmount == self._max:
if len(self._times) <= ((self._max * 3) / 100) or newAmount > self._max:
eta = 0
else:
midTime = sum(self._times) / len(self._times)
midTimeWithLatest = (midTime + deltaTime) / 2
eta = midTimeWithLatest * (self._max - newAmount) / threads
eta = midTimeWithLatest * (self._max - newAmount)
self._times.append(deltaTime)
self.update(newAmount)

View File

@ -391,7 +391,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
if kb.threadContinue:
if showEta:
progress.progress(time.time() - charStart, threadData.shared.index[0], numThreads)
progress.progress(time.time() - charStart, threadData.shared.index[0])
elif conf.verbose >= 1:
startCharIndex = 0
endCharIndex = 0

View File

@ -35,6 +35,7 @@ from lib.core.data import logger
from lib.core.data import queries
from lib.core.dicts import FROM_DUMMY_TABLE
from lib.core.enums import DBMS
from lib.core.progress import ProgressBar
from lib.core.settings import CHECK_ZERO_COLUMNS_THRESHOLD
from lib.core.settings import MYSQL_ERROR_CHUNK_LENGTH
from lib.core.settings import MSSQL_ERROR_CHUNK_LENGTH
@ -162,7 +163,7 @@ def _oneShotErrorUse(expression, field=None):
return safecharencode(retVal) if kb.safeCharEncode else retVal
def _errorFields(expression, expressionFields, expressionFieldsList, num=None, emptyFields=None):
def _errorFields(expression, expressionFields, expressionFieldsList, num=None, emptyFields=None, suppressOutput=False):
values = []
origExpr = None
@ -188,6 +189,7 @@ def _errorFields(expression, expressionFields, expressionFieldsList, num=None, e
if not kb.threadContinue:
return None
if not suppressOutput:
if kb.fileReadMode and output and output.strip():
print
elif output is not None and not (threadData.resumed and kb.suppressResumeInfo) and not (emptyFields and field in emptyFields):
@ -308,12 +310,18 @@ def errorUse(expression, dump=False):
if _ and _[0] in ("y", "Y"):
expression = expression[:expression.index(" ORDER BY ")]
numThreads = min(conf.threads, (stopLimit - startLimit))
threadData = getCurrentThreadData()
threadData.shared.limits = iter(xrange(startLimit, stopLimit))
numThreads = min(conf.threads, (stopLimit - startLimit))
threadData.shared.value = BigArray()
threadData.shared.buffered = []
threadData.shared.counter = 0
threadData.shared.lastFlushed = startLimit - 1
threadData.shared.showEta = conf.eta and (stopLimit - startLimit) > 1
if threadData.shared.showEta:
threadData.shared.progress = ProgressBar(maxValue=(stopLimit - startLimit))
if kb.dumpTable and (len(expressionFieldsList) < (stopLimit - startLimit) > CHECK_ZERO_COLUMNS_THRESHOLD):
for field in expressionFieldsList:
@ -336,11 +344,13 @@ def errorUse(expression, dump=False):
while kb.threadContinue:
with kb.locks.limit:
try:
valueStart = time.time()
threadData.shared.counter += 1
num = threadData.shared.limits.next()
except StopIteration:
break
output = _errorFields(expression, expressionFields, expressionFieldsList, num, emptyFields)
output = _errorFields(expression, expressionFields, expressionFieldsList, num, emptyFields, threadData.shared.showEta)
if not kb.threadContinue:
break
@ -350,6 +360,8 @@ def errorUse(expression, dump=False):
with kb.locks.value:
index = None
if threadData.shared.showEta:
threadData.shared.progress.progress(time.time() - valueStart, threadData.shared.counter)
for index in xrange(len(threadData.shared.buffered)):
if threadData.shared.buffered[index][0] >= num:
break