Minor restyling

This commit is contained in:
Miroslav Stampar 2012-12-20 11:06:52 +01:00
parent 19e2f3bb76
commit c2c4601d6e
5 changed files with 36 additions and 36 deletions

View File

@ -1538,7 +1538,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
kb.lastParserStatus = None kb.lastParserStatus = None
kb.locks = AttribDict() kb.locks = AttribDict()
for _ in ("cache", "count", "index", "io", "limits", "log", "outputs", "value"): for _ in ("cache", "count", "index", "io", "limit", "log", "value"):
kb.locks[_] = threading.Lock() kb.locks[_] = threading.Lock()
kb.matchRatio = None kb.matchRatio = None

View File

@ -66,7 +66,7 @@ def tableExists(tableFile, regex=None):
threadData = getCurrentThreadData() threadData = getCurrentThreadData()
threadData.shared.count = 0 threadData.shared.count = 0
threadData.shared.limit = len(tables) threadData.shared.limit = len(tables)
threadData.shared.outputs = [] threadData.shared.values = []
threadData.shared.unique = set() threadData.shared.unique = set()
def tableExistsThread(): def tableExistsThread():
@ -92,7 +92,7 @@ def tableExists(tableFile, regex=None):
kb.locks.io.acquire() kb.locks.io.acquire()
if result and table.lower() not in threadData.shared.unique: if result and table.lower() not in threadData.shared.unique:
threadData.shared.outputs.append(table) threadData.shared.values.append(table)
threadData.shared.unique.add(table.lower()) threadData.shared.unique.add(table.lower())
if conf.verbose in (1, 2): if conf.verbose in (1, 2):
@ -117,17 +117,17 @@ def tableExists(tableFile, regex=None):
clearConsoleLine(True) clearConsoleLine(True)
dataToStdout("\n") dataToStdout("\n")
if not threadData.shared.outputs: if not threadData.shared.values:
warnMsg = "no table(s) found" warnMsg = "no table(s) found"
logger.warn(warnMsg) logger.warn(warnMsg)
else: else:
for item in threadData.shared.outputs: for item in threadData.shared.values:
if conf.db not in kb.data.cachedTables: if conf.db not in kb.data.cachedTables:
kb.data.cachedTables[conf.db] = [item] kb.data.cachedTables[conf.db] = [item]
else: else:
kb.data.cachedTables[conf.db].append(item) kb.data.cachedTables[conf.db].append(item)
for _ in map(lambda x: (conf.db, x), threadData.shared.outputs): for _ in map(lambda x: (conf.db, x), threadData.shared.values):
if _ not in kb.brute.tables: if _ not in kb.brute.tables:
kb.brute.tables.append(_) kb.brute.tables.append(_)
@ -164,7 +164,7 @@ def columnExists(columnFile, regex=None):
threadData = getCurrentThreadData() threadData = getCurrentThreadData()
threadData.shared.count = 0 threadData.shared.count = 0
threadData.shared.limit = len(columns) threadData.shared.limit = len(columns)
threadData.shared.outputs = [] threadData.shared.values = []
def columnExistsThread(): def columnExistsThread():
threadData = getCurrentThreadData() threadData = getCurrentThreadData()
@ -184,7 +184,7 @@ def columnExists(columnFile, regex=None):
kb.locks.io.acquire() kb.locks.io.acquire()
if result: if result:
threadData.shared.outputs.append(column) threadData.shared.values.append(column)
if conf.verbose in (1, 2): if conf.verbose in (1, 2):
clearConsoleLine(True) clearConsoleLine(True)
@ -208,13 +208,13 @@ def columnExists(columnFile, regex=None):
clearConsoleLine(True) clearConsoleLine(True)
dataToStdout("\n") dataToStdout("\n")
if not threadData.shared.outputs: if not threadData.shared.values:
warnMsg = "no column(s) found" warnMsg = "no column(s) found"
logger.warn(warnMsg) logger.warn(warnMsg)
else: else:
columns = {} columns = {}
for column in threadData.shared.outputs: for column in threadData.shared.values:
result = inject.checkBooleanExpression("%s" % safeStringFormat("EXISTS(SELECT %s FROM %s WHERE ROUND(%s)=ROUND(%s))", (column, table, column, column))) result = inject.checkBooleanExpression("%s" % safeStringFormat("EXISTS(SELECT %s FROM %s WHERE ROUND(%s)=ROUND(%s))", (column, table, column, column)))
if result: if result:

View File

@ -158,7 +158,7 @@ def _oneShotErrorUse(expression, field=None):
return safecharencode(retVal) if kb.safeCharEncode else retVal return safecharencode(retVal) if kb.safeCharEncode else retVal
def _errorFields(expression, expressionFields, expressionFieldsList, num=None, emptyFields=None): def _errorFields(expression, expressionFields, expressionFieldsList, num=None, emptyFields=None):
outputs = [] values = []
origExpr = None origExpr = None
threadData = getCurrentThreadData() threadData = getCurrentThreadData()
@ -191,9 +191,9 @@ def _errorFields(expression, expressionFields, expressionFieldsList, num=None, e
if isinstance(num, int): if isinstance(num, int):
expression = origExpr expression = origExpr
outputs.append(output) values.append(output)
return outputs return values
def _errorReplaceChars(value): def _errorReplaceChars(value):
""" """
@ -237,7 +237,7 @@ def errorUse(expression, dump=False):
startLimit = 0 startLimit = 0
stopLimit = None stopLimit = None
output = None output = None
outputs = None value = None
_, _, _, _, _, expressionFieldsList, expressionFields, _ = agent.getFields(expression) _, _, _, _, _, expressionFieldsList, expressionFields, _ = agent.getFields(expression)
@ -290,21 +290,21 @@ def errorUse(expression, dump=False):
warnMsg += "return any output" warnMsg += "return any output"
logger.warn(warnMsg) logger.warn(warnMsg)
else: else:
outputs = [] # for empty tables value = [] # for empty tables
return outputs return value
if " ORDER BY " in expression and (stopLimit - startLimit) > SLOW_ORDER_COUNT_THRESHOLD: if " ORDER BY " in expression and (stopLimit - startLimit) > SLOW_ORDER_COUNT_THRESHOLD:
message = "due to huge table size do you want to remove " message = "due to huge table size do you want to remove "
message += "ORDER BY clause gaining speed over consistency? [y/N] " message += "ORDER BY clause gaining speed over consistency? [y/N] "
output = readInput(message, default="N") _ = readInput(message, default="N")
if output and output[0] in ("y", "Y"): if _ and _[0] in ("y", "Y"):
expression = expression[:expression.index(" ORDER BY ")] expression = expression[:expression.index(" ORDER BY ")]
threadData = getCurrentThreadData() threadData = getCurrentThreadData()
threadData.shared.limits = iter(xrange(startLimit, stopLimit)) threadData.shared.limits = iter(xrange(startLimit, stopLimit))
numThreads = min(conf.threads, (stopLimit - startLimit)) numThreads = min(conf.threads, (stopLimit - startLimit))
threadData.shared.outputs = BigArray() threadData.shared.value = BigArray()
threadData.shared.buffered = [] threadData.shared.buffered = []
threadData.shared.lastFlushed = startLimit - 1 threadData.shared.lastFlushed = startLimit - 1
@ -327,7 +327,7 @@ def errorUse(expression, dump=False):
threadData = getCurrentThreadData() threadData = getCurrentThreadData()
while kb.threadContinue: while kb.threadContinue:
with kb.locks.limits: with kb.locks.limit:
try: try:
num = threadData.shared.limits.next() num = threadData.shared.limits.next()
except StopIteration: except StopIteration:
@ -341,7 +341,7 @@ def errorUse(expression, dump=False):
if output and isListLike(output) and len(output) == 1: if output and isListLike(output) and len(output) == 1:
output = output[0] output = output[0]
with kb.locks.outputs: with kb.locks.value:
index = None index = None
for index in xrange(len(threadData.shared.buffered)): for index in xrange(len(threadData.shared.buffered)):
if threadData.shared.buffered[index][0] >= num: if threadData.shared.buffered[index][0] >= num:
@ -349,7 +349,7 @@ def errorUse(expression, dump=False):
threadData.shared.buffered.insert(index or 0, (num, output)) threadData.shared.buffered.insert(index or 0, (num, output))
while threadData.shared.buffered and threadData.shared.lastFlushed + 1 == threadData.shared.buffered[0][0]: while threadData.shared.buffered and threadData.shared.lastFlushed + 1 == threadData.shared.buffered[0][0]:
threadData.shared.lastFlushed += 1 threadData.shared.lastFlushed += 1
threadData.shared.outputs.append(threadData.shared.buffered[0][1]) threadData.shared.value.append(threadData.shared.buffered[0][1])
del threadData.shared.buffered[0] del threadData.shared.buffered[0]
runThreads(numThreads, errorThread) runThreads(numThreads, errorThread)
@ -361,15 +361,15 @@ def errorUse(expression, dump=False):
logger.warn(warnMsg) logger.warn(warnMsg)
finally: finally:
threadData.shared.outputs.extend(_[1] for _ in sorted(threadData.shared.buffered)) threadData.shared.value.extend(_[1] for _ in sorted(threadData.shared.buffered))
outputs = threadData.shared.outputs value = threadData.shared.value
kb.suppressResumeInfo = False kb.suppressResumeInfo = False
if not outputs and not abortedFlag: if not value and not abortedFlag:
outputs = _errorFields(expression, expressionFields, expressionFieldsList) value = _errorFields(expression, expressionFields, expressionFieldsList)
if outputs and isListLike(outputs) and len(outputs) == 1 and isinstance(outputs[0], basestring): if value and isListLike(value) and len(value) == 1 and isinstance(value[0], basestring):
outputs = outputs[0] value = value[0]
duration = calculateDeltaSeconds(start) duration = calculateDeltaSeconds(start)
@ -377,4 +377,4 @@ def errorUse(expression, dump=False):
debugMsg = "performed %d queries in %d seconds" % (kb.counters[kb.technique], duration) debugMsg = "performed %d queries in %d seconds" % (kb.counters[kb.technique], duration)
logger.debug(debugMsg) logger.debug(debugMsg)
return outputs return value

View File

@ -237,7 +237,7 @@ def unionUse(expression, unpack=True, dump=False):
threadData = getCurrentThreadData() threadData = getCurrentThreadData()
while kb.threadContinue: while kb.threadContinue:
with kb.locks.limits: with kb.locks.limit:
try: try:
num = threadData.shared.limits.next() num = threadData.shared.limits.next()
except StopIteration: except StopIteration:

View File

@ -34,13 +34,13 @@ class Crawler(object):
def getTargetUrls(self): def getTargetUrls(self):
try: try:
threadData = getCurrentThreadData() threadData = getCurrentThreadData()
threadData.shared.outputs = oset() threadData.shared.values = oset()
def crawlThread(): def crawlThread():
threadData = getCurrentThreadData() threadData = getCurrentThreadData()
while kb.threadContinue: while kb.threadContinue:
with kb.locks.limits: with kb.locks.limit:
if threadData.shared.unprocessed: if threadData.shared.unprocessed:
current = threadData.shared.unprocessed.pop() current = threadData.shared.unprocessed.pop()
else: else:
@ -79,10 +79,10 @@ class Crawler(object):
continue continue
if url.split('.')[-1].lower() not in CRAWL_EXCLUDE_EXTENSIONS: if url.split('.')[-1].lower() not in CRAWL_EXCLUDE_EXTENSIONS:
with kb.locks.outputs: with kb.locks.value:
threadData.shared.deeper.add(url) threadData.shared.deeper.add(url)
if re.search(r"(.*?)\?(.+)", url): if re.search(r"(.*?)\?(.+)", url):
threadData.shared.outputs.add(url) threadData.shared.values.add(url)
except UnicodeEncodeError: # for non-HTML files except UnicodeEncodeError: # for non-HTML files
pass pass
finally: finally:
@ -121,9 +121,9 @@ class Crawler(object):
finally: finally:
clearConsoleLine(True) clearConsoleLine(True)
if not threadData.shared.outputs: if not threadData.shared.values:
warnMsg = "no usable links found (with GET parameters)" warnMsg = "no usable links found (with GET parameters)"
logger.warn(warnMsg) logger.warn(warnMsg)
else: else:
for url in threadData.shared.outputs: for url in threadData.shared.values:
kb.targets.add(( url, None, None, None )) kb.targets.add(( url, None, None, None ))