minor speed optimizations (as a result of profiling)

This commit is contained in:
Miroslav Stampar 2011-08-12 13:40:37 +00:00
parent 36280b33fa
commit 10bdd90e60
5 changed files with 11 additions and 6 deletions

View File

@ -12,7 +12,7 @@ from lib.core.datatype import AttribDict
_defaults = { _defaults = {
"timeSec": 5, "timeSec": 5,
"googlePage": 1, "googlePage": 1,
"cpuThrottle": 10, "cpuThrottle": 5,
"verbose": 1, "verbose": 1,
"cDel": ";", "cDel": ";",
"delay": 0, "delay": 0,

View File

@ -1397,6 +1397,7 @@ def __setKnowledgeBaseAttributes(flushAll=True):
kb.delayCandidates = TIME_DELAY_CANDIDATES * [0] kb.delayCandidates = TIME_DELAY_CANDIDATES * [0]
kb.dep = None kb.dep = None
kb.docRoot = None kb.docRoot = None
kb.dumpMode = False
kb.dynamicMarkings = [] kb.dynamicMarkings = []
kb.dynamicParameters = False kb.dynamicParameters = False
kb.endDetection = False kb.endDetection = False

View File

@ -209,12 +209,11 @@ def decodePage(page, contentEncoding, contentType):
return page return page
def processResponse(page, responseHeaders): def processResponse(page, responseHeaders):
parseResponse(page, responseHeaders) if not kb.dumpMode:
parseResponse(page, responseHeaders)
if conf.parseErrors: if conf.parseErrors:
msg = extractErrorMessage(page) msg = extractErrorMessage(page)
if msg: if msg:
logger.info("parsed error message: '%s'" % msg) logger.info("parsed error message: '%s'" % msg)
return page

View File

@ -419,7 +419,7 @@ class Connect:
else: else:
debugMsg = "got HTTP error code: %d (%s)" % (code, status) debugMsg = "got HTTP error code: %d (%s)" % (code, status)
logger.debug(debugMsg) logger.debug(debugMsg)
page = processResponse(page, responseHeaders) processResponse(page, responseHeaders)
return page, responseHeaders return page, responseHeaders
except (urllib2.URLError, socket.error, socket.timeout, httplib.BadStatusLine, httplib.IncompleteRead), e: except (urllib2.URLError, socket.error, socket.timeout, httplib.BadStatusLine, httplib.IncompleteRead), e:
@ -465,7 +465,7 @@ class Connect:
finally: finally:
socket.setdefaulttimeout(conf.timeout) socket.setdefaulttimeout(conf.timeout)
page = processResponse(page, responseHeaders) processResponse(page, responseHeaders)
responseMsg += "[#%d] (%d %s):\n" % (threadData.lastRequestUID, code, status) responseMsg += "[#%d] (%d %s):\n" % (threadData.lastRequestUID, code, status)
if responseHeaders: if responseHeaders:

View File

@ -1549,6 +1549,8 @@ class Enumeration:
kb.data.cachedColumns = foundData kb.data.cachedColumns = foundData
try: try:
kb.dumpMode = True
if not safeSQLIdentificatorNaming(conf.db) in kb.data.cachedColumns \ if not safeSQLIdentificatorNaming(conf.db) in kb.data.cachedColumns \
or safeSQLIdentificatorNaming(tbl, True) not in \ or safeSQLIdentificatorNaming(tbl, True) not in \
kb.data.cachedColumns[safeSQLIdentificatorNaming(conf.db)] \ kb.data.cachedColumns[safeSQLIdentificatorNaming(conf.db)] \
@ -1768,6 +1770,9 @@ class Enumeration:
errMsg += "'%s'" % e errMsg += "'%s'" % e
logger.critical(errMsg) logger.critical(errMsg)
finally:
kb.dumpMode = False
def dumpAll(self): def dumpAll(self):
if conf.db is not None and conf.tbl is None: if conf.db is not None and conf.tbl is None:
self.dumpTable() self.dumpTable()