refactoring

This commit is contained in:
Miroslav Stampar 2011-04-19 22:54:13 +00:00
parent de2479b864
commit 3b133303bf
3 changed files with 17 additions and 14 deletions

View File

@ -10,6 +10,7 @@ See the file 'doc/COPYING' for copying permission
import codecs
import ctypes
import inspect
import logging
import os
import random
import re
@ -600,23 +601,27 @@ def filePathToString(filePath):
return strRepl
def singleTimeLogMessage(message, level, flag):
if flag not in kb.singleLogFlags:
kb.singleLogFlags.add(flag)
logger.log(level, message)
def dataToStdout(data, forceOutput=False):
if not ('threadException' in kb and kb.threadException):
if forceOutput or not getCurrentThreadData().disableStdOut:
try:
if IS_WIN:
output = data.encode('ascii', errors="replace")
if output != data and 'dataToStdout' not in kb.warningFlags:
kb.warningFlags.add('dataToStdout')
if output != data:
warnMsg = "cannot properly display Unicode characters "
warnMsg += "inside Windows OS command prompt "
warnMsg += "(http://bugs.python.org/issue1602). all "
warnMsg += "similar occurances will result in "
warnMsg += "replacement with '?' character. please, find "
warnMsg += "proper character representations inside "
warnMsg += "proper character representation inside "
warnMsg += "coresponding output files. "
warnMsg += "p.s. FORMAT C: /U is highly recommended"
logger.critical(warnMsg)
singleTimeLogMessage(warnMsg, logging.WARN, 'dataToStdout')
sys.stdout.write(output)
else:
sys.stdout.write(data.encode(sys.stdout.encoding))

View File

@ -1320,6 +1320,7 @@ def __setKnowledgeBaseAttributes(flushAll=True):
kb.responseTimes = []
kb.resumedQueries = {}
kb.retriesCount = 0
kb.singleLogFlags = set()
kb.skipOthersDbms = None
kb.suppressSession = False
kb.technique = None
@ -1328,7 +1329,6 @@ def __setKnowledgeBaseAttributes(flushAll=True):
kb.threadContinue = True
kb.threadException = False
kb.threadData = {}
kb.warningFlags = set()
kb.misc = advancedDict()
kb.misc.delimiter = randomStr(length=6, lowercase=True)

View File

@ -9,6 +9,7 @@ See the file 'doc/COPYING' for copying permission
import codecs
import gzip
import logging
import os
import re
import StringIO
@ -22,6 +23,7 @@ from lib.core.common import getUnicode
from lib.core.common import isWindowsDriveLetterPath
from lib.core.common import posixToNtSlashes
from lib.core.common import sanitizeAsciiString
from lib.core.common import singleTimeLogMessage
from lib.core.data import conf
from lib.core.data import kb
from lib.core.data import logger
@ -127,11 +129,9 @@ def checkCharEncoding(encoding):
try:
codecs.lookup(encoding)
except LookupError:
if encoding not in kb.warningFlags:
kb.warningFlags.add(encoding)
warnMsg = "unknown web page charset '%s'. " % encoding
warnMsg += "Please report by e-mail to %s." % ML
logger.warn(warnMsg)
singleTimeLogMessage(warnMsg, logging.WARN, encoding)
encoding = None
return encoding
@ -143,10 +143,8 @@ def getHeuristicCharEncoding(page):
"""
retVal = detect(page)['encoding']
if retVal not in kb.warningFlags:
kb.warningFlags.add(retVal)
warnMsg = "heuristics detected web page charset '%s'." % retVal
logger.warn(warnMsg)
singleTimeLogMessage(warnMsg, logging.WARN, retVal)
return retVal