refactoring

lib/core/common.py

@@ -10,6 +10,7 @@ See the file 'doc/COPYING' for copying permission
 import codecs
 import ctypes
 import inspect
+import logging
 import os
 import random
 import re
@@ -600,23 +601,27 @@ def filePathToString(filePath):
 
     return strRepl
 
+def singleTimeLogMessage(message, level, flag):
+    if flag not in kb.singleLogFlags:
+        kb.singleLogFlags.add(flag)
+        logger.log(level, message)
+
 def dataToStdout(data, forceOutput=False):
     if not ('threadException' in kb and kb.threadException):
         if forceOutput or not getCurrentThreadData().disableStdOut:
             try:
                 if IS_WIN:
                     output = data.encode('ascii', errors="replace")
-                    if output != data and 'dataToStdout' not in kb.warningFlags:
+                    if output != data:
-                        kb.warningFlags.add('dataToStdout')
                         warnMsg  = "cannot properly display Unicode characters "
                         warnMsg += "inside Windows OS command prompt "
                         warnMsg += "(http://bugs.python.org/issue1602). all "
                         warnMsg += "similar occurances will result in "
                         warnMsg += "replacement with '?' character. please, find "
-                        warnMsg += "proper character representations inside "
+                        warnMsg += "proper character representation inside "
                         warnMsg += "coresponding output files. "
                         warnMsg += "p.s. FORMAT C: /U is highly recommended"
-                        logger.critical(warnMsg)
+                        singleTimeLogMessage(warnMsg, logging.WARN, 'dataToStdout')
                     sys.stdout.write(output)
                 else:
                     sys.stdout.write(data.encode(sys.stdout.encoding))

lib/core/option.py

@@ -1320,6 +1320,7 @@ def __setKnowledgeBaseAttributes(flushAll=True):
     kb.responseTimes   = []
     kb.resumedQueries  = {}
     kb.retriesCount    = 0
+    kb.singleLogFlags  = set()
     kb.skipOthersDbms  = None
     kb.suppressSession = False
     kb.technique       = None
@@ -1328,7 +1329,6 @@ def __setKnowledgeBaseAttributes(flushAll=True):
     kb.threadContinue  = True
     kb.threadException = False
     kb.threadData      = {}
-    kb.warningFlags    = set()
 
     kb.misc            = advancedDict()
     kb.misc.delimiter  = randomStr(length=6, lowercase=True)

lib/request/basic.py

@@ -9,6 +9,7 @@ See the file 'doc/COPYING' for copying permission
 
 import codecs
 import gzip
+import logging
 import os
 import re
 import StringIO
@@ -22,6 +23,7 @@ from lib.core.common import getUnicode
 from lib.core.common import isWindowsDriveLetterPath
 from lib.core.common import posixToNtSlashes
 from lib.core.common import sanitizeAsciiString
+from lib.core.common import singleTimeLogMessage
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
@@ -127,11 +129,9 @@ def checkCharEncoding(encoding):
     try:
         codecs.lookup(encoding)
     except LookupError:
-        if encoding not in kb.warningFlags:
-            kb.warningFlags.add(encoding)
         warnMsg  = "unknown web page charset '%s'. " % encoding
         warnMsg += "Please report by e-mail to %s." % ML
-            logger.warn(warnMsg)
+        singleTimeLogMessage(warnMsg, logging.WARN, encoding)
         encoding = None
 
     return encoding
@@ -143,10 +143,8 @@ def getHeuristicCharEncoding(page):
     """
     retVal = detect(page)['encoding']
 
-    if retVal not in kb.warningFlags:
-        kb.warningFlags.add(retVal)
     warnMsg  = "heuristics detected web page charset '%s'." % retVal
-        logger.warn(warnMsg)
+    singleTimeLogMessage(warnMsg, logging.WARN, retVal)
 
     return retVal