diff --git a/lib/request/basic.py b/lib/request/basic.py index 90f21f3f9..3cdaf3460 100755 --- a/lib/request/basic.py +++ b/lib/request/basic.py @@ -17,6 +17,7 @@ from lib.core.common import extractErrorMessage from lib.core.common import extractRegexResult from lib.core.common import getPublicTypeMembers from lib.core.common import getUnicode +from lib.core.common import randomStr from lib.core.common import readInput from lib.core.common import resetCookieJar from lib.core.common import singleTimeLogMessage @@ -206,6 +207,15 @@ def checkCharEncoding(encoding, warn=True): singleTimeLogMessage(warnMsg, logging.WARN, encoding) encoding = None + if encoding: + try: + unicode(randomStr(), encoding) + except: + if warn: + warnMsg = "invalid web page charset '%s'" % encoding + singleTimeLogMessage(warnMsg, logging.WARN, encoding) + encoding = None + return encoding def getHeuristicCharEncoding(page):