diff --git a/lib/core/common.py b/lib/core/common.py index 96eb25297..45aeb1997 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -99,6 +99,7 @@ from lib.core.settings import INVALID_UNICODE_CHAR_FORMAT from lib.core.settings import ISSUES_PAGE from lib.core.settings import IS_WIN from lib.core.settings import LARGE_OUTPUT_THRESHOLD +from lib.core.settings import MIN_ENCODED_LEN_CHECK from lib.core.settings import MIN_TIME_RESPONSES from lib.core.settings import ML from lib.core.settings import NULL @@ -570,7 +571,7 @@ def paramToDict(place, parameters=None): for encoding in ("hex", "base64"): try: decoded = value.decode(encoding) - if all(_ in string.printable for _ in decoded): + if len(decoded) > MIN_ENCODED_LEN_CHECK and all(_ in string.printable for _ in decoded): warnMsg = "provided parameter '%s' " % parameter warnMsg += "seems to be '%s' encoded" % encoding logger.warn(warnMsg) @@ -768,13 +769,6 @@ def dataToOutFile(filename, data): return retVal -def strToHex(value): - """ - Converts string value to it's hexadecimal representation - """ - - return (value if not isinstance(value, unicode) else value.encode(UNICODE_ENCODING)).encode("hex").upper() - def readInput(message, default=None, checkBatch=True): """ Reads input from terminal @@ -1313,20 +1307,6 @@ def getCharset(charsetType=None): return asciiTbl -def searchEnvPath(filename): - retVal = None - path = os.environ.get("PATH", "") - paths = path.split(";") if IS_WIN else path.split(":") - - for _ in paths: - _ = _.replace(";", "") - retVal = os.path.exists(os.path.normpath(os.path.join(_, filename))) - - if retVal: - break - - return retVal - def directoryPath(filepath): """ Returns directory path for a given filepath @@ -1434,13 +1414,6 @@ def showStaticWords(firstPage, secondPage): logger.info(infoMsg) -def isWindowsPath(filepath): - """ - Returns True if given filepath is in Windows format - """ - - return re.search("\A[\w]\:\\\\", filepath) is not None - def isWindowsDriveLetterPath(filepath): """ Returns True if given filepath starts with a Windows drive letter @@ -1470,18 +1443,6 @@ def ntToPosixSlashes(filepath): return filepath.replace('\\', '/') -def isBase64EncodedString(subject): - """ - Checks if the provided string is Base64 encoded - - >>> isBase64EncodedString('dGVzdA==') - True - >>> isBase64EncodedString('123456') - False - """ - - return re.match(r"\A(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?\Z", subject) is not None - def isHexEncodedString(subject): """ Checks if the provided string is hex encoded @@ -2485,20 +2446,6 @@ def showHttpErrorCodes(): for code, count in kb.httpErrorCodes.items()) logger.warn(warnMsg) -def getComparePageRatio(firstPage, secondPage, filtered=False): - """ - Returns comparison ratio between two given pages - """ - - if filtered: - (firstPage, secondPage) = map(getFilteredPageContent, (firstPage, secondPage)) - - seqMatcher = getCurrentThreadData().seqMatcher - seqMatcher.set_seq1(firstPage) - seqMatcher.set_seq2(secondPage) - - return seqMatcher.quick_ratio() - def openFile(filename, mode='r'): """ Returns file handle of a given filename @@ -2752,16 +2699,6 @@ def unsafeSQLIdentificatorNaming(name): return retVal -def isBinaryData(value): - """ - Tests given value for binary content - """ - - retVal = False - if isinstance(value, basestring): - retVal = reduce(lambda x, y: x or not (y in string.printable or ord(y) > 255), value, False) - return retVal - def isNoneValue(value): """ Returns whether the value is unusable (None or '') diff --git a/lib/core/convert.py b/lib/core/convert.py index 792b6ea31..48c904e9e 100644 --- a/lib/core/convert.py +++ b/lib/core/convert.py @@ -47,25 +47,6 @@ def hexdecode(value): def hexencode(value): return utf8encode(value).encode("hex") -def md5hash(value): - if "hashlib" in sys.modules: - return hashlib.md5(value).hexdigest() - else: - return md5.new(value).hexdigest() - -def orddecode(value): - packedString = struct.pack("!" + "I" * len(value), *value) - return "".join(chr(char) for char in struct.unpack("!" + "I" * (len(packedString) / 4), packedString)) - -def ordencode(value): - return tuple(ord(char) for char in value) - -def sha1hash(value): - if "hashlib" in sys.modules: - return hashlib.sha1(value).hexdigest() - else: - return sha.new(value).hexdigest() - def unicodeencode(value, encoding=None): """ Return 8-bit string representation of the supplied unicode value: diff --git a/lib/core/option.py b/lib/core/option.py index 25e40d2fa..f71ea9b6b 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -131,7 +131,6 @@ from lib.parse.payloads import loadPayloads from lib.request.basic import checkCharEncoding from lib.request.connect import Connect as Request from lib.request.dns import DNSServer -from lib.request.proxy import ProxyHTTPSHandler from lib.request.basicauthhandler import SmartHTTPBasicAuthHandler from lib.request.certhandler import HTTPSCertAuthHandler from lib.request.httpshandler import HTTPSHandler @@ -970,17 +969,7 @@ def _setHTTPProxy(): proxyString = "" proxyString += "%s:%d" % (hostname, port) - - # Workaround for http://bugs.python.org/issue1424152 (urllib/urllib2: - # HTTPS over (Squid) Proxy fails) as long as HTTP over SSL requests - # can't be tunneled over an HTTP proxy natively by Python (<= 2.5) - # urllib2 standard library - if PYVERSION >= "2.6": - proxyHandler = urllib2.ProxyHandler({"http": proxyString, "https": proxyString}) - elif conf.scheme == "https": - proxyHandler = ProxyHTTPSHandler(proxyString) - else: - proxyHandler = urllib2.ProxyHandler({"http": proxyString}) + proxyHandler = urllib2.ProxyHandler({"http": proxyString, "https": proxyString}) def _setSafeUrl(): """ diff --git a/lib/core/settings.py b/lib/core/settings.py index e6c075e6e..a5a8f5d4a 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -506,6 +506,9 @@ MIN_BINARY_DISK_DUMP_SIZE = 100 # Regular expression used for extracting form tags FORM_SEARCH_REGEX = r"(?si)" +# Minimum field entry length needed for encoded content (hex, base64,...) check +MIN_ENCODED_LEN_CHECK = 5 + # CSS style used in HTML dump format HTML_DUMP_CSS_STYLE = """