Merge branch 'master' of github.com:sqlmapproject/sqlmap

lib/controller/checks.py

@@ -36,8 +36,8 @@ from lib.core.common import readInput
 from lib.core.common import showStaticWords
 from lib.core.common import singleTimeLogMessage
 from lib.core.common import singleTimeWarnMessage
-from lib.core.common import wasLastRequestDBMSError
+from lib.core.common import wasLastResponseDBMSError
-from lib.core.common import wasLastRequestHTTPError
+from lib.core.common import wasLastResponseHTTPError
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
@@ -695,7 +695,7 @@ def heuristicCheckSqlInjection(place, parameter):
         logger.debug(debugMsg)
         return None
 
-    if wasLastRequestDBMSError():
+    if wasLastResponseDBMSError():
         debugMsg = "heuristic checking skipped "
         debugMsg += "because original page content "
         debugMsg += "contains DBMS error"
@@ -723,7 +723,7 @@ def heuristicCheckSqlInjection(place, parameter):
     page, _ = Request.queryPage(payload, place, content=True, raise404=False)
 
     parseFilePaths(page)
-    result = wasLastRequestDBMSError()
+    result = wasLastResponseDBMSError()
 
     infoMsg = "heuristic test shows that %s " % place
     infoMsg += "parameter '%s' might " % parameter
@@ -1083,14 +1083,14 @@ def checkConnection(suppressOutput=False):
 
         kb.errorIsNone = False
 
-        if not kb.originalPage and wasLastRequestHTTPError():
+        if not kb.originalPage and wasLastResponseHTTPError():
             errMsg = "unable to retrieve page content"
             raise SqlmapConnectionException(errMsg)
-        elif wasLastRequestDBMSError():
+        elif wasLastResponseDBMSError():
             warnMsg = "there is a DBMS error found in the HTTP response body "
             warnMsg += "which could interfere with the results of the tests"
             logger.warn(warnMsg)
-        elif wasLastRequestHTTPError():
+        elif wasLastResponseHTTPError():
             warnMsg = "the web server responded with an HTTP error code (%d) " % getLastRequestHTTPError()
             warnMsg += "which could interfere with the results of the tests"
             logger.warn(warnMsg)

lib/core/common.py

@@ -100,6 +100,7 @@ from lib.core.settings import IS_WIN
 from lib.core.settings import LARGE_OUTPUT_THRESHOLD
 from lib.core.settings import MIN_ENCODED_LEN_CHECK
 from lib.core.settings import MIN_TIME_RESPONSES
+from lib.core.settings import MIN_VALID_DELAYED_RESPONSE
 from lib.core.settings import ML
 from lib.core.settings import NULL
 from lib.core.settings import PARAMETER_AMP_MARKER
@@ -1878,7 +1879,7 @@ def popValue():
 
     return getCurrentThreadData().valueStack.pop()
 
-def wasLastRequestDBMSError():
+def wasLastResponseDBMSError():
     """
     Returns True if the last web request resulted in a (recognized) DBMS error page
     """
@@ -1886,7 +1887,7 @@ def wasLastRequestDBMSError():
     threadData = getCurrentThreadData()
     return threadData.lastErrorPage and threadData.lastErrorPage[0] == threadData.lastRequestUID
 
-def wasLastRequestHTTPError():
+def wasLastResponseHTTPError():
     """
     Returns True if the last web request resulted in an errornous HTTP code (like 500)
     """
@@ -1894,7 +1895,7 @@ def wasLastRequestHTTPError():
     threadData = getCurrentThreadData()
     return threadData.lastHTTPError and threadData.lastHTTPError[0] == threadData.lastRequestUID
 
-def wasLastRequestDelayed():
+def wasLastResponseDelayed():
     """
     Returns True if the last web request resulted in a time-delay
     """
@@ -1913,7 +1914,7 @@ def wasLastRequestDelayed():
             logger.warn(warnMsg)
 
         lowerStdLimit = average(kb.responseTimes) + TIME_STDEV_COEFF * deviation
-        retVal = (threadData.lastQueryDuration >= lowerStdLimit)
+        retVal = (threadData.lastQueryDuration >= max(MIN_VALID_DELAYED_RESPONSE, lowerStdLimit))
 
         if not kb.testMode and retVal:
             if kb.adjustTimeDelay is None:
@@ -1956,6 +1957,9 @@ def getLastRequestHTTPError():
 def extractErrorMessage(page):
     """
     Returns reported error message from page if it founds one
+
+    >>> extractErrorMessage(u'<html><title>Test</title>\\n<b>Warning</b>: oci_parse() [function.oci-parse]: ORA-01756: quoted string not properly terminated<br><p>Only a test page</p></html>')
+    u'oci_parse() [function.oci-parse]: ORA-01756: quoted string not properly terminated'
     """
 
     retVal = None
@@ -2022,7 +2026,14 @@ def urldecode(value, encoding=None, unsafe="%%&=;+%s" % CUSTOM_INJECTION_MARK_CH
     return result
 
 def urlencode(value, safe="%&=", convall=False, limit=False, spaceplus=False):
-    if conf.direct:
+    """
+    URL encodes given value
+
+    >>> urlencode('AND 1>(2+3)#')
+    'AND%201%3E%282%2B3%29%23'
+    """
+
+    if conf.get("direct"):
         return value
 
     count = 0
@@ -2104,6 +2115,9 @@ def getPageTemplate(payload, place):  # Cross-linked function
 def getPublicTypeMembers(type_, onlyValues=False):
     """
     Useful for getting members from types (e.g. in enums)
+
+    >>> [_ for _ in getPublicTypeMembers(OS, True)]
+    ['Linux', 'Windows']
     """
 
     for name, value in inspect.getmembers(type_):
@@ -2116,6 +2130,9 @@ def getPublicTypeMembers(type_, onlyValues=False):
 def enumValueToNameLookup(type_, value_):
     """
     Returns name of a enum member with a given value
+
+    >>> enumValueToNameLookup(SORT_ORDER, 100)
+    'LAST'
     """
 
     retVal = None
@@ -2131,11 +2148,14 @@ def extractRegexResult(regex, content, flags=0):
     """
     Returns 'result' group value from a possible match with regex on a given
     content
+
+    >>> extractRegexResult(r'a(?P<result>[^g]+)g', 'abcdefg')
+    'bcdef'
     """
 
     retVal = None
 
-    if regex and content and '?P<result>' in regex:
+    if regex and content and "?P<result>" in regex:
         match = re.search(regex, content, flags)
 
         if match:
@@ -2146,6 +2166,9 @@ def extractRegexResult(regex, content, flags=0):
 def extractTextTagContent(page):
     """
     Returns list containing content from "textual" tags
+
+    >>> extractTextTagContent(u'<html><head><title>Title</title></head><body><pre>foobar</pre><a href="#link">Link</a></body></html>')
+    [u'Title', u'foobar']
     """
 
     page = re.sub(r"(?si)[^\s>]*%s[^<]*" % REFLECTED_VALUE_MARKER, "", page or "")
@@ -2154,6 +2177,9 @@ def extractTextTagContent(page):
 def trimAlphaNum(value):
     """
     Trims alpha numeric characters from start and ending of a given value
+
+    >>> trimAlphaNum(u'AND 1>(2+3)-- foobar')
+    u' 1>(2+3)-- '
     """
 
     while value and value[-1].isalnum():
@@ -2167,14 +2193,26 @@ def trimAlphaNum(value):
 def isNumPosStrValue(value):
     """
     Returns True if value is a string (or integer) with a positive integer representation
+
+    >>> isNumPosStrValue(1)
+    True
+    >>> isNumPosStrValue('1')
+    True
+    >>> isNumPosStrValue(0)
+    False
+    >>> isNumPosStrValue('-2')
+    False
     """
 
-    return (value and isinstance(value, basestring) and value.isdigit() and value != "0") or (isinstance(value, int) and value != 0)
+    return (value and isinstance(value, basestring) and value.isdigit() and int(value) > 0) or (isinstance(value, int) and value > 0)
 
 @cachedmethod
 def aliasToDbmsEnum(dbms):
     """
     Returns major DBMS name from a given alias
+
+    >>> aliasToDbmsEnum('mssql')
+    'Microsoft SQL Server'
     """
 
     retVal = None
@@ -2251,22 +2289,28 @@ def removeDynamicContent(page):
 
     return page
 
-def filterStringValue(value, regex, replacement=""):
+def filterStringValue(value, charRegex, replacement=""):
     """
     Returns string value consisting only of chars satisfying supplied
     regular expression (note: it has to be in form [...])
+
+    >>> filterStringValue(u'wzydeadbeef0123#', r'[0-9a-f]')
+    u'deadbeef0123'
     """
 
     retVal = value
 
     if value:
-        retVal = re.sub(regex.replace("[", "[^") if "[^" not in regex else regex.replace("[^", "["), replacement, value)
+        retVal = re.sub(charRegex.replace("[", "[^") if "[^" not in charRegex else charRegex.replace("[^", "["), replacement, value)
 
     return retVal
 
 def filterControlChars(value):
     """
     Returns string value with control chars being supstituted with ' '
+
+    >>> filterControlChars(u'AND 1>(2+3)\\n--')
+    u'AND 1>(2+3) --'
     """
 
     return filterStringValue(value, PRINTABLE_CHAR_REGEX, ' ')
@@ -2397,6 +2441,9 @@ def initTechnique(technique=None):
 def arrayizeValue(value):
     """
     Makes a list out of value if it is not already a list or tuple itself
+
+    >>> arrayizeValue(u'1')
+    [u'1']
     """
 
     if not isListLike(value):
@@ -2407,6 +2454,9 @@ def arrayizeValue(value):
 def unArrayizeValue(value):
     """
     Makes a value out of iterable if it is a list or tuple itself
+
+    >>> unArrayizeValue([u'1'])
+    u'1'
     """
 
     if isListLike(value):
@@ -2417,6 +2467,9 @@ def unArrayizeValue(value):
 def flattenValue(value):
     """
     Returns an iterator representing flat representation of a given value
+
+    >>> [_ for _ in flattenValue([[u'1'], [[u'2'], u'3']])]
+    [u'1', u'2', u'3']
     """
 
     for i in iter(value):
@@ -2429,6 +2482,11 @@ def flattenValue(value):
 def isListLike(value):
     """
     Returns True if the given value is a list-like instance
+
+    >>> isListLike([1, 2, 3])
+    True
+    >>> isListLike(u'2')
+    False
     """
 
     return isinstance(value, (list, tuple, set, BigArray))
@@ -2464,6 +2522,9 @@ def filterListValue(value, regex):
     """
     Returns list with items that have parts satisfying given regular
     expression
+
+    >>> filterListValue(['users', 'admins', 'logs'], r'(users|admins)')
+    ['users', 'admins']
     """
 
     if isinstance(value, list) and regex:
@@ -2502,6 +2563,11 @@ def openFile(filename, mode='r'):
 def decodeIntToUnicode(value):
     """
     Decodes inferenced integer value to an unicode character
+
+    >>> decodeIntToUnicode(35)
+    u'#'
+    >>> decodeIntToUnicode(64)
+    u'@'
     """
     retVal = value
 
@@ -2592,6 +2658,9 @@ def getExceptionFrameLocals():
 def intersect(valueA, valueB, lowerCase=False):
     """
     Returns intersection of the array-ized values
+
+    >>> intersect([1, 2, 3], set([1,3]))
+    [1, 3]
     """
 
     retVal = None
@@ -2741,6 +2810,17 @@ def unsafeSQLIdentificatorNaming(name):
 def isNoneValue(value):
     """
     Returns whether the value is unusable (None or '')
+
+    >>> isNoneValue(None)
+    True
+    >>> isNoneValue('None')
+    True
+    >>> isNoneValue('')
+    True
+    >>> isNoneValue([])
+    True
+    >>> isNoneValue([2])
+    False
     """
 
     if isinstance(value, basestring):
@@ -2755,6 +2835,9 @@ def isNoneValue(value):
 def isNullValue(value):
     """
     Returns whether the value contains explicit 'NULL' value
+
+    >>> isNullValue(u'NULL')
+    True
     """
 
     return isinstance(value, basestring) and value.upper() == NULL
@@ -2846,13 +2929,18 @@ def safeCSValue(value):
     """
     Returns value safe for CSV dumping
     Reference: http://tools.ietf.org/html/rfc4180
+
+    >>> safeCSValue(u'foo, bar')
+    u'"foo, bar"'
+    >>> safeCSValue(u'foobar')
+    u'foobar'
     """
 
     retVal = value
 
     if retVal and isinstance(retVal, basestring):
         if not (retVal[0] == retVal[-1] == '"'):
-            if any(_ in retVal for _ in (conf.csvDel, '"', '\n')):
+            if any(_ in retVal for _ in (conf.get("csvDel", ','), '"', '\n')):
                 retVal = '"%s"' % retVal.replace('"', '""')
 
     return retVal
@@ -2860,6 +2948,9 @@ def safeCSValue(value):
 def filterPairValues(values):
     """
     Returns only list-like values with length 2
+
+    >>> filterPairValues([[1, 2], [3], 1, [4, 5]])
+    [[1, 2], [4, 5]]
     """
 
     retVal = []

lib/core/decorators.py

@@ -11,9 +11,11 @@ def cachedmethod(f, cache={}):
 
     Reference: http://code.activestate.com/recipes/325205-cache-decorator-in-python-24/
     """
+
     def _(*args, **kwargs):
         key = (f, tuple(args), frozenset(kwargs.items()))
         if key not in cache:
             cache[key] = f(*args, **kwargs)
         return cache[key]
+
     return _

lib/core/settings.py

@@ -64,6 +64,9 @@ CONCAT_VALUE_DELIMITER = '|'
 # Coefficient used for a time-based query delay checking (must be >= 7)
 TIME_STDEV_COEFF = 7
 
+# Minimum response time that can be even considered as delayed (not a complete requirement)
+MIN_VALID_DELAYED_RESPONSE = 0.5
+
 # Standard deviation after which a warning message should be displayed about connection lags
 WARN_TIME_STDEV = 0.5
 

lib/request/comparison.py

@@ -11,8 +11,8 @@ from lib.core.common import extractRegexResult
 from lib.core.common import getFilteredPageContent
 from lib.core.common import listToStrValue
 from lib.core.common import removeDynamicContent
-from lib.core.common import wasLastRequestDBMSError
+from lib.core.common import wasLastResponseDBMSError
-from lib.core.common import wasLastRequestHTTPError
+from lib.core.common import wasLastResponseHTTPError
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
@@ -77,7 +77,7 @@ def _comparison(page, headers, code, getRatioValue, pageLength):
 
     if page:
         # In case of an DBMS error page return None
-        if kb.errorIsNone and (wasLastRequestDBMSError() or wasLastRequestHTTPError()):
+        if kb.errorIsNone and (wasLastResponseDBMSError() or wasLastResponseHTTPError()):
             return None
 
         # Dynamic content lines to be excluded before comparison

lib/request/connect.py

@@ -34,7 +34,7 @@ from lib.core.common import readInput
 from lib.core.common import removeReflectiveValues
 from lib.core.common import singleTimeWarnMessage
 from lib.core.common import stdev
-from lib.core.common import wasLastRequestDelayed
+from lib.core.common import wasLastResponseDelayed
 from lib.core.common import unicodeencode
 from lib.core.common import urlencode
 from lib.core.data import conf
@@ -776,7 +776,7 @@ class Connect(object):
 
             elif not kb.testMode:
                 warnMsg = "it is very important not to stress the network adapter's "
-                warnMsg += "bandwidth during usage of time-based queries"
+                warnMsg += "bandwidth during usage of time-based payloads"
                 singleTimeWarnMessage(warnMsg)
 
         if conf.safUrl and conf.saFreq > 0:
@@ -827,7 +827,7 @@ class Connect(object):
             kb.testQueryCount += 1
 
         if timeBasedCompare:
-            return wasLastRequestDelayed()
+            return wasLastResponseDelayed()
         elif noteResponseTime:
             kb.responseTimes.append(threadData.lastQueryDuration)
 

lib/takeover/xp_cmdshell.py

@@ -18,7 +18,7 @@ from lib.core.common import pushValue
 from lib.core.common import popValue
 from lib.core.common import randomStr
 from lib.core.common import readInput
-from lib.core.common import wasLastRequestDelayed
+from lib.core.common import wasLastResponseDelayed
 from lib.core.convert import hexencode
 from lib.core.data import conf
 from lib.core.data import kb
@@ -94,7 +94,7 @@ class Xp_cmdshell:
         cmd = "ping -n %d 127.0.0.1" % (conf.timeSec * 2)
         self.xpCmdshellExecCmd(cmd)
 
-        return wasLastRequestDelayed()
+        return wasLastResponseDelayed()
 
     def _xpCmdshellTest(self):
         threadData = getCurrentThreadData()

lib/techniques/union/test.py

@@ -22,7 +22,7 @@ from lib.core.common import removeReflectiveValues
 from lib.core.common import singleTimeLogMessage
 from lib.core.common import singleTimeWarnMessage
 from lib.core.common import stdev
-from lib.core.common import wasLastRequestDBMSError
+from lib.core.common import wasLastResponseDBMSError
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
@@ -223,7 +223,7 @@ def _unionPosition(comment, place, parameter, prefix, suffix, count, where=PAYLO
                         logger.warn(warnMsg)
                         vector = (position, count, comment, prefix, suffix, kb.uChar, PAYLOAD.WHERE.NEGATIVE, kb.unionDuplicates)
 
-            unionErrorCase = kb.errorIsNone and wasLastRequestDBMSError()
+            unionErrorCase = kb.errorIsNone and wasLastResponseDBMSError()
 
             if unionErrorCase and count > 1:
                 warnMsg = "combined UNION/error-based SQL injection case found on "

lib/techniques/union/use.py

@@ -33,7 +33,7 @@ from lib.core.common import removeReflectiveValues
 from lib.core.common import singleTimeDebugMessage
 from lib.core.common import singleTimeWarnMessage
 from lib.core.common import unArrayizeValue
-from lib.core.common import wasLastRequestDBMSError
+from lib.core.common import wasLastResponseDBMSError
 from lib.core.convert import htmlunescape
 from lib.core.data import conf
 from lib.core.data import kb
@@ -94,7 +94,7 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
             retVal = getUnicode(retVal, kb.pageEncoding)
 
             # Special case when DBMS is Microsoft SQL Server and error message is used as a result of union injection
-            if Backend.isDbms(DBMS.MSSQL) and wasLastRequestDBMSError():
+            if Backend.isDbms(DBMS.MSSQL) and wasLastResponseDBMSError():
                 retVal = htmlunescape(retVal).replace("<br>", "\n")
 
             hashDBWrite("%s%s" % (conf.hexConvert, expression), retVal)

plugins/dbms/access/fingerprint.py

@@ -12,7 +12,7 @@ from lib.core.common import Format
 from lib.core.common import getCurrentThreadData
 from lib.core.common import randomInt
 from lib.core.common import randomStr
-from lib.core.common import wasLastRequestDBMSError
+from lib.core.common import wasLastResponseDBMSError
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
@@ -95,7 +95,7 @@ class Fingerprint(GenericFingerprint):
         randStr = randomStr()
         inject.checkBooleanExpression("EXISTS(SELECT * FROM %s.%s WHERE %d=%d)" % (randStr, randStr, randInt, randInt))
 
-        if wasLastRequestDBMSError():
+        if wasLastResponseDBMSError():
             threadData = getCurrentThreadData()
             match = re.search("Could not find file\s+'([^']+?)'", threadData.lastErrorPage[1])