sqlmapproject/sqlmap
1
1
Fork 0
mirror of https://github.com/sqlmapproject/sqlmap.git synced 2025-07-24 23:19:45 +03:00
Code Issues Packages Projects Releases Wiki Activity

rewriting --eval / fixing json-like bugs

Browse Source
This commit is contained in:
Rend 2022-03-06 22:04:17 +03:30
parent bacf18832a
commit 08ac50d4d5
51 changed files with 6125 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lib/core/agent.py
View File

@ -129,7 +129,7 @@ class Agent(object):
if kb.postHint in (POST_HINT.SOAP, POST_HINT.XML):
origValue = re.split(r"['\">]", origValue)[-1]
elif kb.postHint in (POST_HINT.JSON, POST_HINT.JSON_LIKE):
origValue = extractRegexResult(r"(?s)\"\s*:\s*(?P<result>\d+\Z)", origValue) or extractRegexResult(r'(?s)[\s:]*(?P<result>[^"\[,]+\Z)', origValue)
origValue = (re.findall(r'\d+\Z', origValue) or re.findall(r'[^\'"\[,]+\Z', origValue))[0]
else:
_ = extractRegexResult(r"(?s)(?P<result>[^\s<>{}();'\"&]+\Z)", origValue) or ""
origValue = _.split('=', 1)[1] if '=' in _ else ""
@ -197,7 +197,10 @@ class Agent(object):
if kb.postHint == POST_HINT.JSON and not isNumber(newValue) and '"%s"' % _ not in paramString:
newValue = '"%s"' % self.addPayloadDelimiters(newValue)
elif kb.postHint == POST_HINT.JSON_LIKE and not isNumber(newValue) and re.search(r"['\"]%s['\"]" % re.escape(_), paramString) is None:
newValue = "'%s'" % self.addPayloadDelimiters(newValue)
if re.search(r":\s*'", conf.data):
newValue = "'%s'" % self.addPayloadDelimiters(newValue)
else:
newValue = '"%s"' % self.addPayloadDelimiters(newValue)
else:
newValue = self.addPayloadDelimiters(newValue)

185
lib/request/connect.py
View File

@ -6,7 +6,9 @@ See the file 'LICENSE' for copying permission
"""
import binascii
import copy
import inspect
import json
import logging
import os
import random
@ -141,6 +143,8 @@ from thirdparty.six import unichr as _unichr
from thirdparty.six.moves import http_client as _http_client
from thirdparty.six.moves import urllib as _urllib
from thirdparty.socks.socks import ProxyError
from thirdparty.requests_toolbelt.multipart.decoder import MultipartDecoder
from thirdparty.xmltodict import xmltodict
class Connect(object):
"""
@ -1009,6 +1013,15 @@ class Connect(object):
contentType = POST_HINT_CONTENT_TYPES.get(kb.postHint, PLAIN_TEXT_CONTENT_TYPE)
conf.httpHeaders.append((HTTP_HEADER.CONTENT_TYPE, contentType))
if kb.postHint == POST_HINT.JSON_LIKE:
json_like_type = 1 # {"key": "value"}
if re.search(r"'\s*:", conf.data):
json_like_type = 2 # {'key': 'value'}
elif re.search(r':\s*"', conf.data):
json_like_type = 3 # {key: "value"}
elif re.search(r":\s*'", conf.data):
json_like_type = 4 # {key: 'value'}
if payload:
delimiter = conf.paramDel or (DEFAULT_GET_POST_DELIMITER if place != PLACE.COOKIE else DEFAULT_COOKIE_DELIMITER)
@ -1052,9 +1065,11 @@ class Connect(object):
elif kb.postHint == POST_HINT.JSON:
payload = escapeJsonValue(payload)
elif kb.postHint == POST_HINT.JSON_LIKE:
payload = payload.replace("'", REPLACEMENT_MARKER).replace('"', "'").replace(REPLACEMENT_MARKER, '"')
if json_like_type in (2, 4):
payload = payload.replace("'", REPLACEMENT_MARKER).replace('"', "'").replace(REPLACEMENT_MARKER, '"')
payload = escapeJsonValue(payload)
payload = payload.replace("'", REPLACEMENT_MARKER).replace('"', "'").replace(REPLACEMENT_MARKER, '"')
if json_like_type in (2, 4):
payload = payload.replace("'", REPLACEMENT_MARKER).replace('"', "'").replace(REPLACEMENT_MARKER, '"')
value = agent.replacePayload(value, payload)
else:
# GET, POST, URI and Cookie payload needs to be thoroughly URL encoded
@ -1272,121 +1287,81 @@ class Connect(object):
if conf.evalCode:
delimiter = conf.paramDel or DEFAULT_GET_POST_DELIMITER
variables = {"uri": uri, "lastPage": threadData.lastPage, "_locals": locals(), "cookie": cookie}
originals = {}
variables = {"uri": uri, "get_query": get, "headers": headers, "post_body": post, "get_data": {}, "post_data": {}, "lastPage": threadData.lastPage, "_locals": locals()}
if not get and PLACE.URI in conf.parameters:
query = _urllib.parse.urlsplit(uri).query or ""
else:
query = None
original_get = get
original_post = post
for item in filterNone((get, post if not kb.postHint else None, query)):
for part in item.split(delimiter):
if get:
for part in get.split(delimiter):
if '=' in part:
name, value = part.split('=', 1)
name = name.strip()
if safeVariableNaming(name) != name:
conf.evalCode = re.sub(r"\b%s\b" % re.escape(name), safeVariableNaming(name), conf.evalCode)
name = safeVariableNaming(name)
value = urldecode(value, convall=True, spaceplus=(item == post and kb.postSpaceToPlus))
variables[name] = value
if cookie:
for part in cookie.split(conf.cookieDel or DEFAULT_COOKIE_DELIMITER):
if '=' in part:
name, value = part.split('=', 1)
name = name.strip()
if safeVariableNaming(name) != name:
conf.evalCode = re.sub(r"\b%s\b" % re.escape(name), safeVariableNaming(name), conf.evalCode)
name = safeVariableNaming(name)
value = urldecode(value, convall=True)
variables[name] = value
variables['get_data'][name] = value
while True:
try:
compile(getBytes(re.sub(r"\s*;\s*", "\n", conf.evalCode)), "", "exec")
except SyntaxError as ex:
if ex.text:
original = replacement = ex.text.strip()
if kb.postHint:
if kb.postHint in (POST_HINT.XML, POST_HINT.SOAP):
variables['post_data'] = xmltodict.parse(post)
if kb.postHint == POST_HINT.JSON:
variables['post_data'] = json.loads(post)
if kb.postHint == POST_HINT.JSON_LIKE:
if json_like_type == 3:
post = re.sub(r'(,|\{)\s*([^\'\s{,]+)\s*:', '\g<1>"\g<2>":', post)
if json_like_type == 4:
post = re.sub(r'(,|\{)\s*([^\'\s{,]+)\s*:', "\g<1>'\g<2>':", post)
if json_like_type in (2, 4):
post = post.replace("\\'", REPLACEMENT_MARKER).replace('\"','\\"').replace("'",'"').replace(REPLACEMENT_MARKER, "'")
variables['post_data'] = json.loads(post)
if kb.postHint == POST_HINT.MULTIPART:
multipart = MultipartDecoder(bytes(post, 'utf-8'), contentType)
boundary = '--' + multipart.boundary.decode('utf-8')
for part in multipart.parts:
name = re.search(r'"([^\"]*)"', part.headers._store[b'content-disposition'][1].decode('utf-8')).group(1)
value = part.text
variables['post_data'][name] = value
if kb.postHint == POST_HINT.ARRAY_LIKE:
post = re.sub(r"\A%s" % delimiter, "", post)
array_name = re.findall(r"%s(.*?)\[\]=" % delimiter, post)[0].strip()
variables['post_data'] = []
for value in post.split("%s[]=" % array_name)[1:]:
variables['post_data'].append(value.replace(delimiter, ""))
elif post:
for part in post.split(delimiter):
if '=' in part:
name, value = part.split('=', 1)
name = name.strip()
value = urldecode(value, convall=True, spaceplus=kb.postSpaceToPlus)
variables['post_data'][name] = value
if '=' in original:
name, value = original.split('=', 1)
name = name.strip()
if safeVariableNaming(name) != name:
replacement = re.sub(r"\b%s\b" % re.escape(name), safeVariableNaming(name), replacement)
else:
for _ in re.findall(r"[A-Za-z_]+", original)[::-1]:
if safeVariableNaming(_) != _:
replacement = replacement.replace(_, safeVariableNaming(_))
break
if original == replacement:
conf.evalCode = conf.evalCode.replace(EVALCODE_ENCODED_PREFIX, "")
break
else:
conf.evalCode = conf.evalCode.replace(getUnicode(ex.text.strip(), UNICODE_ENCODING), replacement)
else:
break
else:
break
originals.update(variables)
evaluateCode(conf.evalCode, variables)
for variable in list(variables.keys()):
if unsafeVariableNaming(variable) != variable:
value = variables[variable]
del variables[variable]
variables[unsafeVariableNaming(variable)] = value
if kb.postHint:
if kb.postHint in (POST_HINT.XML, POST_HINT.SOAP):
post = xmltodict.unparse(variables['post_data'])
if kb.postHint == POST_HINT.JSON:
post = json.dumps(variables['post_data'])
if kb.postHint == POST_HINT.JSON_LIKE:
post = json.dumps(variables['post_data'])
if json_like_type in (3, 4):
post = re.sub(r'"([^"]+)":', '\g<1>:', post)
if json_like_type in (2, 4):
post = post.replace('\\"', REPLACEMENT_MARKER).replace("'", "\\'").replace('"', "'").replace(REPLACEMENT_MARKER, '"')
if kb.postHint == POST_HINT.MULTIPART:
for name, value in variables['post_data'].items():
post = re.sub(r"(?s)(name=\"%s\"(?:; ?filename=.+?)?\r\n\r\n).*?(%s)" % (name, boundary), r"\g<1>%s\r\n\g<2>" % value.replace('\\', r'\\'), post)
if kb.postHint == POST_HINT.ARRAY_LIKE:
post = array_name + "[]=" + (delimiter + array_name + "[]=").join(variables['post_data'])
else:
post = delimiter.join(f'{key}={value}' for key, value in variables['post_data'].items())
uri = variables["uri"]
cookie = variables["cookie"]
uri = variables['uri']
get = delimiter.join(f'{key}={value}' for key, value in variables['get_data'].items())
auxHeaders.update(variables['headers'])
cookie = variables['headers']['Cookie'] if 'Cookie' in variables['headers'] else None
for name, value in variables.items():
if name != "__builtins__" and originals.get(name, "") != value:
if isinstance(value, (int, float, six.string_types, six.binary_type)):
found = False
value = getUnicode(value, UNICODE_ENCODING)
if kb.postHint and re.search(r"\b%s\b" % re.escape(name), post or ""):
if kb.postHint in (POST_HINT.XML, POST_HINT.SOAP):
if re.search(r"<%s\b" % re.escape(name), post):
found = True
post = re.sub(r"(?s)(<%s\b[^>]*>)(.*?)(</%s)" % (re.escape(name), re.escape(name)), r"\g<1>%s\g<3>" % value.replace('\\', r'\\'), post)
elif re.search(r"\b%s>" % re.escape(name), post):
found = True
post = re.sub(r"(?s)(\b%s>)(.*?)(</[^<]*\b%s>)" % (re.escape(name), re.escape(name)), r"\g<1>%s\g<3>" % value.replace('\\', r'\\'), post)
regex = r"\b(%s)\b([^\w]+)(\w+)" % re.escape(name)
if not found and re.search(regex, (post or "")):
found = True
post = re.sub(regex, r"\g<1>\g<2>%s" % value.replace('\\', r'\\'), post)
regex = r"((\A|%s)%s=).+?(%s|\Z)" % (re.escape(delimiter), re.escape(name), re.escape(delimiter))
if not found and re.search(regex, (post or "")):
found = True
post = re.sub(regex, r"\g<1>%s\g<3>" % value.replace('\\', r'\\'), post)
if re.search(regex, (get or "")):
found = True
get = re.sub(regex, r"\g<1>%s\g<3>" % value.replace('\\', r'\\'), get)
if re.search(regex, (query or "")):
found = True
uri = re.sub(regex.replace(r"\A", r"\?"), r"\g<1>%s\g<3>" % value.replace('\\', r'\\'), uri)
regex = r"((\A|%s)%s=).+?(%s|\Z)" % (re.escape(conf.cookieDel or DEFAULT_COOKIE_DELIMITER), re.escape(name), re.escape(conf.cookieDel or DEFAULT_COOKIE_DELIMITER))
if re.search(regex, (cookie or "")):
found = True
cookie = re.sub(regex, r"\g<1>%s\g<3>" % value.replace('\\', r'\\'), cookie)
if not found:
if post is not None:
post += "%s%s=%s" % (delimiter, name, value)
elif get is not None:
get += "%s%s=%s" % (delimiter, name, value)
elif cookie is not None:
cookie += "%s%s=%s" % (conf.cookieDel or DEFAULT_COOKIE_DELIMITER, name, value)
get = variables['get_query'] if variables['get_query'] != original_get else get
post = variables['post_body'] if variables['post_body'] != original_post else post
if not conf.skipUrlEncode:
get = urlencode(get, limit=True)

34
thirdparty/requests_toolbelt/__init__.py vendored Normal file
View File

@ -0,0 +1,34 @@
# -*- coding: utf-8 -*-
"""
requests-toolbelt
=================
See http://toolbelt.rtfd.org/ for documentation
:copyright: (c) 2014 by Ian Cordasco and Cory Benfield
:license: Apache v2.0, see LICENSE for more details
"""
from .adapters import SSLAdapter, SourceAddressAdapter
from .auth.guess import GuessAuth
from .multipart import (
MultipartEncoder, MultipartEncoderMonitor, MultipartDecoder,
ImproperBodyPartContentException, NonMultipartContentTypeException
)
from .streaming_iterator import StreamingIterator
from .utils.user_agent import user_agent
__title__ = 'requests-toolbelt'
__authors__ = 'Ian Cordasco, Cory Benfield'
__license__ = 'Apache v2.0'
__copyright__ = 'Copyright 2014 Ian Cordasco, Cory Benfield'
__version__ = '0.9.1'
__version_info__ = tuple(int(i) for i in __version__.split('.'))
__all__ = [
'GuessAuth', 'MultipartEncoder', 'MultipartEncoderMonitor',
'MultipartDecoder', 'SSLAdapter', 'SourceAddressAdapter',
'StreamingIterator', 'user_agent', 'ImproperBodyPartContentException',
'NonMultipartContentTypeException', '__title__', '__authors__',
'__license__', '__copyright__', '__version__', '__version_info__',
]

324
thirdparty/requests_toolbelt/_compat.py vendored Normal file
View File

@ -0,0 +1,324 @@
"""Private module full of compatibility hacks.
Primarily this is for downstream redistributions of requests that unvendor
urllib3 without providing a shim.
.. warning::
This module is private. If you use it, and something breaks, you were
warned
"""
import sys
import requests
try:
from requests.packages.urllib3 import fields
from requests.packages.urllib3 import filepost
from requests.packages.urllib3 import poolmanager
except ImportError:
from urllib3 import fields
from urllib3 import filepost
from urllib3 import poolmanager
try:
from requests.packages.urllib3.connection import HTTPConnection
from requests.packages.urllib3 import connection
except ImportError:
try:
from urllib3.connection import HTTPConnection
from urllib3 import connection
except ImportError:
HTTPConnection = None
connection = None
if requests.__build__ < 0x020300:
timeout = None
else:
try:
from requests.packages.urllib3.util import timeout
except ImportError:
from urllib3.util import timeout
if requests.__build__ < 0x021000:
gaecontrib = None
else:
try:
from requests.packages.urllib3.contrib import appengine as gaecontrib
except ImportError:
from urllib3.contrib import appengine as gaecontrib
if requests.__build__ < 0x021200:
PyOpenSSLContext = None
else:
try:
from requests.packages.urllib3.contrib.pyopenssl \
import PyOpenSSLContext
except ImportError:
try:
from urllib3.contrib.pyopenssl import PyOpenSSLContext
except ImportError:
PyOpenSSLContext = None
PY3 = sys.version_info > (3, 0)
if PY3:
from collections.abc import Mapping, MutableMapping
import queue
from urllib.parse import urlencode, urljoin
else:
from collections import Mapping, MutableMapping
import Queue as queue
from urllib import urlencode
from urlparse import urljoin
try:
basestring = basestring
except NameError:
basestring = (str, bytes)
class HTTPHeaderDict(MutableMapping):
"""
:param headers:
An iterable of field-value pairs. Must not contain multiple field names
when compared case-insensitively.
:param kwargs:
Additional field-value pairs to pass in to ``dict.update``.
A ``dict`` like container for storing HTTP Headers.
Field names are stored and compared case-insensitively in compliance with
RFC 7230. Iteration provides the first case-sensitive key seen for each
case-insensitive pair.
Using ``__setitem__`` syntax overwrites fields that compare equal
case-insensitively in order to maintain ``dict``'s api. For fields that
compare equal, instead create a new ``HTTPHeaderDict`` and use ``.add``
in a loop.
If multiple fields that are equal case-insensitively are passed to the
constructor or ``.update``, the behavior is undefined and some will be
lost.
>>> headers = HTTPHeaderDict()
>>> headers.add('Set-Cookie', 'foo=bar')
>>> headers.add('set-cookie', 'baz=quxx')
>>> headers['content-length'] = '7'
>>> headers['SET-cookie']
'foo=bar, baz=quxx'
>>> headers['Content-Length']
'7'
"""
def __init__(self, headers=None, **kwargs):
super(HTTPHeaderDict, self).__init__()
self._container = {}
if headers is not None:
if isinstance(headers, HTTPHeaderDict):
self._copy_from(headers)
else:
self.extend(headers)
if kwargs:
self.extend(kwargs)
def __setitem__(self, key, val):
self._container[key.lower()] = (key, val)
return self._container[key.lower()]
def __getitem__(self, key):
val = self._container[key.lower()]
return ', '.join(val[1:])
def __delitem__(self, key):
del self._container[key.lower()]
def __contains__(self, key):
return key.lower() in self._container
def __eq__(self, other):
if not isinstance(other, Mapping) and not hasattr(other, 'keys'):
return False
if not isinstance(other, type(self)):
other = type(self)(other)
return (dict((k.lower(), v) for k, v in self.itermerged()) ==
dict((k.lower(), v) for k, v in other.itermerged()))
def __ne__(self, other):
return not self.__eq__(other)
if not PY3: # Python 2
iterkeys = MutableMapping.iterkeys
itervalues = MutableMapping.itervalues
__marker = object()
def __len__(self):
return len(self._container)
def __iter__(self):
# Only provide the originally cased names
for vals in self._container.values():
yield vals[0]
def pop(self, key, default=__marker):
"""D.pop(k[,d]) -> v, remove specified key and return its value.
If key is not found, d is returned if given, otherwise KeyError is
raised.
"""
# Using the MutableMapping function directly fails due to the private
# marker.
# Using ordinary dict.pop would expose the internal structures.
# So let's reinvent the wheel.
try:
value = self[key]
except KeyError:
if default is self.__marker:
raise
return default
else:
del self[key]
return value
def discard(self, key):
try:
del self[key]
except KeyError:
pass
def add(self, key, val):
"""Adds a (name, value) pair, doesn't overwrite the value if it already
exists.
>>> headers = HTTPHeaderDict(foo='bar')
>>> headers.add('Foo', 'baz')
>>> headers['foo']
'bar, baz'
"""
key_lower = key.lower()
new_vals = key, val
# Keep the common case aka no item present as fast as possible
vals = self._container.setdefault(key_lower, new_vals)
if new_vals is not vals:
# new_vals was not inserted, as there was a previous one
if isinstance(vals, list):
# If already several items got inserted, we have a list
vals.append(val)
else:
# vals should be a tuple then, i.e. only one item so far
# Need to convert the tuple to list for further extension
self._container[key_lower] = [vals[0], vals[1], val]
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def getlist(self, key):
"""Returns a list of all the values for the named field. Returns an
empty list if the key doesn't exist."""
try:
vals = self._container[key.lower()]
except KeyError:
return []
else:
if isinstance(vals, tuple):
return [vals[1]]
else:
return vals[1:]
# Backwards compatibility for httplib
getheaders = getlist
getallmatchingheaders = getlist
iget = getlist
def __repr__(self):
return "%s(%s)" % (type(self).__name__, dict(self.itermerged()))
def _copy_from(self, other):
for key in other:
val = other.getlist(key)
if isinstance(val, list):
# Don't need to convert tuples
val = list(val)
self._container[key.lower()] = [key] + val
def copy(self):
clone = type(self)()
clone._copy_from(self)
return clone
def iteritems(self):
"""Iterate over all header lines, including duplicate ones."""
for key in self:
vals = self._container[key.lower()]
for val in vals[1:]:
yield vals[0], val
def itermerged(self):
"""Iterate over all headers, merging duplicate ones together."""
for key in self:
val = self._container[key.lower()]
yield val[0], ', '.join(val[1:])
def items(self):
return list(self.iteritems())
@classmethod
def from_httplib(cls, message): # Python 2
"""Read headers from a Python 2 httplib message object."""
# python2.7 does not expose a proper API for exporting multiheaders
# efficiently. This function re-reads raw lines from the message
# object and extracts the multiheaders properly.
headers = []
for line in message.headers:
if line.startswith((' ', '\t')):
key, value = headers[-1]
headers[-1] = (key, value + '\r\n' + line.rstrip())
continue
key, value = line.split(':', 1)
headers.append((key, value.strip()))
return cls(headers)
__all__ = (
'basestring',
'connection',
'fields',
'filepost',
'poolmanager',
'timeout',
'HTTPHeaderDict',
'queue',
'urlencode',
'gaecontrib',
'urljoin',
'PyOpenSSLContext',
)

15
thirdparty/requests_toolbelt/adapters/__init__.py vendored Normal file
View File

@ -0,0 +1,15 @@
# -*- coding: utf-8 -*-
"""
requests-toolbelt.adapters
==========================
See http://toolbelt.rtfd.org/ for documentation
:copyright: (c) 2014 by Ian Cordasco and Cory Benfield
:license: Apache v2.0, see LICENSE for more details
"""
from .ssl import SSLAdapter
from .source import SourceAddressAdapter
__all__ = ['SSLAdapter', 'SourceAddressAdapter']

206
thirdparty/requests_toolbelt/adapters/appengine.py vendored Normal file
View File

@ -0,0 +1,206 @@
# -*- coding: utf-8 -*-
"""The App Engine Transport Adapter for requests.
.. versionadded:: 0.6.0
This requires a version of requests >= 2.10.0 and Python 2.
There are two ways to use this library:
#. If you're using requests directly, you can use code like:
.. code-block:: python
>>> import requests
>>> import ssl
>>> import requests.packages.urllib3.contrib.appengine as ul_appengine
>>> from requests_toolbelt.adapters import appengine
>>> s = requests.Session()
>>> if ul_appengine.is_appengine_sandbox():
... s.mount('http://', appengine.AppEngineAdapter())
... s.mount('https://', appengine.AppEngineAdapter())
#. If you depend on external libraries which use requests, you can use code
like:
.. code-block:: python
>>> from requests_toolbelt.adapters import appengine
>>> appengine.monkeypatch()
which will ensure all requests.Session objects use AppEngineAdapter properly.
You are also able to :ref:`disable certificate validation <insecure_appengine>`
when monkey-patching.
"""
import requests
import warnings
from requests import adapters
from requests import sessions
from .. import exceptions as exc
from .._compat import gaecontrib
from .._compat import timeout
class AppEngineMROHack(adapters.HTTPAdapter):
"""Resolves infinite recursion when monkeypatching.
This works by injecting itself as the base class of both the
:class:`AppEngineAdapter` and Requests' default HTTPAdapter, which needs to
be done because default HTTPAdapter's MRO is recompiled when we
monkeypatch, at which point this class becomes HTTPAdapter's base class.
In addition, we use an instantiation flag to avoid infinite recursion.
"""
_initialized = False
def __init__(self, *args, **kwargs):
if not self._initialized:
self._initialized = True
super(AppEngineMROHack, self).__init__(*args, **kwargs)
class AppEngineAdapter(AppEngineMROHack, adapters.HTTPAdapter):
"""The transport adapter for Requests to use urllib3's GAE support.
Implements Requests's HTTPAdapter API.
When deploying to Google's App Engine service, some of Requests'
functionality is broken. There is underlying support for GAE in urllib3.
This functionality, however, is opt-in and needs to be enabled explicitly
for Requests to be able to use it.
"""
__attrs__ = adapters.HTTPAdapter.__attrs__ + ['_validate_certificate']
def __init__(self, validate_certificate=True, *args, **kwargs):
_check_version()
self._validate_certificate = validate_certificate
super(AppEngineAdapter, self).__init__(*args, **kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = _AppEnginePoolManager(self._validate_certificate)
class InsecureAppEngineAdapter(AppEngineAdapter):
"""An always-insecure GAE adapter for Requests.
This is a variant of the the transport adapter for Requests to use
urllib3's GAE support that does not validate certificates. Use with
caution!
.. note::
The ``validate_certificate`` keyword argument will not be honored here
and is not part of the signature because we always force it to
``False``.
See :class:`AppEngineAdapter` for further details.
"""
def __init__(self, *args, **kwargs):
if kwargs.pop("validate_certificate", False):
warnings.warn("Certificate validation cannot be specified on the "
"InsecureAppEngineAdapter, but was present. This "
"will be ignored and certificate validation will "
"remain off.", exc.IgnoringGAECertificateValidation)
super(InsecureAppEngineAdapter, self).__init__(
validate_certificate=False, *args, **kwargs)
class _AppEnginePoolManager(object):
"""Implements urllib3's PoolManager API expected by requests.
While a real PoolManager map hostnames to reusable Connections,
AppEngine has no concept of a reusable connection to a host.
So instead, this class constructs a small Connection per request,
that is returned to the Adapter and used to access the URL.
"""
def __init__(self, validate_certificate=True):
self.appengine_manager = gaecontrib.AppEngineManager(
validate_certificate=validate_certificate)
def connection_from_url(self, url):
return _AppEngineConnection(self.appengine_manager, url)
def clear(self):
pass
class _AppEngineConnection(object):
"""Implements urllib3's HTTPConnectionPool API's urlopen().
This Connection's urlopen() is called with a host-relative path,
so in order to properly support opening the URL, we need to store
the full URL when this Connection is constructed from the PoolManager.
This code wraps AppEngineManager.urlopen(), which exposes a different
API than in the original urllib3 urlopen(), and thus needs this adapter.
"""
def __init__(self, appengine_manager, url):
self.appengine_manager = appengine_manager
self.url = url
def urlopen(self, method, url, body=None, headers=None, retries=None,
redirect=True, assert_same_host=True,
timeout=timeout.Timeout.DEFAULT_TIMEOUT,
pool_timeout=None, release_conn=None, **response_kw):
# This function's url argument is a host-relative URL,
# but the AppEngineManager expects an absolute URL.
# So we saved out the self.url when the AppEngineConnection
# was constructed, which we then can use down below instead.
# We once tried to verify our assumptions here, but sometimes the
# passed-in URL differs on url fragments, or "http://a.com" vs "/".
# urllib3's App Engine adapter only uses Timeout.total, not read or
# connect.
if not timeout.total:
timeout.total = timeout._read or timeout._connect
# Jump through the hoops necessary to call AppEngineManager's API.
return self.appengine_manager.urlopen(
method,
self.url,
body=body,
headers=headers,
retries=retries,
redirect=redirect,
timeout=timeout,
**response_kw)
def monkeypatch(validate_certificate=True):
"""Sets up all Sessions to use AppEngineAdapter by default.
If you don't want to deal with configuring your own Sessions,
or if you use libraries that use requests directly (ie requests.post),
then you may prefer to monkeypatch and auto-configure all Sessions.
.. warning: :
If ``validate_certificate`` is ``False``, certification validation will
effectively be disabled for all requests.
"""
_check_version()
# HACK: We should consider modifying urllib3 to support this cleanly,
# so that we can set a module-level variable in the sessions module,
# instead of overriding an imported HTTPAdapter as is done here.
adapter = AppEngineAdapter
if not validate_certificate:
adapter = InsecureAppEngineAdapter
sessions.HTTPAdapter = adapter
adapters.HTTPAdapter = adapter
def _check_version():
if gaecontrib is None:
raise exc.VersionMismatchError(
"The toolbelt requires at least Requests 2.10.0 to be "
"installed. Version {0} was found instead.".format(
requests.__version__
)
)

48
thirdparty/requests_toolbelt/adapters/fingerprint.py vendored Normal file
View File

@ -0,0 +1,48 @@
# -*- coding: utf-8 -*-
"""Submodule containing the implementation for the FingerprintAdapter.
This file contains an implementation of a Transport Adapter that validates
the fingerprints of SSL certificates presented upon connection.
"""
from requests.adapters import HTTPAdapter
from .._compat import poolmanager
class FingerprintAdapter(HTTPAdapter):
"""
A HTTPS Adapter for Python Requests that verifies certificate fingerprints,
instead of certificate hostnames.
Example usage:
.. code-block:: python
import requests
import ssl
from requests_toolbelt.adapters.fingerprint import FingerprintAdapter
twitter_fingerprint = '...'
s = requests.Session()
s.mount(
'https://twitter.com',
FingerprintAdapter(twitter_fingerprint)
)
The fingerprint should be provided as a hexadecimal string, optionally
containing colons.
"""
__attrs__ = HTTPAdapter.__attrs__ + ['fingerprint']
def __init__(self, fingerprint, **kwargs):
self.fingerprint = fingerprint
super(FingerprintAdapter, self).__init__(**kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = poolmanager.PoolManager(
num_pools=connections,
maxsize=maxsize,
block=block,
assert_fingerprint=self.fingerprint)

43
thirdparty/requests_toolbelt/adapters/host_header_ssl.py vendored Normal file
View File

@ -0,0 +1,43 @@
# -*- coding: utf-8 -*-
"""
requests_toolbelt.adapters.host_header_ssl
==========================================
This file contains an implementation of the HostHeaderSSLAdapter.
"""
from requests.adapters import HTTPAdapter
class HostHeaderSSLAdapter(HTTPAdapter):
"""
A HTTPS Adapter for Python Requests that sets the hostname for certificate
verification based on the Host header.
This allows requesting the IP address directly via HTTPS without getting
a "hostname doesn't match" exception.
Example usage:
>>> s.mount('https://', HostHeaderSSLAdapter())
>>> s.get("https://93.184.216.34", headers={"Host": "example.org"})
"""
def send(self, request, **kwargs):
# HTTP headers are case-insensitive (RFC 7230)
host_header = None
for header in request.headers:
if header.lower() == "host":
host_header = request.headers[header]
break
connection_pool_kwargs = self.poolmanager.connection_pool_kw
if host_header:
connection_pool_kwargs["assert_hostname"] = host_header
elif "assert_hostname" in connection_pool_kwargs:
# an assert_hostname from a previous request may have been left
connection_pool_kwargs.pop("assert_hostname", None)
return super(HostHeaderSSLAdapter, self).send(request, **kwargs)

129
thirdparty/requests_toolbelt/adapters/socket_options.py vendored Normal file
View File

@ -0,0 +1,129 @@
# -*- coding: utf-8 -*-
"""The implementation of the SocketOptionsAdapter."""
import socket
import warnings
import sys
import requests
from requests import adapters
from .._compat import connection
from .._compat import poolmanager
from .. import exceptions as exc
class SocketOptionsAdapter(adapters.HTTPAdapter):
"""An adapter for requests that allows users to specify socket options.
Since version 2.4.0 of requests, it is possible to specify a custom list
of socket options that need to be set before establishing the connection.
Example usage::
>>> import socket
>>> import requests
>>> from requests_toolbelt.adapters import socket_options
>>> s = requests.Session()
>>> opts = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 0)]
>>> adapter = socket_options.SocketOptionsAdapter(socket_options=opts)
>>> s.mount('http://', adapter)
You can also take advantage of the list of default options on this class
to keep using the original options in addition to your custom options. In
that case, ``opts`` might look like::
>>> opts = socket_options.SocketOptionsAdapter.default_options + opts
"""
if connection is not None:
default_options = getattr(
connection.HTTPConnection,
'default_socket_options',
[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]
)
else:
default_options = []
warnings.warn(exc.RequestsVersionTooOld,
"This version of Requests is only compatible with a "
"version of urllib3 which is too old to support "
"setting options on a socket. This adapter is "
"functionally useless.")
def __init__(self, **kwargs):
self.socket_options = kwargs.pop('socket_options',
self.default_options)
super(SocketOptionsAdapter, self).__init__(**kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
if requests.__build__ >= 0x020400:
# NOTE(Ian): Perhaps we should raise a warning
self.poolmanager = poolmanager.PoolManager(
num_pools=connections,
maxsize=maxsize,
block=block,
socket_options=self.socket_options
)
else:
super(SocketOptionsAdapter, self).init_poolmanager(
connections, maxsize, block
)
class TCPKeepAliveAdapter(SocketOptionsAdapter):
"""An adapter for requests that turns on TCP Keep-Alive by default.
The adapter sets 4 socket options:
- ``SOL_SOCKET`` ``SO_KEEPALIVE`` - This turns on TCP Keep-Alive
- ``IPPROTO_TCP`` ``TCP_KEEPINTVL`` 20 - Sets the keep alive interval
- ``IPPROTO_TCP`` ``TCP_KEEPCNT`` 5 - Sets the number of keep alive probes
- ``IPPROTO_TCP`` ``TCP_KEEPIDLE`` 60 - Sets the keep alive time if the
socket library has the ``TCP_KEEPIDLE`` constant
The latter three can be overridden by keyword arguments (respectively):
- ``idle``
- ``interval``
- ``count``
You can use this adapter like so::
>>> from requests_toolbelt.adapters import socket_options
>>> tcp = socket_options.TCPKeepAliveAdapter(idle=120, interval=10)
>>> s = requests.Session()
>>> s.mount('http://', tcp)
"""
def __init__(self, **kwargs):
socket_options = kwargs.pop('socket_options',
SocketOptionsAdapter.default_options)
idle = kwargs.pop('idle', 60)
interval = kwargs.pop('interval', 20)
count = kwargs.pop('count', 5)
socket_options = socket_options + [
(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
]
# NOTE(Ian): OSX does not have these constants defined, so we
# set them conditionally.
if getattr(socket, 'TCP_KEEPINTVL', None) is not None:
socket_options += [(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL,
interval)]
elif sys.platform == 'darwin':
# On OSX, TCP_KEEPALIVE from netinet/tcp.h is not exported
# by python's socket module
TCP_KEEPALIVE = getattr(socket, 'TCP_KEEPALIVE', 0x10)
socket_options += [(socket.IPPROTO_TCP, TCP_KEEPALIVE, interval)]
if getattr(socket, 'TCP_KEEPCNT', None) is not None:
socket_options += [(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, count)]
if getattr(socket, 'TCP_KEEPIDLE', None) is not None:
socket_options += [(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, idle)]
super(TCPKeepAliveAdapter, self).__init__(
socket_options=socket_options, **kwargs
)

67
thirdparty/requests_toolbelt/adapters/source.py vendored Normal file
View File

@ -0,0 +1,67 @@
# -*- coding: utf-8 -*-
"""
requests_toolbelt.source_adapter
================================
This file contains an implementation of the SourceAddressAdapter originally
demonstrated on the Requests GitHub page.
"""
from requests.adapters import HTTPAdapter
from .._compat import poolmanager, basestring
class SourceAddressAdapter(HTTPAdapter):
"""
A Source Address Adapter for Python Requests that enables you to choose the
local address to bind to. This allows you to send your HTTP requests from a
specific interface and IP address.
Two address formats are accepted. The first is a string: this will set the
local IP address to the address given in the string, and will also choose a
semi-random high port for the local port number.
The second is a two-tuple of the form (ip address, port): for example,
``('10.10.10.10', 8999)``. This will set the local IP address to the first
element, and the local port to the second element. If ``0`` is used as the
port number, a semi-random high port will be selected.
.. warning:: Setting an explicit local port can have negative interactions
with connection-pooling in Requests: in particular, it risks
the possibility of getting "Address in use" errors. The
string-only argument is generally preferred to the tuple-form.
Example usage:
.. code-block:: python
import requests
from requests_toolbelt.adapters.source import SourceAddressAdapter
s = requests.Session()
s.mount('http://', SourceAddressAdapter('10.10.10.10'))
s.mount('https://', SourceAddressAdapter(('10.10.10.10', 8999)))
"""
def __init__(self, source_address, **kwargs):
if isinstance(source_address, basestring):
self.source_address = (source_address, 0)
elif isinstance(source_address, tuple):
self.source_address = source_address
else:
raise TypeError(
"source_address must be IP address string or (ip, port) tuple"
)
super(SourceAddressAdapter, self).__init__(**kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = poolmanager.PoolManager(
num_pools=connections,
maxsize=maxsize,
block=block,
source_address=self.source_address)
def proxy_manager_for(self, *args, **kwargs):
kwargs['source_address'] = self.source_address
return super(SourceAddressAdapter, self).proxy_manager_for(
*args, **kwargs)

66
thirdparty/requests_toolbelt/adapters/ssl.py vendored Normal file
View File

@ -0,0 +1,66 @@
# -*- coding: utf-8 -*-
"""
requests_toolbelt.ssl_adapter
=============================
This file contains an implementation of the SSLAdapter originally demonstrated
in this blog post:
https://lukasa.co.uk/2013/01/Choosing_SSL_Version_In_Requests/
"""
import requests
from requests.adapters import HTTPAdapter
from .._compat import poolmanager
class SSLAdapter(HTTPAdapter):
"""
A HTTPS Adapter for Python Requests that allows the choice of the SSL/TLS
version negotiated by Requests. This can be used either to enforce the
choice of high-security TLS versions (where supported), or to work around
misbehaving servers that fail to correctly negotiate the default TLS
version being offered.
Example usage:
>>> import requests
>>> import ssl
>>> from requests_toolbelt import SSLAdapter
>>> s = requests.Session()
>>> s.mount('https://', SSLAdapter(ssl.PROTOCOL_TLSv1))
You can replace the chosen protocol with any that are available in the
default Python SSL module. All subsequent requests that match the adapter
prefix will use the chosen SSL version instead of the default.
This adapter will also attempt to change the SSL/TLS version negotiated by
Requests when using a proxy. However, this may not always be possible:
prior to Requests v2.4.0 the adapter did not have access to the proxy setup
code. In earlier versions of Requests, this adapter will not function
properly when used with proxies.
"""
__attrs__ = HTTPAdapter.__attrs__ + ['ssl_version']
def __init__(self, ssl_version=None, **kwargs):
self.ssl_version = ssl_version
super(SSLAdapter, self).__init__(**kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = poolmanager.PoolManager(
num_pools=connections,
maxsize=maxsize,
block=block,
ssl_version=self.ssl_version)
if requests.__build__ >= 0x020400:
# Earlier versions of requests either don't have this method or, worse,
# don't allow passing arbitrary keyword arguments. As a result, only
# conditionally define this method.
def proxy_manager_for(self, *args, **kwargs):
kwargs['ssl_version'] = self.ssl_version
return super(SSLAdapter, self).proxy_manager_for(*args, **kwargs)

178
thirdparty/requests_toolbelt/adapters/x509.py vendored Normal file
View File

@ -0,0 +1,178 @@
# -*- coding: utf-8 -*-
"""A X509Adapter for use with the requests library.
This file contains an implementation of the X509Adapter that will
allow users to authenticate a request using an arbitrary
X.509 certificate without needing to convert it to a .pem file
"""
from OpenSSL.crypto import PKey, X509
from cryptography import x509
from cryptography.hazmat.primitives.serialization import (load_pem_private_key,
load_der_private_key)
from cryptography.hazmat.primitives.serialization import Encoding
from cryptography.hazmat.backends import default_backend
from datetime import datetime
from requests.adapters import HTTPAdapter
import requests
from .._compat import PyOpenSSLContext
from .. import exceptions as exc
"""
importing the protocol constants from _ssl instead of ssl because only the
constants are needed and to handle issues caused by importing from ssl on
the 2.7.x line.
"""
try:
from _ssl import PROTOCOL_TLS as PROTOCOL
except ImportError:
from _ssl import PROTOCOL_SSLv23 as PROTOCOL
class X509Adapter(HTTPAdapter):
r"""Adapter for use with X.509 certificates.
Provides an interface for Requests sessions to contact HTTPS urls and
authenticate with an X.509 cert by implementing the Transport Adapter
interface. This class will need to be manually instantiated and mounted
to the session
:param pool_connections: The number of urllib3 connection pools to
cache.
:param pool_maxsize: The maximum number of connections to save in the
pool.
:param max_retries: The maximum number of retries each connection
should attempt. Note, this applies only to failed DNS lookups,
socket connections and connection timeouts, never to requests where
data has made it to the server. By default, Requests does not retry
failed connections. If you need granular control over the
conditions under which we retry a request, import urllib3's
``Retry`` class and pass that instead.
:param pool_block: Whether the connection pool should block for
connections.
:param bytes cert_bytes:
bytes object containing contents of a cryptography.x509Certificate
object using the encoding specified by the ``encoding`` parameter.
:param bytes pk_bytes:
bytes object containing contents of a object that implements
``cryptography.hazmat.primitives.serialization.PrivateFormat``
using the encoding specified by the ``encoding`` parameter.
:param password:
string or utf8 encoded bytes containing the passphrase used for the
private key. None if unencrypted. Defaults to None.
:param encoding:
Enumeration detailing the encoding method used on the ``cert_bytes``
parameter. Can be either PEM or DER. Defaults to PEM.
:type encoding:
:class: `cryptography.hazmat.primitives.serialization.Encoding`
Usage::
>>> import requests
>>> from requests_toolbelt.adapters.x509 import X509Adapter
>>> s = requests.Session()
>>> a = X509Adapter(max_retries=3,
cert_bytes=b'...', pk_bytes=b'...', encoding='...'
>>> s.mount('https://', a)
"""
def __init__(self, *args, **kwargs):
self._check_version()
cert_bytes = kwargs.pop('cert_bytes', None)
pk_bytes = kwargs.pop('pk_bytes', None)
password = kwargs.pop('password', None)
encoding = kwargs.pop('encoding', Encoding.PEM)
password_bytes = None
if cert_bytes is None or not isinstance(cert_bytes, bytes):
raise ValueError('Invalid cert content provided. '
'You must provide an X.509 cert '
'formatted as a byte array.')
if pk_bytes is None or not isinstance(pk_bytes, bytes):
raise ValueError('Invalid private key content provided. '
'You must provide a private key '
'formatted as a byte array.')
if isinstance(password, bytes):
password_bytes = password
elif password:
password_bytes = password.encode('utf8')
self.ssl_context = create_ssl_context(cert_bytes, pk_bytes,
password_bytes, encoding)
super(X509Adapter, self).__init__(*args, **kwargs)
def init_poolmanager(self, *args, **kwargs):
if self.ssl_context:
kwargs['ssl_context'] = self.ssl_context
return super(X509Adapter, self).init_poolmanager(*args, **kwargs)
def proxy_manager_for(self, *args, **kwargs):
if self.ssl_context:
kwargs['ssl_context'] = self.ssl_context
return super(X509Adapter, self).proxy_manager_for(*args, **kwargs)
def _check_version(self):
if PyOpenSSLContext is None:
raise exc.VersionMismatchError(
"The X509Adapter requires at least Requests 2.12.0 to be "
"installed. Version {0} was found instead.".format(
requests.__version__
)
)
def check_cert_dates(cert):
"""Verify that the supplied client cert is not invalid."""
now = datetime.utcnow()
if cert.not_valid_after < now or cert.not_valid_before > now:
raise ValueError('Client certificate expired: Not After: '
'{0:%Y-%m-%d %H:%M:%SZ} '
'Not Before: {1:%Y-%m-%d %H:%M:%SZ}'
.format(cert.not_valid_after, cert.not_valid_before))
def create_ssl_context(cert_byes, pk_bytes, password=None,
encoding=Encoding.PEM):
"""Create an SSL Context with the supplied cert/password.
:param cert_bytes array of bytes containing the cert encoded
using the method supplied in the ``encoding`` parameter
:param pk_bytes array of bytes containing the private key encoded
using the method supplied in the ``encoding`` parameter
:param password array of bytes containing the passphrase to be used
with the supplied private key. None if unencrypted.
Defaults to None.
:param encoding ``cryptography.hazmat.primitives.serialization.Encoding``
details the encoding method used on the ``cert_bytes`` and
``pk_bytes`` parameters. Can be either PEM or DER.
Defaults to PEM.
"""
backend = default_backend()
cert = None
key = None
if encoding == Encoding.PEM:
cert = x509.load_pem_x509_certificate(cert_byes, backend)
key = load_pem_private_key(pk_bytes, password, backend)
elif encoding == Encoding.DER:
cert = x509.load_der_x509_certificate(cert_byes, backend)
key = load_der_private_key(pk_bytes, password, backend)
else:
raise ValueError('Invalid encoding provided: Must be PEM or DER')
if not (cert and key):
raise ValueError('Cert and key could not be parsed from '
'provided data')
check_cert_dates(cert)
ssl_context = PyOpenSSLContext(PROTOCOL)
ssl_context._ctx.use_certificate(X509.from_cryptography(cert))
ssl_context._ctx.use_privatekey(PKey.from_cryptography_key(key))
return ssl_context

0
thirdparty/requests_toolbelt/auth/__init__.py vendored Normal file
View File

29
thirdparty/requests_toolbelt/auth/_digest_auth_compat.py vendored Normal file
View File

@ -0,0 +1,29 @@
"""Provide a compatibility layer for requests.auth.HTTPDigestAuth."""
import requests
class _ThreadingDescriptor(object):
def __init__(self, prop, default):
self.prop = prop
self.default = default
def __get__(self, obj, objtype=None):
return getattr(obj._thread_local, self.prop, self.default)
def __set__(self, obj, value):
setattr(obj._thread_local, self.prop, value)
class _HTTPDigestAuth(requests.auth.HTTPDigestAuth):
init = _ThreadingDescriptor('init', True)
last_nonce = _ThreadingDescriptor('last_nonce', '')
nonce_count = _ThreadingDescriptor('nonce_count', 0)
chal = _ThreadingDescriptor('chal', {})
pos = _ThreadingDescriptor('pos', None)
num_401_calls = _ThreadingDescriptor('num_401_calls', 1)
if requests.__build__ < 0x020800:
HTTPDigestAuth = requests.auth.HTTPDigestAuth
else:
HTTPDigestAuth = _HTTPDigestAuth

146
thirdparty/requests_toolbelt/auth/guess.py vendored Normal file
View File

@ -0,0 +1,146 @@
# -*- coding: utf-8 -*-
"""The module containing the code for GuessAuth."""
from requests import auth
from requests import cookies
from . import _digest_auth_compat as auth_compat, http_proxy_digest
class GuessAuth(auth.AuthBase):
"""Guesses the auth type by the WWW-Authentication header."""
def __init__(self, username, password):
self.username = username
self.password = password
self.auth = None
self.pos = None
def _handle_basic_auth_401(self, r, kwargs):
if self.pos is not None:
r.request.body.seek(self.pos)
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.raw.release_conn()
prep = r.request.copy()
if not hasattr(prep, '_cookies'):
prep._cookies = cookies.RequestsCookieJar()
cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
self.auth = auth.HTTPBasicAuth(self.username, self.password)
prep = self.auth(prep)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
def _handle_digest_auth_401(self, r, kwargs):
self.auth = auth_compat.HTTPDigestAuth(self.username, self.password)
try:
self.auth.init_per_thread_state()
except AttributeError:
# If we're not on requests 2.8.0+ this method does not exist and
# is not relevant.
pass
# Check that the attr exists because much older versions of requests
# set this attribute lazily. For example:
# https://github.com/kennethreitz/requests/blob/33735480f77891754304e7f13e3cdf83aaaa76aa/requests/auth.py#L59
if (hasattr(self.auth, 'num_401_calls') and
self.auth.num_401_calls is None):
self.auth.num_401_calls = 1
# Digest auth would resend the request by itself. We can take a
# shortcut here.
return self.auth.handle_401(r, **kwargs)
def handle_401(self, r, **kwargs):
"""Resends a request with auth headers, if needed."""
www_authenticate = r.headers.get('www-authenticate', '').lower()
if 'basic' in www_authenticate:
return self._handle_basic_auth_401(r, kwargs)
if 'digest' in www_authenticate:
return self._handle_digest_auth_401(r, kwargs)
def __call__(self, request):
if self.auth is not None:
return self.auth(request)
try:
self.pos = request.body.tell()
except AttributeError:
pass
request.register_hook('response', self.handle_401)
return request
class GuessProxyAuth(GuessAuth):
"""
Guesses the auth type by WWW-Authentication and Proxy-Authentication
headers
"""
def __init__(self, username=None, password=None,
proxy_username=None, proxy_password=None):
super(GuessProxyAuth, self).__init__(username, password)
self.proxy_username = proxy_username
self.proxy_password = proxy_password
self.proxy_auth = None
def _handle_basic_auth_407(self, r, kwargs):
if self.pos is not None:
r.request.body.seek(self.pos)
r.content
r.raw.release_conn()
prep = r.request.copy()
if not hasattr(prep, '_cookies'):
prep._cookies = cookies.RequestsCookieJar()
cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
self.proxy_auth = auth.HTTPProxyAuth(self.proxy_username,
self.proxy_password)
prep = self.proxy_auth(prep)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
def _handle_digest_auth_407(self, r, kwargs):
self.proxy_auth = http_proxy_digest.HTTPProxyDigestAuth(
username=self.proxy_username,
password=self.proxy_password)
try:
self.auth.init_per_thread_state()
except AttributeError:
pass
return self.proxy_auth.handle_407(r, **kwargs)
def handle_407(self, r, **kwargs):
proxy_authenticate = r.headers.get('Proxy-Authenticate', '').lower()
if 'basic' in proxy_authenticate:
return self._handle_basic_auth_407(r, kwargs)
if 'digest' in proxy_authenticate:
return self._handle_digest_auth_407(r, kwargs)
def __call__(self, request):
if self.proxy_auth is not None:
request = self.proxy_auth(request)
try:
self.pos = request.body.tell()
except AttributeError:
pass
request.register_hook('response', self.handle_407)
return super(GuessProxyAuth, self).__call__(request)

142
thirdparty/requests_toolbelt/auth/handler.py vendored Normal file
View File

@ -0,0 +1,142 @@
# -*- coding: utf-8 -*-
"""
requests_toolbelt.auth.handler
==============================
This holds all of the implementation details of the Authentication Handler.
"""
from requests.auth import AuthBase, HTTPBasicAuth
from requests.compat import urlparse, urlunparse
class AuthHandler(AuthBase):
"""
The ``AuthHandler`` object takes a dictionary of domains paired with
authentication strategies and will use this to determine which credentials
to use when making a request. For example, you could do the following:
.. code-block:: python
from requests import HTTPDigestAuth
from requests_toolbelt.auth.handler import AuthHandler
import requests
auth = AuthHandler({
'https://api.github.com': ('sigmavirus24', 'fakepassword'),
'https://example.com': HTTPDigestAuth('username', 'password')
})
r = requests.get('https://api.github.com/user', auth=auth)
# => <Response [200]>
r = requests.get('https://example.com/some/path', auth=auth)
# => <Response [200]>
s = requests.Session()
s.auth = auth
r = s.get('https://api.github.com/user')
# => <Response [200]>
.. warning::
:class:`requests.auth.HTTPDigestAuth` is not yet thread-safe. If you
use :class:`AuthHandler` across multiple threads you should
instantiate a new AuthHandler for each thread with a new
HTTPDigestAuth instance for each thread.
"""
def __init__(self, strategies):
self.strategies = dict(strategies)
self._make_uniform()
def __call__(self, request):
auth = self.get_strategy_for(request.url)
return auth(request)
def __repr__(self):
return '<AuthHandler({0!r})>'.format(self.strategies)
def _make_uniform(self):
existing_strategies = list(self.strategies.items())
self.strategies = {}
for (k, v) in existing_strategies:
self.add_strategy(k, v)
@staticmethod
def _key_from_url(url):
parsed = urlparse(url)
return urlunparse((parsed.scheme.lower(),
parsed.netloc.lower(),
'', '', '', ''))
def add_strategy(self, domain, strategy):
"""Add a new domain and authentication strategy.
:param str domain: The domain you wish to match against. For example:
``'https://api.github.com'``
:param str strategy: The authentication strategy you wish to use for
that domain. For example: ``('username', 'password')`` or
``requests.HTTPDigestAuth('username', 'password')``
.. code-block:: python
a = AuthHandler({})
a.add_strategy('https://api.github.com', ('username', 'password'))
"""
# Turn tuples into Basic Authentication objects
if isinstance(strategy, tuple):
strategy = HTTPBasicAuth(*strategy)
key = self._key_from_url(domain)
self.strategies[key] = strategy
def get_strategy_for(self, url):
"""Retrieve the authentication strategy for a specified URL.
:param str url: The full URL you will be making a request against. For
example, ``'https://api.github.com/user'``
:returns: Callable that adds authentication to a request.
.. code-block:: python
import requests
a = AuthHandler({'example.com', ('foo', 'bar')})
strategy = a.get_strategy_for('http://example.com/example')
assert isinstance(strategy, requests.auth.HTTPBasicAuth)
"""
key = self._key_from_url(url)
return self.strategies.get(key, NullAuthStrategy())
def remove_strategy(self, domain):
"""Remove the domain and strategy from the collection of strategies.
:param str domain: The domain you wish remove. For example,
``'https://api.github.com'``.
.. code-block:: python
a = AuthHandler({'example.com', ('foo', 'bar')})
a.remove_strategy('example.com')
assert a.strategies == {}
"""
key = self._key_from_url(domain)
if key in self.strategies:
del self.strategies[key]
class NullAuthStrategy(AuthBase):
def __repr__(self):
return '<NullAuthStrategy>'
def __call__(self, r):
return r

103
thirdparty/requests_toolbelt/auth/http_proxy_digest.py vendored Normal file
View File

@ -0,0 +1,103 @@
# -*- coding: utf-8 -*-
"""The module containing HTTPProxyDigestAuth."""
import re
from requests import cookies, utils
from . import _digest_auth_compat as auth
class HTTPProxyDigestAuth(auth.HTTPDigestAuth):
"""HTTP digest authentication between proxy
:param stale_rejects: The number of rejects indicate that:
the client may wish to simply retry the request
with a new encrypted response, without reprompting the user for a
new username and password. i.e., retry build_digest_header
:type stale_rejects: int
"""
_pat = re.compile(r'digest ', flags=re.IGNORECASE)
def __init__(self, *args, **kwargs):
super(HTTPProxyDigestAuth, self).__init__(*args, **kwargs)
self.stale_rejects = 0
self.init_per_thread_state()
@property
def stale_rejects(self):
thread_local = getattr(self, '_thread_local', None)
if thread_local is None:
return self._stale_rejects
return thread_local.stale_rejects
@stale_rejects.setter
def stale_rejects(self, value):
thread_local = getattr(self, '_thread_local', None)
if thread_local is None:
self._stale_rejects = value
else:
thread_local.stale_rejects = value
def init_per_thread_state(self):
try:
super(HTTPProxyDigestAuth, self).init_per_thread_state()
except AttributeError:
# If we're not on requests 2.8.0+ this method does not exist
pass
def handle_407(self, r, **kwargs):
"""Handle HTTP 407 only once, otherwise give up
:param r: current response
:returns: responses, along with the new response
"""
if r.status_code == 407 and self.stale_rejects < 2:
s_auth = r.headers.get("proxy-authenticate")
if s_auth is None:
raise IOError(
"proxy server violated RFC 7235:"
"407 response MUST contain header proxy-authenticate")
elif not self._pat.match(s_auth):
return r
self.chal = utils.parse_dict_header(
self._pat.sub('', s_auth, count=1))
# if we present the user/passwd and still get rejected
# http://tools.ietf.org/html/rfc2617#section-3.2.1
if ('Proxy-Authorization' in r.request.headers and
'stale' in self.chal):
if self.chal['stale'].lower() == 'true': # try again
self.stale_rejects += 1
# wrong user/passwd
elif self.chal['stale'].lower() == 'false':
raise IOError("User or password is invalid")
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.close()
prep = r.request.copy()
cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
prep.headers['Proxy-Authorization'] = self.build_digest_header(
prep.method, prep.url)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
else: # give up authenticate
return r
def __call__(self, r):
self.init_per_thread_state()
# if we have nonce, then just use it, otherwise server will tell us
if self.last_nonce:
r.headers['Proxy-Authorization'] = self.build_digest_header(
r.method, r.url
)
r.register_hook('response', self.handle_407)
return r

0
thirdparty/requests_toolbelt/cookies/__init__.py vendored Normal file
View File

7
thirdparty/requests_toolbelt/cookies/forgetful.py vendored Normal file
View File

@ -0,0 +1,7 @@
"""The module containing the code for ForgetfulCookieJar."""
from requests.cookies import RequestsCookieJar
class ForgetfulCookieJar(RequestsCookieJar):
def set_cookie(self, *args, **kwargs):
return

0
thirdparty/requests_toolbelt/downloadutils/__init__.py vendored Normal file
View File

177
thirdparty/requests_toolbelt/downloadutils/stream.py vendored Normal file
View File

@ -0,0 +1,177 @@
# -*- coding: utf-8 -*-
"""Utilities for dealing with streamed requests."""
import collections
import os.path
import re
from .. import exceptions as exc
# Regular expressions stolen from werkzeug/http.py
# cd2c97bb0a076da2322f11adce0b2731f9193396 L62-L64
_QUOTED_STRING_RE = r'"[^"\\]*(?:\\.[^"\\]*)*"'
_OPTION_HEADER_PIECE_RE = re.compile(
r';\s*(%s|[^\s;=]+)\s*(?:=\s*(%s|[^;]+))?\s*' % (_QUOTED_STRING_RE,
_QUOTED_STRING_RE)
)
_DEFAULT_CHUNKSIZE = 512
def _get_filename(content_disposition):
for match in _OPTION_HEADER_PIECE_RE.finditer(content_disposition):
k, v = match.groups()
if k == 'filename':
# ignore any directory paths in the filename
return os.path.split(v)[1]
return None
def get_download_file_path(response, path):
"""
Given a response and a path, return a file path for a download.
If a ``path`` parameter is a directory, this function will parse the
``Content-Disposition`` header on the response to determine the name of the
file as reported by the server, and return a file path in the specified
directory.
If ``path`` is empty or None, this function will return a path relative
to the process' current working directory.
If path is a full file path, return it.
:param response: A Response object from requests
:type response: requests.models.Response
:param str path: Directory or file path.
:returns: full file path to download as
:rtype: str
:raises: :class:`requests_toolbelt.exceptions.StreamingError`
"""
path_is_dir = path and os.path.isdir(path)
if path and not path_is_dir:
# fully qualified file path
filepath = path
else:
response_filename = _get_filename(
response.headers.get('content-disposition', '')
)
if not response_filename:
raise exc.StreamingError('No filename given to stream response to')
if path_is_dir:
# directory to download to
filepath = os.path.join(path, response_filename)
else:
# fallback to downloading to current working directory
filepath = response_filename
return filepath
def stream_response_to_file(response, path=None, chunksize=_DEFAULT_CHUNKSIZE):
"""Stream a response body to the specified file.
Either use the ``path`` provided or use the name provided in the
``Content-Disposition`` header.
.. warning::
If you pass this function an open file-like object as the ``path``
parameter, the function will not close that file for you.
.. warning::
This function will not automatically close the response object
passed in as the ``response`` parameter.
If a ``path`` parameter is a directory, this function will parse the
``Content-Disposition`` header on the response to determine the name of the
file as reported by the server, and return a file path in the specified
directory. If no ``path`` parameter is supplied, this function will default
to the process' current working directory.
.. code-block:: python
import requests
from requests_toolbelt import exceptions
from requests_toolbelt.downloadutils import stream
r = requests.get(url, stream=True)
try:
filename = stream.stream_response_to_file(r)
except exceptions.StreamingError as e:
# The toolbelt could not find the filename in the
# Content-Disposition
print(e.message)
You can also specify the filename as a string. This will be passed to
the built-in :func:`open` and we will read the content into the file.
.. code-block:: python
import requests
from requests_toolbelt.downloadutils import stream
r = requests.get(url, stream=True)
filename = stream.stream_response_to_file(r, path='myfile')
If the calculated download file path already exists, this function will
raise a StreamingError.
Instead, if you want to manage the file object yourself, you need to
provide either a :class:`io.BytesIO` object or a file opened with the
`'b'` flag. See the two examples below for more details.
.. code-block:: python
import requests
from requests_toolbelt.downloadutils import stream
with open('myfile', 'wb') as fd:
r = requests.get(url, stream=True)
filename = stream.stream_response_to_file(r, path=fd)
print('{0} saved to {1}'.format(url, filename))
.. code-block:: python
import io
import requests
from requests_toolbelt.downloadutils import stream
b = io.BytesIO()
r = requests.get(url, stream=True)
filename = stream.stream_response_to_file(r, path=b)
assert filename is None
:param response: A Response object from requests
:type response: requests.models.Response
:param path: *(optional)*, Either a string with the path to the location
to save the response content, or a file-like object expecting bytes.
:type path: :class:`str`, or object with a :meth:`write`
:param int chunksize: (optional), Size of chunk to attempt to stream
(default 512B).
:returns: The name of the file, if one can be determined, else None
:rtype: str
:raises: :class:`requests_toolbelt.exceptions.StreamingError`
"""
pre_opened = False
fd = None
filename = None
if path and isinstance(getattr(path, 'write', None), collections.Callable):
pre_opened = True
fd = path
filename = getattr(fd, 'name', None)
else:
filename = get_download_file_path(response, path)
if os.path.exists(filename):
raise exc.StreamingError("File already exists: %s" % filename)
fd = open(filename, 'wb')
for chunk in response.iter_content(chunk_size=chunksize):
fd.write(chunk)
if not pre_opened:
fd.close()
return filename

123
thirdparty/requests_toolbelt/downloadutils/tee.py vendored Normal file
View File

@ -0,0 +1,123 @@
"""Tee function implementations."""
import io
_DEFAULT_CHUNKSIZE = 65536
__all__ = ['tee', 'tee_to_file', 'tee_to_bytearray']
def _tee(response, callback, chunksize, decode_content):
for chunk in response.raw.stream(amt=chunksize,
decode_content=decode_content):
callback(chunk)
yield chunk
def tee(response, fileobject, chunksize=_DEFAULT_CHUNKSIZE,
decode_content=None):
"""Stream the response both to the generator and a file.
This will stream the response body while writing the bytes to
``fileobject``.
Example usage:
.. code-block:: python
resp = requests.get(url, stream=True)
with open('save_file', 'wb') as save_file:
for chunk in tee(resp, save_file):
# do stuff with chunk
.. code-block:: python
import io
resp = requests.get(url, stream=True)
fileobject = io.BytesIO()
for chunk in tee(resp, fileobject):
# do stuff with chunk
:param response: Response from requests.
:type response: requests.Response
:param fileobject: Writable file-like object.
:type fileobject: file, io.BytesIO
:param int chunksize: (optional), Size of chunk to attempt to stream.
:param bool decode_content: (optional), If True, this will decode the
compressed content of the response.
:raises: TypeError if the fileobject wasn't opened with the right mode
or isn't a BytesIO object.
"""
# We will be streaming the raw bytes from over the wire, so we need to
# ensure that writing to the fileobject will preserve those bytes. On
# Python3, if the user passes an io.StringIO, this will fail, so we need
# to check for BytesIO instead.
if not ('b' in getattr(fileobject, 'mode', '') or
isinstance(fileobject, io.BytesIO)):
raise TypeError('tee() will write bytes directly to this fileobject'
', it must be opened with the "b" flag if it is a file'
' or inherit from io.BytesIO.')
return _tee(response, fileobject.write, chunksize, decode_content)
def tee_to_file(response, filename, chunksize=_DEFAULT_CHUNKSIZE,
decode_content=None):
"""Stream the response both to the generator and a file.
This will open a file named ``filename`` and stream the response body
while writing the bytes to the opened file object.
Example usage:
.. code-block:: python
resp = requests.get(url, stream=True)
for chunk in tee_to_file(resp, 'save_file'):
# do stuff with chunk
:param response: Response from requests.
:type response: requests.Response
:param str filename: Name of file in which we write the response content.
:param int chunksize: (optional), Size of chunk to attempt to stream.
:param bool decode_content: (optional), If True, this will decode the
compressed content of the response.
"""
with open(filename, 'wb') as fd:
for chunk in tee(response, fd, chunksize, decode_content):
yield chunk
def tee_to_bytearray(response, bytearr, chunksize=_DEFAULT_CHUNKSIZE,
decode_content=None):
"""Stream the response both to the generator and a bytearray.
This will stream the response provided to the function, add them to the
provided :class:`bytearray` and yield them to the user.
.. note::
This uses the :meth:`bytearray.extend` by default instead of passing
the bytearray into the ``readinto`` method.
Example usage:
.. code-block:: python
b = bytearray()
resp = requests.get(url, stream=True)
for chunk in tee_to_bytearray(resp, b):
# do stuff with chunk
:param response: Response from requests.
:type response: requests.Response
:param bytearray bytearr: Array to add the streamed bytes to.
:param int chunksize: (optional), Size of chunk to attempt to stream.
:param bool decode_content: (optional), If True, this will decode the
compressed content of the response.
"""
if not isinstance(bytearr, bytearray):
raise TypeError('tee_to_bytearray() expects bytearr to be a '
'bytearray')
return _tee(response, bytearr.extend, chunksize, decode_content)

37
thirdparty/requests_toolbelt/exceptions.py vendored Normal file
View File

@ -0,0 +1,37 @@
# -*- coding: utf-8 -*-
"""Collection of exceptions raised by requests-toolbelt."""
class StreamingError(Exception):
"""Used in :mod:`requests_toolbelt.downloadutils.stream`."""
pass
class VersionMismatchError(Exception):
"""Used to indicate a version mismatch in the version of requests required.
The feature in use requires a newer version of Requests to function
appropriately but the version installed is not sufficient.
"""
pass
class RequestsVersionTooOld(Warning):
"""Used to indicate that the Requests version is too old.
If the version of Requests is too old to support a feature, we will issue
this warning to the user.
"""
pass
class IgnoringGAECertificateValidation(Warning):
"""Used to indicate that given GAE validation behavior will be ignored.
If the user has tried to specify certificate validation when using the
insecure AppEngine adapter, it will be ignored (certificate validation will
remain off), so we will issue this warning to the user.
In :class:`requests_toolbelt.adapters.appengine.InsecureAppEngineAdapter`.
"""
pass

31
thirdparty/requests_toolbelt/multipart/__init__.py vendored Normal file
View File

@ -0,0 +1,31 @@
"""
requests_toolbelt.multipart
===========================
See http://toolbelt.rtfd.org/ for documentation
:copyright: (c) 2014 by Ian Cordasco and Cory Benfield
:license: Apache v2.0, see LICENSE for more details
"""
from .encoder import MultipartEncoder, MultipartEncoderMonitor
from .decoder import MultipartDecoder
from .decoder import ImproperBodyPartContentException
from .decoder import NonMultipartContentTypeException
__title__ = 'requests-toolbelt'
__authors__ = 'Ian Cordasco, Cory Benfield'
__license__ = 'Apache v2.0'
__copyright__ = 'Copyright 2014 Ian Cordasco, Cory Benfield'
__all__ = [
'MultipartEncoder',
'MultipartEncoderMonitor',
'MultipartDecoder',
'ImproperBodyPartContentException',
'NonMultipartContentTypeException',
'__title__',
'__authors__',
'__license__',
'__copyright__',
]

156
thirdparty/requests_toolbelt/multipart/decoder.py vendored Normal file
View File

@ -0,0 +1,156 @@
# -*- coding: utf-8 -*-
"""
requests_toolbelt.multipart.decoder
===================================
This holds all the implementation details of the MultipartDecoder
"""
import sys
import email.parser
from .encoder import encode_with
from requests.structures import CaseInsensitiveDict
def _split_on_find(content, bound):
point = content.find(bound)
return content[:point], content[point + len(bound):]
class ImproperBodyPartContentException(Exception):
pass
class NonMultipartContentTypeException(Exception):
pass
def _header_parser(string, encoding):
major = sys.version_info[0]
if major == 3:
string = string.decode(encoding)
headers = email.parser.HeaderParser().parsestr(string).items()
return (
(encode_with(k, encoding), encode_with(v, encoding))
for k, v in headers
)
class BodyPart(object):
"""
The ``BodyPart`` object is a ``Response``-like interface to an individual
subpart of a multipart response. It is expected that these will
generally be created by objects of the ``MultipartDecoder`` class.
Like ``Response``, there is a ``CaseInsensitiveDict`` object named headers,
``content`` to access bytes, ``text`` to access unicode, and ``encoding``
to access the unicode codec.
"""
def __init__(self, content, encoding):
self.encoding = encoding
headers = {}
# Split into header section (if any) and the content
if b'\r\n\r\n' in content:
first, self.content = _split_on_find(content, b'\r\n\r\n')
if first != b'':
headers = _header_parser(first.lstrip(), encoding)
else:
raise ImproperBodyPartContentException(
'content does not contain CR-LF-CR-LF'
)
self.headers = CaseInsensitiveDict(headers)
@property
def text(self):
"""Content of the ``BodyPart`` in unicode."""
return self.content.decode(self.encoding)
class MultipartDecoder(object):
"""
The ``MultipartDecoder`` object parses the multipart payload of
a bytestring into a tuple of ``Response``-like ``BodyPart`` objects.
The basic usage is::
import requests
from requests_toolbelt import MultipartDecoder
response = request.get(url)
decoder = MultipartDecoder.from_response(response)
for part in decoder.parts:
print(part.headers['content-type'])
If the multipart content is not from a response, basic usage is::
from requests_toolbelt import MultipartDecoder
decoder = MultipartDecoder(content, content_type)
for part in decoder.parts:
print(part.headers['content-type'])
For both these usages, there is an optional ``encoding`` parameter. This is
a string, which is the name of the unicode codec to use (default is
``'utf-8'``).
"""
def __init__(self, content, content_type, encoding='utf-8'):
#: Original Content-Type header
self.content_type = content_type
#: Response body encoding
self.encoding = encoding
#: Parsed parts of the multipart response body
self.parts = tuple()
self._find_boundary()
self._parse_body(content)
def _find_boundary(self):
ct_info = tuple(x.strip() for x in self.content_type.split(';'))
mimetype = ct_info[0]
if mimetype.split('/')[0].lower() != 'multipart':
raise NonMultipartContentTypeException(
"Unexpected mimetype in content-type: '{0}'".format(mimetype)
)
for item in ct_info[1:]:
attr, value = _split_on_find(
item,
'='
)
if attr.lower() == 'boundary':
self.boundary = encode_with(value.strip('"'), self.encoding)
@staticmethod
def _fix_first_part(part, boundary_marker):
bm_len = len(boundary_marker)
if boundary_marker == part[:bm_len]:
return part[bm_len:]
else:
return part
def _parse_body(self, content):
boundary = b''.join((b'--', self.boundary))
def body_part(part):
fixed = MultipartDecoder._fix_first_part(part, boundary)
return BodyPart(fixed, self.encoding)
def test_part(part):
return (part != b'' and
part != b'\r\n' and
part[:4] != b'--\r\n' and
part != b'--')
parts = content.split(b''.join((b'\r\n', boundary)))
self.parts = tuple(body_part(x) for x in parts if test_part(x))
@classmethod
def from_response(cls, response, encoding='utf-8'):
content = response.content
content_type = response.headers.get('content-type', None)
return cls(content, content_type, encoding)

655
thirdparty/requests_toolbelt/multipart/encoder.py vendored Normal file
View File

@ -0,0 +1,655 @@
# -*- coding: utf-8 -*-
"""
requests_toolbelt.multipart.encoder
===================================
This holds all of the implementation details of the MultipartEncoder
"""
import contextlib
import io
import os
from uuid import uuid4
import requests
from .._compat import fields
class FileNotSupportedError(Exception):
"""File not supported error."""
class MultipartEncoder(object):
"""
The ``MultipartEncoder`` object is a generic interface to the engine that
will create a ``multipart/form-data`` body for you.
The basic usage is:
.. code-block:: python
import requests
from requests_toolbelt import MultipartEncoder
encoder = MultipartEncoder({'field': 'value',
'other_field', 'other_value'})
r = requests.post('https://httpbin.org/post', data=encoder,
headers={'Content-Type': encoder.content_type})
If you do not need to take advantage of streaming the post body, you can
also do:
.. code-block:: python
r = requests.post('https://httpbin.org/post',
data=encoder.to_string(),
headers={'Content-Type': encoder.content_type})
If you want the encoder to use a specific order, you can use an
OrderedDict or more simply, a list of tuples:
.. code-block:: python
encoder = MultipartEncoder([('field', 'value'),
('other_field', 'other_value')])
.. versionchanged:: 0.4.0
You can also provide tuples as part values as you would provide them to
requests' ``files`` parameter.
.. code-block:: python
encoder = MultipartEncoder({
'field': ('file_name', b'{"a": "b"}', 'application/json',
{'X-My-Header': 'my-value'})
])
.. warning::
This object will end up directly in :mod:`httplib`. Currently,
:mod:`httplib` has a hard-coded read size of **8192 bytes**. This
means that it will loop until the file has been read and your upload
could take a while. This is **not** a bug in requests. A feature is
being considered for this object to allow you, the user, to specify
what size should be returned on a read. If you have opinions on this,
please weigh in on `this issue`_.
.. _this issue:
https://github.com/requests/toolbelt/issues/75
"""
def __init__(self, fields, boundary=None, encoding='utf-8'):
#: Boundary value either passed in by the user or created
self.boundary_value = boundary or uuid4().hex
# Computed boundary
self.boundary = '--{0}'.format(self.boundary_value)
#: Encoding of the data being passed in
self.encoding = encoding
# Pre-encoded boundary
self._encoded_boundary = b''.join([
encode_with(self.boundary, self.encoding),
encode_with('\r\n', self.encoding)
])
#: Fields provided by the user
self.fields = fields
#: Whether or not the encoder is finished
self.finished = False
#: Pre-computed parts of the upload
self.parts = []
# Pre-computed parts iterator
self._iter_parts = iter([])
# The part we're currently working with
self._current_part = None
# Cached computation of the body's length
self._len = None
# Our buffer
self._buffer = CustomBytesIO(encoding=encoding)
# Pre-compute each part's headers
self._prepare_parts()
# Load boundary into buffer
self._write_boundary()
@property
def len(self):
"""Length of the multipart/form-data body.
requests will first attempt to get the length of the body by calling
``len(body)`` and then by checking for the ``len`` attribute.
On 32-bit systems, the ``__len__`` method cannot return anything
larger than an integer (in C) can hold. If the total size of the body
is even slightly larger than 4GB users will see an OverflowError. This
manifested itself in `bug #80`_.
As such, we now calculate the length lazily as a property.
.. _bug #80:
https://github.com/requests/toolbelt/issues/80
"""
# If _len isn't already calculated, calculate, return, and set it
return self._len or self._calculate_length()
def __repr__(self):
return '<MultipartEncoder: {0!r}>'.format(self.fields)
def _calculate_length(self):
"""
This uses the parts to calculate the length of the body.
This returns the calculated length so __len__ can be lazy.
"""
boundary_len = len(self.boundary) # Length of --{boundary}
# boundary length + header length + body length + len('\r\n') * 2
self._len = sum(
(boundary_len + total_len(p) + 4) for p in self.parts
) + boundary_len + 4
return self._len
def _calculate_load_amount(self, read_size):
"""This calculates how many bytes need to be added to the buffer.
When a consumer read's ``x`` from the buffer, there are two cases to
satisfy:
1. Enough data in the buffer to return the requested amount
2. Not enough data
This function uses the amount of unread bytes in the buffer and
determines how much the Encoder has to load before it can return the
requested amount of bytes.
:param int read_size: the number of bytes the consumer requests
:returns: int -- the number of bytes that must be loaded into the
buffer before the read can be satisfied. This will be strictly
non-negative
"""
amount = read_size - total_len(self._buffer)
return amount if amount > 0 else 0
def _load(self, amount):
"""Load ``amount`` number of bytes into the buffer."""
self._buffer.smart_truncate()
part = self._current_part or self._next_part()
while amount == -1 or amount > 0:
written = 0
if part and not part.bytes_left_to_write():
written += self._write(b'\r\n')
written += self._write_boundary()
part = self._next_part()
if not part:
written += self._write_closing_boundary()
self.finished = True
break
written += part.write_to(self._buffer, amount)
if amount != -1:
amount -= written
def _next_part(self):
try:
p = self._current_part = next(self._iter_parts)
except StopIteration:
p = None
return p
def _iter_fields(self):
_fields = self.fields
if hasattr(self.fields, 'items'):
_fields = list(self.fields.items())
for k, v in _fields:
file_name = None
file_type = None
file_headers = None
if isinstance(v, (list, tuple)):
if len(v) == 2:
file_name, file_pointer = v
elif len(v) == 3:
file_name, file_pointer, file_type = v
else:
file_name, file_pointer, file_type, file_headers = v
else:
file_pointer = v
field = fields.RequestField(name=k, data=file_pointer,
filename=file_name,
headers=file_headers)
field.make_multipart(content_type=file_type)
yield field
def _prepare_parts(self):
"""This uses the fields provided by the user and creates Part objects.
It populates the `parts` attribute and uses that to create a
generator for iteration.
"""
enc = self.encoding
self.parts = [Part.from_field(f, enc) for f in self._iter_fields()]
self._iter_parts = iter(self.parts)
def _write(self, bytes_to_write):
"""Write the bytes to the end of the buffer.
:param bytes bytes_to_write: byte-string (or bytearray) to append to
the buffer
:returns: int -- the number of bytes written
"""
return self._buffer.append(bytes_to_write)
def _write_boundary(self):
"""Write the boundary to the end of the buffer."""
return self._write(self._encoded_boundary)
def _write_closing_boundary(self):
"""Write the bytes necessary to finish a multipart/form-data body."""
with reset(self._buffer):
self._buffer.seek(-2, 2)
self._buffer.write(b'--\r\n')
return 2
def _write_headers(self, headers):
"""Write the current part's headers to the buffer."""
return self._write(encode_with(headers, self.encoding))
@property
def content_type(self):
return str(
'multipart/form-data; boundary={0}'.format(self.boundary_value)
)
def to_string(self):
"""Return the entirety of the data in the encoder.
.. note::
This simply reads all of the data it can. If you have started
streaming or reading data from the encoder, this method will only
return whatever data is left in the encoder.
.. note::
This method affects the internal state of the encoder. Calling
this method will exhaust the encoder.
:returns: the multipart message
:rtype: bytes
"""
return self.read()
def read(self, size=-1):
"""Read data from the streaming encoder.
:param int size: (optional), If provided, ``read`` will return exactly
that many bytes. If it is not provided, it will return the
remaining bytes.
:returns: bytes
"""
if self.finished:
return self._buffer.read(size)
bytes_to_load = size
if bytes_to_load != -1 and bytes_to_load is not None:
bytes_to_load = self._calculate_load_amount(int(size))
self._load(bytes_to_load)
return self._buffer.read(size)
def IDENTITY(monitor):
return monitor
class MultipartEncoderMonitor(object):
"""
An object used to monitor the progress of a :class:`MultipartEncoder`.
The :class:`MultipartEncoder` should only be responsible for preparing and
streaming the data. For anyone who wishes to monitor it, they shouldn't be
using that instance to manage that as well. Using this class, they can
monitor an encoder and register a callback. The callback receives the
instance of the monitor.
To use this monitor, you construct your :class:`MultipartEncoder` as you
normally would.
.. code-block:: python
from requests_toolbelt import (MultipartEncoder,
MultipartEncoderMonitor)
import requests
def callback(monitor):
# Do something with this information
pass
m = MultipartEncoder(fields={'field0': 'value0'})
monitor = MultipartEncoderMonitor(m, callback)
headers = {'Content-Type': monitor.content_type}
r = requests.post('https://httpbin.org/post', data=monitor,
headers=headers)
Alternatively, if your use case is very simple, you can use the following
pattern.
.. code-block:: python
from requests_toolbelt import MultipartEncoderMonitor
import requests
def callback(monitor):
# Do something with this information
pass
monitor = MultipartEncoderMonitor.from_fields(
fields={'field0': 'value0'}, callback
)
headers = {'Content-Type': montior.content_type}
r = requests.post('https://httpbin.org/post', data=monitor,
headers=headers)
"""
def __init__(self, encoder, callback=None):
#: Instance of the :class:`MultipartEncoder` being monitored
self.encoder = encoder
#: Optionally function to call after a read
self.callback = callback or IDENTITY
#: Number of bytes already read from the :class:`MultipartEncoder`
#: instance
self.bytes_read = 0
#: Avoid the same problem in bug #80
self.len = self.encoder.len
@classmethod
def from_fields(cls, fields, boundary=None, encoding='utf-8',
callback=None):
encoder = MultipartEncoder(fields, boundary, encoding)
return cls(encoder, callback)
@property
def content_type(self):
return self.encoder.content_type
def to_string(self):
return self.read()
def read(self, size=-1):
string = self.encoder.read(size)
self.bytes_read += len(string)
self.callback(self)
return string
def encode_with(string, encoding):
"""Encoding ``string`` with ``encoding`` if necessary.
:param str string: If string is a bytes object, it will not encode it.
Otherwise, this function will encode it with the provided encoding.
:param str encoding: The encoding with which to encode string.
:returns: encoded bytes object
"""
if not (string is None or isinstance(string, bytes)):
return string.encode(encoding)
return string
def readable_data(data, encoding):
"""Coerce the data to an object with a ``read`` method."""
if hasattr(data, 'read'):
return data
return CustomBytesIO(data, encoding)
def total_len(o):
if hasattr(o, '__len__'):
return len(o)
if hasattr(o, 'len'):
return o.len
if hasattr(o, 'fileno'):
try:
fileno = o.fileno()
except io.UnsupportedOperation:
pass
else:
return os.fstat(fileno).st_size
if hasattr(o, 'getvalue'):
# e.g. BytesIO, cStringIO.StringIO
return len(o.getvalue())
@contextlib.contextmanager
def reset(buffer):
"""Keep track of the buffer's current position and write to the end.
This is a context manager meant to be used when adding data to the buffer.
It eliminates the need for every function to be concerned with the
position of the cursor in the buffer.
"""
original_position = buffer.tell()
buffer.seek(0, 2)
yield
buffer.seek(original_position, 0)
def coerce_data(data, encoding):
"""Ensure that every object's __len__ behaves uniformly."""
if not isinstance(data, CustomBytesIO):
if hasattr(data, 'getvalue'):
return CustomBytesIO(data.getvalue(), encoding)
if hasattr(data, 'fileno'):
return FileWrapper(data)
if not hasattr(data, 'read'):
return CustomBytesIO(data, encoding)
return data
def to_list(fields):
if hasattr(fields, 'items'):
return list(fields.items())
return list(fields)
class Part(object):
def __init__(self, headers, body):
self.headers = headers
self.body = body
self.headers_unread = True
self.len = len(self.headers) + total_len(self.body)
@classmethod
def from_field(cls, field, encoding):
"""Create a part from a Request Field generated by urllib3."""
headers = encode_with(field.render_headers(), encoding)
body = coerce_data(field.data, encoding)
return cls(headers, body)
def bytes_left_to_write(self):
"""Determine if there are bytes left to write.
:returns: bool -- ``True`` if there are bytes left to write, otherwise
``False``
"""
to_read = 0
if self.headers_unread:
to_read += len(self.headers)
return (to_read + total_len(self.body)) > 0
def write_to(self, buffer, size):
"""Write the requested amount of bytes to the buffer provided.
The number of bytes written may exceed size on the first read since we
load the headers ambitiously.
:param CustomBytesIO buffer: buffer we want to write bytes to
:param int size: number of bytes requested to be written to the buffer
:returns: int -- number of bytes actually written
"""
written = 0
if self.headers_unread:
written += buffer.append(self.headers)
self.headers_unread = False
while total_len(self.body) > 0 and (size == -1 or written < size):
amount_to_read = size
if size != -1:
amount_to_read = size - written
written += buffer.append(self.body.read(amount_to_read))
return written
class CustomBytesIO(io.BytesIO):
def __init__(self, buffer=None, encoding='utf-8'):
buffer = encode_with(buffer, encoding)
super(CustomBytesIO, self).__init__(buffer)
def _get_end(self):
current_pos = self.tell()
self.seek(0, 2)
length = self.tell()
self.seek(current_pos, 0)
return length
@property
def len(self):
length = self._get_end()
return length - self.tell()
def append(self, bytes):
with reset(self):
written = self.write(bytes)
return written
def smart_truncate(self):
to_be_read = total_len(self)
already_read = self._get_end() - to_be_read
if already_read >= to_be_read:
old_bytes = self.read()
self.seek(0, 0)
self.truncate()
self.write(old_bytes)
self.seek(0, 0) # We want to be at the beginning
class FileWrapper(object):
def __init__(self, file_object):
self.fd = file_object
@property
def len(self):
return total_len(self.fd) - self.fd.tell()
def read(self, length=-1):
return self.fd.read(length)
class FileFromURLWrapper(object):
"""File from URL wrapper.
The :class:`FileFromURLWrapper` object gives you the ability to stream file
from provided URL in chunks by :class:`MultipartEncoder`.
Provide a stateless solution for streaming file from one server to another.
You can use the :class:`FileFromURLWrapper` without a session or with
a session as demonstated by the examples below:
.. code-block:: python
# no session
import requests
from requests_toolbelt import MultipartEncoder, FileFromURLWrapper
url = 'https://httpbin.org/image/png'
streaming_encoder = MultipartEncoder(
fields={
'file': FileFromURLWrapper(url)
}
)
r = requests.post(
'https://httpbin.org/post', data=streaming_encoder,
headers={'Content-Type': streaming_encoder.content_type}
)
.. code-block:: python
# using a session
import requests
from requests_toolbelt import MultipartEncoder, FileFromURLWrapper
session = requests.Session()
url = 'https://httpbin.org/image/png'
streaming_encoder = MultipartEncoder(
fields={
'file': FileFromURLWrapper(url, session=session)
}
)
r = session.post(
'https://httpbin.org/post', data=streaming_encoder,
headers={'Content-Type': streaming_encoder.content_type}
)
"""
def __init__(self, file_url, session=None):
self.session = session or requests.Session()
requested_file = self._request_for_file(file_url)
self.len = int(requested_file.headers['content-length'])
self.raw_data = requested_file.raw
def _request_for_file(self, file_url):
"""Make call for file under provided URL."""
response = self.session.get(file_url, stream=True)
content_length = response.headers.get('content-length', None)
if content_length is None:
error_msg = (
"Data from provided URL {url} is not supported. Lack of "
"content-length Header in requested file response.".format(
url=file_url)
)
raise FileNotSupportedError(error_msg)
elif not content_length.isdigit():
error_msg = (
"Data from provided URL {url} is not supported. content-length"
" header value is not a digit.".format(url=file_url)
)
raise FileNotSupportedError(error_msg)
return response
def read(self, chunk_size):
"""Read file in chunks."""
chunk_size = chunk_size if chunk_size >= 0 else self.len
chunk = self.raw_data.read(chunk_size) or b''
self.len -= len(chunk) if chunk else 0 # left to read
return chunk

70
thirdparty/requests_toolbelt/sessions.py vendored Normal file
View File

@ -0,0 +1,70 @@
import requests
from ._compat import urljoin
class BaseUrlSession(requests.Session):
"""A Session with a URL that all requests will use as a base.
Let's start by looking at an example:
.. code-block:: python
>>> from requests_toolbelt import sessions
>>> s = sessions.BaseUrlSession(
... base_url='https://example.com/resource/')
>>> r = s.get('sub-resource/' params={'foo': 'bar'})
>>> print(r.request.url)
https://example.com/resource/sub-resource/?foo=bar
Our call to the ``get`` method will make a request to the URL passed in
when we created the Session and the partial resource name we provide.
We implement this by overriding the ``request`` method so most uses of a
Session are covered. (This, however, precludes the use of PreparedRequest
objects).
.. note::
The base URL that you provide and the path you provide are **very**
important.
Let's look at another *similar* example
.. code-block:: python
>>> from requests_toolbelt import sessions
>>> s = sessions.BaseUrlSession(
... base_url='https://example.com/resource/')
>>> r = s.get('/sub-resource/' params={'foo': 'bar'})
>>> print(r.request.url)
https://example.com/sub-resource/?foo=bar
The key difference here is that we called ``get`` with ``/sub-resource/``,
i.e., there was a leading ``/``. This changes how we create the URL
because we rely on :mod:`urllib.parse.urljoin`.
To override how we generate the URL, sub-class this method and override the
``create_url`` method.
Based on implementation from
https://github.com/kennethreitz/requests/issues/2554#issuecomment-109341010
"""
base_url = None
def __init__(self, base_url=None):
if base_url:
self.base_url = base_url
super(BaseUrlSession, self).__init__()
def request(self, method, url, *args, **kwargs):
"""Send the request after generating the complete URL."""
url = self.create_url(url)
return super(BaseUrlSession, self).request(
method, url, *args, **kwargs
)
def create_url(self, url):
"""Create the URL based off this partial path."""
return urljoin(self.base_url, url)

116
thirdparty/requests_toolbelt/streaming_iterator.py vendored Normal file
View File

@ -0,0 +1,116 @@
# -*- coding: utf-8 -*-
"""
requests_toolbelt.streaming_iterator
====================================
This holds the implementation details for the :class:`StreamingIterator`. It
is designed for the case where you, the user, know the size of the upload but
need to provide the data as an iterator. This class will allow you to specify
the size and stream the data without using a chunked transfer-encoding.
"""
from requests.utils import super_len
from .multipart.encoder import CustomBytesIO, encode_with
class StreamingIterator(object):
"""
This class provides a way of allowing iterators with a known size to be
streamed instead of chunked.
In requests, if you pass in an iterator it assumes you want to use
chunked transfer-encoding to upload the data, which not all servers
support well. Additionally, you may want to set the content-length
yourself to avoid this but that will not work. The only way to preempt
requests using a chunked transfer-encoding and forcing it to stream the
uploads is to mimic a very specific interace. Instead of having to know
these details you can instead just use this class. You simply provide the
size and iterator and pass the instance of StreamingIterator to requests
via the data parameter like so:
.. code-block:: python
from requests_toolbelt import StreamingIterator
import requests
# Let iterator be some generator that you already have and size be
# the size of the data produced by the iterator
r = requests.post(url, data=StreamingIterator(size, iterator))
You can also pass file-like objects to :py:class:`StreamingIterator` in
case requests can't determize the filesize itself. This is the case with
streaming file objects like ``stdin`` or any sockets. Wrapping e.g. files
that are on disk with ``StreamingIterator`` is unnecessary, because
requests can determine the filesize itself.
Naturally, you should also set the `Content-Type` of your upload
appropriately because the toolbelt will not attempt to guess that for you.
"""
def __init__(self, size, iterator, encoding='utf-8'):
#: The expected size of the upload
self.size = int(size)
if self.size < 0:
raise ValueError(
'The size of the upload must be a positive integer'
)
#: Attribute that requests will check to determine the length of the
#: body. See bug #80 for more details
self.len = self.size
#: Encoding the input data is using
self.encoding = encoding
#: The iterator used to generate the upload data
self.iterator = iterator
if hasattr(iterator, 'read'):
self._file = iterator
else:
self._file = _IteratorAsBinaryFile(iterator, encoding)
def read(self, size=-1):
return encode_with(self._file.read(size), self.encoding)
class _IteratorAsBinaryFile(object):
def __init__(self, iterator, encoding='utf-8'):
#: The iterator used to generate the upload data
self.iterator = iterator
#: Encoding the iterator is using
self.encoding = encoding
# The buffer we use to provide the correct number of bytes requested
# during a read
self._buffer = CustomBytesIO()
def _get_bytes(self):
try:
return encode_with(next(self.iterator), self.encoding)
except StopIteration:
return b''
def _load_bytes(self, size):
self._buffer.smart_truncate()
amount_to_load = size - super_len(self._buffer)
bytes_to_append = True
while amount_to_load > 0 and bytes_to_append:
bytes_to_append = self._get_bytes()
amount_to_load -= self._buffer.append(bytes_to_append)
def read(self, size=-1):
size = int(size)
if size == -1:
return b''.join(self.iterator)
self._load_bytes(size)
return self._buffer.read(size)

97
thirdparty/requests_toolbelt/threaded/__init__.py vendored Normal file
View File

@ -0,0 +1,97 @@
"""
This module provides the API for ``requests_toolbelt.threaded``.
The module provides a clean and simple API for making requests via a thread
pool. The thread pool will use sessions for increased performance.
A simple use-case is:
.. code-block:: python
from requests_toolbelt import threaded
urls_to_get = [{
'url': 'https://api.github.com/users/sigmavirus24',
'method': 'GET',
}, {
'url': 'https://api.github.com/repos/requests/toolbelt',
'method': 'GET',
}, {
'url': 'https://google.com',
'method': 'GET',
}]
responses, errors = threaded.map(urls_to_get)
By default, the threaded submodule will detect the number of CPUs your
computer has and use that if no other number of processes is selected. To
change this, always use the keyword argument ``num_processes``. Using the
above example, we would expand it like so:
.. code-block:: python
responses, errors = threaded.map(urls_to_get, num_processes=10)
You can also customize how a :class:`requests.Session` is initialized by
creating a callback function:
.. code-block:: python
from requests_toolbelt import user_agent
def initialize_session(session):
session.headers['User-Agent'] = user_agent('my-scraper', '0.1')
session.headers['Accept'] = 'application/json'
responses, errors = threaded.map(urls_to_get,
initializer=initialize_session)
.. autofunction:: requests_toolbelt.threaded.map
Inspiration is blatantly drawn from the standard library's multiprocessing
library. See the following references:
- multiprocessing's `pool source`_
- map and map_async `inspiration`_
.. _pool source:
https://hg.python.org/cpython/file/8ef4f75a8018/Lib/multiprocessing/pool.py
.. _inspiration:
https://hg.python.org/cpython/file/8ef4f75a8018/Lib/multiprocessing/pool.py#l340
"""
from . import pool
from .._compat import queue
def map(requests, **kwargs):
r"""Simple interface to the threaded Pool object.
This function takes a list of dictionaries representing requests to make
using Sessions in threads and returns a tuple where the first item is
a generator of successful responses and the second is a generator of
exceptions.
:param list requests:
Collection of dictionaries representing requests to make with the Pool
object.
:param \*\*kwargs:
Keyword arguments that are passed to the
:class:`~requests_toolbelt.threaded.pool.Pool` object.
:returns: Tuple of responses and exceptions from the pool
:rtype: (:class:`~requests_toolbelt.threaded.pool.ThreadResponse`,
:class:`~requests_toolbelt.threaded.pool.ThreadException`)
"""
if not (requests and all(isinstance(r, dict) for r in requests)):
raise ValueError('map expects a list of dictionaries.')
# Build our queue of requests
job_queue = queue.Queue()
for request in requests:
job_queue.put(request)
# Ensure the user doesn't try to pass their own job_queue
kwargs['job_queue'] = job_queue
threadpool = pool.Pool(**kwargs)
threadpool.join_all()
return threadpool.responses(), threadpool.exceptions()

211
thirdparty/requests_toolbelt/threaded/pool.py vendored Normal file
View File

@ -0,0 +1,211 @@
"""Module implementing the Pool for :mod:``requests_toolbelt.threaded``."""
import multiprocessing
import requests
from . import thread
from .._compat import queue
class Pool(object):
"""Pool that manages the threads containing sessions.
:param queue:
The queue you're expected to use to which you should add items.
:type queue: queue.Queue
:param initializer:
Function used to initialize an instance of ``session``.
:type initializer: collections.Callable
:param auth_generator:
Function used to generate new auth credentials for the session.
:type auth_generator: collections.Callable
:param int num_process:
Number of threads to create.
:param session:
:type session: requests.Session
"""
def __init__(self, job_queue, initializer=None, auth_generator=None,
num_processes=None, session=requests.Session):
if num_processes is None:
num_processes = multiprocessing.cpu_count() or 1
if num_processes < 1:
raise ValueError("Number of processes should at least be 1.")
self._job_queue = job_queue
self._response_queue = queue.Queue()
self._exc_queue = queue.Queue()
self._processes = num_processes
self._initializer = initializer or _identity
self._auth = auth_generator or _identity
self._session = session
self._pool = [
thread.SessionThread(self._new_session(), self._job_queue,
self._response_queue, self._exc_queue)
for _ in range(self._processes)
]
def _new_session(self):
return self._auth(self._initializer(self._session()))
@classmethod
def from_exceptions(cls, exceptions, **kwargs):
r"""Create a :class:`~Pool` from an :class:`~ThreadException`\ s.
Provided an iterable that provides :class:`~ThreadException` objects,
this classmethod will generate a new pool to retry the requests that
caused the exceptions.
:param exceptions:
Iterable that returns :class:`~ThreadException`
:type exceptions: iterable
:param kwargs:
Keyword arguments passed to the :class:`~Pool` initializer.
:returns: An initialized :class:`~Pool` object.
:rtype: :class:`~Pool`
"""
job_queue = queue.Queue()
for exc in exceptions:
job_queue.put(exc.request_kwargs)
return cls(job_queue=job_queue, **kwargs)
@classmethod
def from_urls(cls, urls, request_kwargs=None, **kwargs):
"""Create a :class:`~Pool` from an iterable of URLs.
:param urls:
Iterable that returns URLs with which we create a pool.
:type urls: iterable
:param dict request_kwargs:
Dictionary of other keyword arguments to provide to the request
method.
:param kwargs:
Keyword arguments passed to the :class:`~Pool` initializer.
:returns: An initialized :class:`~Pool` object.
:rtype: :class:`~Pool`
"""
request_dict = {'method': 'GET'}
request_dict.update(request_kwargs or {})
job_queue = queue.Queue()
for url in urls:
job = request_dict.copy()
job.update({'url': url})
job_queue.put(job)
return cls(job_queue=job_queue, **kwargs)
def exceptions(self):
"""Iterate over all the exceptions in the pool.
:returns: Generator of :class:`~ThreadException`
"""
while True:
exc = self.get_exception()
if exc is None:
break
yield exc
def get_exception(self):
"""Get an exception from the pool.
:rtype: :class:`~ThreadException`
"""
try:
(request, exc) = self._exc_queue.get_nowait()
except queue.Empty:
return None
else:
return ThreadException(request, exc)
def get_response(self):
"""Get a response from the pool.
:rtype: :class:`~ThreadResponse`
"""
try:
(request, response) = self._response_queue.get_nowait()
except queue.Empty:
return None
else:
return ThreadResponse(request, response)
def responses(self):
"""Iterate over all the responses in the pool.
:returns: Generator of :class:`~ThreadResponse`
"""
while True:
resp = self.get_response()
if resp is None:
break
yield resp
def join_all(self):
"""Join all the threads to the master thread."""
for session_thread in self._pool:
session_thread.join()
class ThreadProxy(object):
proxied_attr = None
def __getattr__(self, attr):
"""Proxy attribute accesses to the proxied object."""
get = object.__getattribute__
if attr not in self.attrs:
response = get(self, self.proxied_attr)
return getattr(response, attr)
else:
return get(self, attr)
class ThreadResponse(ThreadProxy):
"""A wrapper around a requests Response object.
This will proxy most attribute access actions to the Response object. For
example, if you wanted the parsed JSON from the response, you might do:
.. code-block:: python
thread_response = pool.get_response()
json = thread_response.json()
"""
proxied_attr = 'response'
attrs = frozenset(['request_kwargs', 'response'])
def __init__(self, request_kwargs, response):
#: The original keyword arguments provided to the queue
self.request_kwargs = request_kwargs
#: The wrapped response
self.response = response
class ThreadException(ThreadProxy):
"""A wrapper around an exception raised during a request.
This will proxy most attribute access actions to the exception object. For
example, if you wanted the message from the exception, you might do:
.. code-block:: python
thread_exc = pool.get_exception()
msg = thread_exc.message
"""
proxied_attr = 'exception'
attrs = frozenset(['request_kwargs', 'exception'])
def __init__(self, request_kwargs, exception):
#: The original keyword arguments provided to the queue
self.request_kwargs = request_kwargs
#: The captured and wrapped exception
self.exception = exception
def _identity(session_obj):
return session_obj
__all__ = ['ThreadException', 'ThreadResponse', 'Pool']

53
thirdparty/requests_toolbelt/threaded/thread.py vendored Normal file
View File

@ -0,0 +1,53 @@
"""Module containing the SessionThread class."""
import threading
import uuid
import requests.exceptions as exc
from .._compat import queue
class SessionThread(object):
def __init__(self, initialized_session, job_queue, response_queue,
exception_queue):
self._session = initialized_session
self._jobs = job_queue
self._create_worker()
self._responses = response_queue
self._exceptions = exception_queue
def _create_worker(self):
self._worker = threading.Thread(
target=self._make_request,
name=uuid.uuid4(),
)
self._worker.daemon = True
self._worker._state = 0
self._worker.start()
def _handle_request(self, kwargs):
try:
response = self._session.request(**kwargs)
except exc.RequestException as e:
self._exceptions.put((kwargs, e))
else:
self._responses.put((kwargs, response))
finally:
self._jobs.task_done()
def _make_request(self):
while True:
try:
kwargs = self._jobs.get_nowait()
except queue.Empty:
break
self._handle_request(kwargs)
def is_alive(self):
"""Proxy to the thread's ``is_alive`` method."""
return self._worker.is_alive()
def join(self):
"""Join this thread to the master thread."""
self._worker.join()

0
thirdparty/requests_toolbelt/utils/__init__.py vendored Normal file
View File

91
thirdparty/requests_toolbelt/utils/deprecated.py vendored Normal file
View File

@ -0,0 +1,91 @@
# -*- coding: utf-8 -*-
"""A collection of functions deprecated in requests.utils."""
import re
import sys
from requests import utils
find_charset = re.compile(
br'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I
).findall
find_pragma = re.compile(
br'<meta.*?content=["\']*;?charset=(.+?)["\'>]', flags=re.I
).findall
find_xml = re.compile(
br'^<\?xml.*?encoding=["\']*(.+?)["\'>]'
).findall
def get_encodings_from_content(content):
"""Return encodings from given content string.
.. code-block:: python
import requests
from requests_toolbelt.utils import deprecated
r = requests.get(url)
encodings = deprecated.get_encodings_from_content(r)
:param content: bytestring to extract encodings from
:type content: bytes
:return: encodings detected in the provided content
:rtype: list(str)
"""
encodings = (find_charset(content) + find_pragma(content)
+ find_xml(content))
if (3, 0) <= sys.version_info < (4, 0):
encodings = [encoding.decode('utf8') for encoding in encodings]
return encodings
def get_unicode_from_response(response):
"""Return the requested content back in unicode.
This will first attempt to retrieve the encoding from the response
headers. If that fails, it will use
:func:`requests_toolbelt.utils.deprecated.get_encodings_from_content`
to determine encodings from HTML elements.
.. code-block:: python
import requests
from requests_toolbelt.utils import deprecated
r = requests.get(url)
text = deprecated.get_unicode_from_response(r)
:param response: Response object to get unicode content from.
:type response: requests.models.Response
"""
tried_encodings = set()
# Try charset from content-type
encoding = utils.get_encoding_from_headers(response.headers)
if encoding:
try:
return str(response.content, encoding)
except UnicodeError:
tried_encodings.add(encoding.lower())
encodings = get_encodings_from_content(response.content)
for _encoding in encodings:
_encoding = _encoding.lower()
if _encoding in tried_encodings:
continue
try:
return str(response.content, _encoding)
except UnicodeError:
tried_encodings.add(_encoding)
# Fall back:
if encoding:
try:
return str(response.content, encoding, errors='replace')
except TypeError:
pass
return response.text

197
thirdparty/requests_toolbelt/utils/dump.py vendored Normal file
View File

@ -0,0 +1,197 @@
"""This module provides functions for dumping information about responses."""
import collections
from requests import compat
__all__ = ('dump_response', 'dump_all')
HTTP_VERSIONS = {
9: b'0.9',
10: b'1.0',
11: b'1.1',
}
_PrefixSettings = collections.namedtuple('PrefixSettings',
['request', 'response'])
class PrefixSettings(_PrefixSettings):
def __new__(cls, request, response):
request = _coerce_to_bytes(request)
response = _coerce_to_bytes(response)
return super(PrefixSettings, cls).__new__(cls, request, response)
def _get_proxy_information(response):
if getattr(response.connection, 'proxy_manager', False):
proxy_info = {}
request_url = response.request.url
if request_url.startswith('https://'):
proxy_info['method'] = 'CONNECT'
proxy_info['request_path'] = request_url
return proxy_info
return None
def _format_header(name, value):
return (_coerce_to_bytes(name) + b': ' + _coerce_to_bytes(value) +
b'\r\n')
def _build_request_path(url, proxy_info):
uri = compat.urlparse(url)
proxy_url = proxy_info.get('request_path')
if proxy_url is not None:
request_path = _coerce_to_bytes(proxy_url)
return request_path, uri
request_path = _coerce_to_bytes(uri.path)
if uri.query:
request_path += b'?' + _coerce_to_bytes(uri.query)
return request_path, uri
def _dump_request_data(request, prefixes, bytearr, proxy_info=None):
if proxy_info is None:
proxy_info = {}
prefix = prefixes.request
method = _coerce_to_bytes(proxy_info.pop('method', request.method))
request_path, uri = _build_request_path(request.url, proxy_info)
# <prefix><METHOD> <request-path> HTTP/1.1
bytearr.extend(prefix + method + b' ' + request_path + b' HTTP/1.1\r\n')
# <prefix>Host: <request-host> OR host header specified by user
headers = request.headers.copy()
host_header = _coerce_to_bytes(headers.pop('Host', uri.netloc))
bytearr.extend(prefix + b'Host: ' + host_header + b'\r\n')
for name, value in headers.items():
bytearr.extend(prefix + _format_header(name, value))
bytearr.extend(prefix + b'\r\n')
if request.body:
if isinstance(request.body, compat.basestring):
bytearr.extend(prefix + _coerce_to_bytes(request.body))
else:
# In the event that the body is a file-like object, let's not try
# to read everything into memory.
bytearr.extend(b'<< Request body is not a string-like type >>')
bytearr.extend(b'\r\n')
def _dump_response_data(response, prefixes, bytearr):
prefix = prefixes.response
# Let's interact almost entirely with urllib3's response
raw = response.raw
# Let's convert the version int from httplib to bytes
version_str = HTTP_VERSIONS.get(raw.version, b'?')
# <prefix>HTTP/<version_str> <status_code> <reason>
bytearr.extend(prefix + b'HTTP/' + version_str + b' ' +
str(raw.status).encode('ascii') + b' ' +
_coerce_to_bytes(response.reason) + b'\r\n')
headers = raw.headers
for name in headers.keys():
for value in headers.getlist(name):
bytearr.extend(prefix + _format_header(name, value))
bytearr.extend(prefix + b'\r\n')
bytearr.extend(response.content)
def _coerce_to_bytes(data):
if not isinstance(data, bytes) and hasattr(data, 'encode'):
data = data.encode('utf-8')
# Don't bail out with an exception if data is None
return data if data is not None else b''
def dump_response(response, request_prefix=b'< ', response_prefix=b'> ',
data_array=None):
"""Dump a single request-response cycle's information.
This will take a response object and dump only the data that requests can
see for that single request-response cycle.
Example::
import requests
from requests_toolbelt.utils import dump
resp = requests.get('https://api.github.com/users/sigmavirus24')
data = dump.dump_response(resp)
print(data.decode('utf-8'))
:param response:
The response to format
:type response: :class:`requests.Response`
:param request_prefix: (*optional*)
Bytes to prefix each line of the request data
:type request_prefix: :class:`bytes`
:param response_prefix: (*optional*)
Bytes to prefix each line of the response data
:type response_prefix: :class:`bytes`
:param data_array: (*optional*)
Bytearray to which we append the request-response cycle data
:type data_array: :class:`bytearray`
:returns: Formatted bytes of request and response information.
:rtype: :class:`bytearray`
"""
data = data_array if data_array is not None else bytearray()
prefixes = PrefixSettings(request_prefix, response_prefix)
if not hasattr(response, 'request'):
raise ValueError('Response has no associated request')
proxy_info = _get_proxy_information(response)
_dump_request_data(response.request, prefixes, data,
proxy_info=proxy_info)
_dump_response_data(response, prefixes, data)
return data
def dump_all(response, request_prefix=b'< ', response_prefix=b'> '):
"""Dump all requests and responses including redirects.
This takes the response returned by requests and will dump all
request-response pairs in the redirect history in order followed by the
final request-response.
Example::
import requests
from requests_toolbelt.utils import dump
resp = requests.get('https://httpbin.org/redirect/5')
data = dump.dump_all(resp)
print(data.decode('utf-8'))
:param response:
The response to format
:type response: :class:`requests.Response`
:param request_prefix: (*optional*)
Bytes to prefix each line of the request data
:type request_prefix: :class:`bytes`
:param response_prefix: (*optional*)
Bytes to prefix each line of the response data
:type response_prefix: :class:`bytes`
:returns: Formatted bytes of request and response information.
:rtype: :class:`bytearray`
"""
data = bytearray()
history = list(response.history[:])
history.append(response)
for response in history:
dump_response(response, request_prefix, response_prefix, data)
return data

108
thirdparty/requests_toolbelt/utils/formdata.py vendored Normal file
View File

@ -0,0 +1,108 @@
# -*- coding: utf-8 -*-
"""Implementation of nested form-data encoding function(s)."""
from .._compat import basestring
from .._compat import urlencode as _urlencode
__all__ = ('urlencode',)
def urlencode(query, *args, **kwargs):
"""Handle nested form-data queries and serialize them appropriately.
There are times when a website expects a nested form data query to be sent
but, the standard library's urlencode function does not appropriately
handle the nested structures. In that case, you need this function which
will flatten the structure first and then properly encode it for you.
When using this to send data in the body of a request, make sure you
specify the appropriate Content-Type header for the request.
.. code-block:: python
import requests
from requests_toolbelt.utils import formdata
query = {
'my_dict': {
'foo': 'bar',
'biz': 'baz",
},
'a': 'b',
}
resp = requests.get(url, params=formdata.urlencode(query))
# or
resp = requests.post(
url,
data=formdata.urlencode(query),
headers={
'Content-Type': 'application/x-www-form-urlencoded'
},
)
Similarly, you can specify a list of nested tuples, e.g.,
.. code-block:: python
import requests
from requests_toolbelt.utils import formdata
query = [
('my_list', [
('foo', 'bar'),
('biz', 'baz'),
]),
('a', 'b'),
]
resp = requests.get(url, params=formdata.urlencode(query))
# or
resp = requests.post(
url,
data=formdata.urlencode(query),
headers={
'Content-Type': 'application/x-www-form-urlencoded'
},
)
For additional parameter and return information, see the official
`urlencode`_ documentation.
.. _urlencode:
https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode
"""
expand_classes = (dict, list, tuple)
original_query_list = _to_kv_list(query)
if not all(_is_two_tuple(i) for i in original_query_list):
raise ValueError("Expected query to be able to be converted to a "
"list comprised of length 2 tuples.")
query_list = original_query_list
while any(isinstance(v, expand_classes) for _, v in query_list):
query_list = _expand_query_values(query_list)
return _urlencode(query_list, *args, **kwargs)
def _to_kv_list(dict_or_list):
if hasattr(dict_or_list, 'items'):
return list(dict_or_list.items())
return dict_or_list
def _is_two_tuple(item):
return isinstance(item, (list, tuple)) and len(item) == 2
def _expand_query_values(original_query_list):
query_list = []
for key, value in original_query_list:
if isinstance(value, basestring):
query_list.append((key, value))
else:
key_fmt = key + '[%s]'
value_list = _to_kv_list(value)
query_list.extend((key_fmt % k, v) for k, v in value_list)
return query_list

143
thirdparty/requests_toolbelt/utils/user_agent.py vendored Normal file
View File

@ -0,0 +1,143 @@
# -*- coding: utf-8 -*-
import collections
import platform
import sys
def user_agent(name, version, extras=None):
"""Return an internet-friendly user_agent string.
The majority of this code has been wilfully stolen from the equivalent
function in Requests.
:param name: The intended name of the user-agent, e.g. "python-requests".
:param version: The version of the user-agent, e.g. "0.0.1".
:param extras: List of two-item tuples that are added to the user-agent
string.
:returns: Formatted user-agent string
:rtype: str
"""
if extras is None:
extras = []
return UserAgentBuilder(
name, version
).include_extras(
extras
).include_implementation(
).include_system().build()
class UserAgentBuilder(object):
"""Class to provide a greater level of control than :func:`user_agent`.
This is used by :func:`user_agent` to build its User-Agent string.
.. code-block:: python
user_agent_str = UserAgentBuilder(
name='requests-toolbelt',
version='17.4.0',
).include_implementation(
).include_system(
).include_extras([
('requests', '2.14.2'),
('urllib3', '1.21.2'),
]).build()
"""
format_string = '%s/%s'
def __init__(self, name, version):
"""Initialize our builder with the name and version of our user agent.
:param str name:
Name of our user-agent.
:param str version:
The version string for user-agent.
"""
self._pieces = collections.deque([(name, version)])
def build(self):
"""Finalize the User-Agent string.
:returns:
Formatted User-Agent string.
:rtype:
str
"""
return " ".join([self.format_string % piece for piece in self._pieces])
def include_extras(self, extras):
"""Include extra portions of the User-Agent.
:param list extras:
list of tuples of extra-name and extra-version
"""
if any(len(extra) != 2 for extra in extras):
raise ValueError('Extras should be a sequence of two item tuples.')
self._pieces.extend(extras)
return self
def include_implementation(self):
"""Append the implementation string to the user-agent string.
This adds the the information that you're using CPython 2.7.13 to the
User-Agent.
"""
self._pieces.append(_implementation_tuple())
return self
def include_system(self):
"""Append the information about the Operating System."""
self._pieces.append(_platform_tuple())
return self
def _implementation_tuple():
"""Return the tuple of interpreter name and version.
Returns a string that provides both the name and the version of the Python
implementation currently running. For example, on CPython 2.7.5 it will
return "CPython/2.7.5".
This function works best on CPython and PyPy: in particular, it probably
doesn't work for Jython or IronPython. Future investigation should be done
to work out the correct shape of the code for those platforms.
"""
implementation = platform.python_implementation()
if implementation == 'CPython':
implementation_version = platform.python_version()
elif implementation == 'PyPy':
implementation_version = '%s.%s.%s' % (sys.pypy_version_info.major,
sys.pypy_version_info.minor,
sys.pypy_version_info.micro)
if sys.pypy_version_info.releaselevel != 'final':
implementation_version = ''.join([
implementation_version, sys.pypy_version_info.releaselevel
])
elif implementation == 'Jython':
implementation_version = platform.python_version() # Complete Guess
elif implementation == 'IronPython':
implementation_version = platform.python_version() # Complete Guess
else:
implementation_version = 'Unknown'
return (implementation, implementation_version)
def _implementation_string():
return "%s/%s" % _implementation_tuple()
def _platform_tuple():
try:
p_system = platform.system()
p_release = platform.release()
except IOError:
p_system = 'Unknown'
p_release = 'Unknown'
return (p_system, p_release)

7
thirdparty/xmltodict/LICENSE vendored Normal file
View File

@ -0,0 +1,7 @@
Copyright (C) 2012 Martin Blech and individual contributors.
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

4
thirdparty/xmltodict/MANIFEST.in vendored Normal file
View File

@ -0,0 +1,4 @@
include README.md
include LICENSE
include ez_setup.py
recursive-include tests *

232
thirdparty/xmltodict/PKG-INFO vendored Normal file
View File

@ -0,0 +1,232 @@
Metadata-Version: 2.1
Name: xmltodict
Version: 0.12.0
Summary: Makes working with XML feel like you are working with JSON
Home-page: https://github.com/martinblech/xmltodict
Author: Martin Blech
Author-email: martinblech@gmail.com
License: MIT
Description: # xmltodict
`xmltodict` is a Python module that makes working with XML feel like you are working with [JSON](http://docs.python.org/library/json.html), as in this ["spec"](http://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html):
[![Build Status](https://secure.travis-ci.org/martinblech/xmltodict.svg)](http://travis-ci.org/martinblech/xmltodict)
```python
>>> print(json.dumps(xmltodict.parse("""
... <mydocument has="an attribute">
... <and>
... <many>elements</many>
... <many>more elements</many>
... </and>
... <plus a="complex">
... element as well
... </plus>
... </mydocument>
... """), indent=4))
{
"mydocument": {
"@has": "an attribute",
"and": {
"many": [
"elements",
"more elements"
]
},
"plus": {
"@a": "complex",
"#text": "element as well"
}
}
}
```
## Namespace support
By default, `xmltodict` does no XML namespace processing (it just treats namespace declarations as regular node attributes), but passing `process_namespaces=True` will make it expand namespaces for you:
```python
>>> xml = """
... <root xmlns="http://defaultns.com/"
... xmlns:a="http://a.com/"
... xmlns:b="http://b.com/">
... <x>1</x>
... <a:y>2</a:y>
... <b:z>3</b:z>
... </root>
... """
>>> xmltodict.parse(xml, process_namespaces=True) == {
... 'http://defaultns.com/:root': {
... 'http://defaultns.com/:x': '1',
... 'http://a.com/:y': '2',
... 'http://b.com/:z': '3',
... }
... }
True
```
It also lets you collapse certain namespaces to shorthand prefixes, or skip them altogether:
```python
>>> namespaces = {
... 'http://defaultns.com/': None, # skip this namespace
... 'http://a.com/': 'ns_a', # collapse "http://a.com/" -> "ns_a"
... }
>>> xmltodict.parse(xml, process_namespaces=True, namespaces=namespaces) == {
... 'root': {
... 'x': '1',
... 'ns_a:y': '2',
... 'http://b.com/:z': '3',
... },
... }
True
```
## Streaming mode
`xmltodict` is very fast ([Expat](http://docs.python.org/library/pyexpat.html)-based) and has a streaming mode with a small memory footprint, suitable for big XML dumps like [Discogs](http://discogs.com/data/) or [Wikipedia](http://dumps.wikimedia.org/):
```python
>>> def handle_artist(_, artist):
... print(artist['name'])
... return True
>>>
>>> xmltodict.parse(GzipFile('discogs_artists.xml.gz'),
... item_depth=2, item_callback=handle_artist)
A Perfect Circle
Fantômas
King Crimson
Chris Potter
...
```
It can also be used from the command line to pipe objects to a script like this:
```python
import sys, marshal
while True:
_, article = marshal.load(sys.stdin)
print(article['title'])
```
```sh
$ bunzip2 enwiki-pages-articles.xml.bz2 | xmltodict.py 2 | myscript.py
AccessibleComputing
Anarchism
AfghanistanHistory
AfghanistanGeography
AfghanistanPeople
AfghanistanCommunications
Autism
...
```
Or just cache the dicts so you don't have to parse that big XML file again. You do this only once:
```sh
$ bunzip2 enwiki-pages-articles.xml.bz2 | xmltodict.py 2 | gzip > enwiki.dicts.gz
```
And you reuse the dicts with every script that needs them:
```sh
$ gunzip enwiki.dicts.gz | script1.py
$ gunzip enwiki.dicts.gz | script2.py
...
```
## Roundtripping
You can also convert in the other direction, using the `unparse()` method:
```python
>>> mydict = {
... 'response': {
... 'status': 'good',
... 'last_updated': '2014-02-16T23:10:12Z',
... }
... }
>>> print(unparse(mydict, pretty=True))
<?xml version="1.0" encoding="utf-8"?>
<response>
<status>good</status>
<last_updated>2014-02-16T23:10:12Z</last_updated>
</response>
```
Text values for nodes can be specified with the `cdata_key` key in the python dict, while node properties can be specified with the `attr_prefix` prefixed to the key name in the python dict. The default value for `attr_prefix` is `@` and the default value for `cdata_key` is `#text`.
```python
>>> import xmltodict
>>>
>>> mydict = {
... 'text': {
... '@color':'red',
... '@stroke':'2',
... '#text':'This is a test'
... }
... }
>>> print(xmltodict.unparse(mydict, pretty=True))
<?xml version="1.0" encoding="utf-8"?>
<text stroke="2" color="red">This is a test</text>
```
## Ok, how do I get it?
### Using pypi
You just need to
```sh
$ pip install xmltodict
```
### RPM-based distro (Fedora, RHEL, …)
There is an [official Fedora package for xmltodict](https://apps.fedoraproject.org/packages/python-xmltodict).
```sh
$ sudo yum install python-xmltodict
```
### Arch Linux
There is an [official Arch Linux package for xmltodict](https://www.archlinux.org/packages/community/any/python-xmltodict/).
```sh
$ sudo pacman -S python-xmltodict
```
### Debian-based distro (Debian, Ubuntu, …)
There is an [official Debian package for xmltodict](https://tracker.debian.org/pkg/python-xmltodict).
```sh
$ sudo apt install python-xmltodict
```
### FreeBSD
There is an [official FreeBSD port for xmltodict](https://svnweb.freebsd.org/ports/head/devel/py-xmltodict/).
```sh
$ pkg install py36-xmltodict
```
Platform: all
Classifier: Intended Audience :: Developers
Classifier: License :: OSI Approved :: MIT License
Classifier: Operating System :: OS Independent
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 2
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.4
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: Implementation :: Jython
Classifier: Programming Language :: Python :: Implementation :: PyPy
Classifier: Topic :: Text Processing :: Markup :: XML
Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
Description-Content-Type: text/markdown

206
thirdparty/xmltodict/README.md vendored Normal file
View File

@ -0,0 +1,206 @@
# xmltodict
`xmltodict` is a Python module that makes working with XML feel like you are working with [JSON](http://docs.python.org/library/json.html), as in this ["spec"](http://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html):
[![Build Status](https://secure.travis-ci.org/martinblech/xmltodict.svg)](http://travis-ci.org/martinblech/xmltodict)
```python
>>> print(json.dumps(xmltodict.parse("""
... <mydocument has="an attribute">
... <and>
... <many>elements</many>
... <many>more elements</many>
... </and>
... <plus a="complex">
... element as well
... </plus>
... </mydocument>
... """), indent=4))
{
"mydocument": {
"@has": "an attribute",
"and": {
"many": [
"elements",
"more elements"
]
},
"plus": {
"@a": "complex",
"#text": "element as well"
}
}
}
```
## Namespace support
By default, `xmltodict` does no XML namespace processing (it just treats namespace declarations as regular node attributes), but passing `process_namespaces=True` will make it expand namespaces for you:
```python
>>> xml = """
... <root xmlns="http://defaultns.com/"
... xmlns:a="http://a.com/"
... xmlns:b="http://b.com/">
... <x>1</x>
... <a:y>2</a:y>
... <b:z>3</b:z>
... </root>
... """
>>> xmltodict.parse(xml, process_namespaces=True) == {
... 'http://defaultns.com/:root': {
... 'http://defaultns.com/:x': '1',
... 'http://a.com/:y': '2',
... 'http://b.com/:z': '3',
... }
... }
True
```
It also lets you collapse certain namespaces to shorthand prefixes, or skip them altogether:
```python
>>> namespaces = {
... 'http://defaultns.com/': None, # skip this namespace
... 'http://a.com/': 'ns_a', # collapse "http://a.com/" -> "ns_a"
... }
>>> xmltodict.parse(xml, process_namespaces=True, namespaces=namespaces) == {
... 'root': {
... 'x': '1',
... 'ns_a:y': '2',
... 'http://b.com/:z': '3',
... },
... }
True
```
## Streaming mode
`xmltodict` is very fast ([Expat](http://docs.python.org/library/pyexpat.html)-based) and has a streaming mode with a small memory footprint, suitable for big XML dumps like [Discogs](http://discogs.com/data/) or [Wikipedia](http://dumps.wikimedia.org/):
```python
>>> def handle_artist(_, artist):
... print(artist['name'])
... return True
>>>
>>> xmltodict.parse(GzipFile('discogs_artists.xml.gz'),
... item_depth=2, item_callback=handle_artist)
A Perfect Circle
Fantômas
King Crimson
Chris Potter
...
```
It can also be used from the command line to pipe objects to a script like this:
```python
import sys, marshal
while True:
_, article = marshal.load(sys.stdin)
print(article['title'])
```
```sh
$ bunzip2 enwiki-pages-articles.xml.bz2 | xmltodict.py 2 | myscript.py
AccessibleComputing
Anarchism
AfghanistanHistory
AfghanistanGeography
AfghanistanPeople
AfghanistanCommunications
Autism
...
```
Or just cache the dicts so you don't have to parse that big XML file again. You do this only once:
```sh
$ bunzip2 enwiki-pages-articles.xml.bz2 | xmltodict.py 2 | gzip > enwiki.dicts.gz
```
And you reuse the dicts with every script that needs them:
```sh
$ gunzip enwiki.dicts.gz | script1.py
$ gunzip enwiki.dicts.gz | script2.py
...
```
## Roundtripping
You can also convert in the other direction, using the `unparse()` method:
```python
>>> mydict = {
... 'response': {
... 'status': 'good',
... 'last_updated': '2014-02-16T23:10:12Z',
... }
... }
>>> print(unparse(mydict, pretty=True))
<?xml version="1.0" encoding="utf-8"?>
<response>
<status>good</status>
<last_updated>2014-02-16T23:10:12Z</last_updated>
</response>
```
Text values for nodes can be specified with the `cdata_key` key in the python dict, while node properties can be specified with the `attr_prefix` prefixed to the key name in the python dict. The default value for `attr_prefix` is `@` and the default value for `cdata_key` is `#text`.
```python
>>> import xmltodict
>>>
>>> mydict = {
... 'text': {
... '@color':'red',
... '@stroke':'2',
... '#text':'This is a test'
... }
... }
>>> print(xmltodict.unparse(mydict, pretty=True))
<?xml version="1.0" encoding="utf-8"?>
<text stroke="2" color="red">This is a test</text>
```
## Ok, how do I get it?
### Using pypi
You just need to
```sh
$ pip install xmltodict
```
### RPM-based distro (Fedora, RHEL, …)
There is an [official Fedora package for xmltodict](https://apps.fedoraproject.org/packages/python-xmltodict).
```sh
$ sudo yum install python-xmltodict
```
### Arch Linux
There is an [official Arch Linux package for xmltodict](https://www.archlinux.org/packages/community/any/python-xmltodict/).
```sh
$ sudo pacman -S python-xmltodict
```
### Debian-based distro (Debian, Ubuntu, …)
There is an [official Debian package for xmltodict](https://tracker.debian.org/pkg/python-xmltodict).
```sh
$ sudo apt install python-xmltodict
```
### FreeBSD
There is an [official FreeBSD port for xmltodict](https://svnweb.freebsd.org/ports/head/devel/py-xmltodict/).
```sh
$ pkg install py36-xmltodict
```

0
thirdparty/xmltodict/__init__.py vendored Normal file
View File

414
thirdparty/xmltodict/ez_setup.py vendored Normal file
View File

@ -0,0 +1,414 @@
#!/usr/bin/env python
"""
Setuptools bootstrapping installer.
Maintained at https://github.com/pypa/setuptools/tree/bootstrap.
Run this script to install or upgrade setuptools.
This method is DEPRECATED. Check https://github.com/pypa/setuptools/issues/581 for more details.
"""
import os
import shutil
import sys
import tempfile
import zipfile
import optparse
import subprocess
import platform
import textwrap
import contextlib
from distutils import log
try:
from urllib.request import urlopen
except ImportError:
from urllib2 import urlopen
try:
from site import USER_SITE
except ImportError:
USER_SITE = None
# 33.1.1 is the last version that supports setuptools self upgrade/installation.
DEFAULT_VERSION = "33.1.1"
DEFAULT_URL = "https://pypi.io/packages/source/s/setuptools/"
DEFAULT_SAVE_DIR = os.curdir
DEFAULT_DEPRECATION_MESSAGE = "ez_setup.py is deprecated and when using it setuptools will be pinned to {0} since it's the last version that supports setuptools self upgrade/installation, check https://github.com/pypa/setuptools/issues/581 for more info; use pip to install setuptools"
MEANINGFUL_INVALID_ZIP_ERR_MSG = 'Maybe {0} is corrupted, delete it and try again.'
log.warn(DEFAULT_DEPRECATION_MESSAGE.format(DEFAULT_VERSION))
def _python_cmd(*args):
"""
Execute a command.
Return True if the command succeeded.
"""
args = (sys.executable,) + args
return subprocess.call(args) == 0
def _install(archive_filename, install_args=()):
"""Install Setuptools."""
with archive_context(archive_filename):
# installing
log.warn('Installing Setuptools')
if not _python_cmd('setup.py', 'install', *install_args):
log.warn('Something went wrong during the installation.')
log.warn('See the error message above.')
# exitcode will be 2
return 2
def _build_egg(egg, archive_filename, to_dir):
"""Build Setuptools egg."""
with archive_context(archive_filename):
# building an egg
log.warn('Building a Setuptools egg in %s', to_dir)
_python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir)
# returning the result
log.warn(egg)
if not os.path.exists(egg):
raise IOError('Could not build the egg.')
class ContextualZipFile(zipfile.ZipFile):
"""Supplement ZipFile class to support context manager for Python 2.6."""
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.close()
def __new__(cls, *args, **kwargs):
"""Construct a ZipFile or ContextualZipFile as appropriate."""
if hasattr(zipfile.ZipFile, '__exit__'):
return zipfile.ZipFile(*args, **kwargs)
return super(ContextualZipFile, cls).__new__(cls)
@contextlib.contextmanager
def archive_context(filename):
"""
Unzip filename to a temporary directory, set to the cwd.
The unzipped target is cleaned up after.
"""
tmpdir = tempfile.mkdtemp()
log.warn('Extracting in %s', tmpdir)
old_wd = os.getcwd()
try:
os.chdir(tmpdir)
try:
with ContextualZipFile(filename) as archive:
archive.extractall()
except zipfile.BadZipfile as err:
if not err.args:
err.args = ('', )
err.args = err.args + (
MEANINGFUL_INVALID_ZIP_ERR_MSG.format(filename),
)
raise
# going in the directory
subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
os.chdir(subdir)
log.warn('Now working in %s', subdir)
yield
finally:
os.chdir(old_wd)
shutil.rmtree(tmpdir)
def _do_download(version, download_base, to_dir, download_delay):
"""Download Setuptools."""
py_desig = 'py{sys.version_info[0]}.{sys.version_info[1]}'.format(sys=sys)
tp = 'setuptools-{version}-{py_desig}.egg'
egg = os.path.join(to_dir, tp.format(**locals()))
if not os.path.exists(egg):
archive = download_setuptools(version, download_base,
to_dir, download_delay)
_build_egg(egg, archive, to_dir)
sys.path.insert(0, egg)
# Remove previously-imported pkg_resources if present (see
# https://bitbucket.org/pypa/setuptools/pull-request/7/ for details).
if 'pkg_resources' in sys.modules:
_unload_pkg_resources()
import setuptools
setuptools.bootstrap_install_from = egg
def use_setuptools(
version=DEFAULT_VERSION, download_base=DEFAULT_URL,
to_dir=DEFAULT_SAVE_DIR, download_delay=15):
"""
Ensure that a setuptools version is installed.
Return None. Raise SystemExit if the requested version
or later cannot be installed.
"""
to_dir = os.path.abspath(to_dir)
# prior to importing, capture the module state for
# representative modules.
rep_modules = 'pkg_resources', 'setuptools'
imported = set(sys.modules).intersection(rep_modules)
try:
import pkg_resources
pkg_resources.require("setuptools>=" + version)
# a suitable version is already installed
return
except ImportError:
# pkg_resources not available; setuptools is not installed; download
pass
except pkg_resources.DistributionNotFound:
# no version of setuptools was found; allow download
pass
except pkg_resources.VersionConflict as VC_err:
if imported:
_conflict_bail(VC_err, version)
# otherwise, unload pkg_resources to allow the downloaded version to
# take precedence.
del pkg_resources
_unload_pkg_resources()
return _do_download(version, download_base, to_dir, download_delay)
def _conflict_bail(VC_err, version):
"""
Setuptools was imported prior to invocation, so it is
unsafe to unload it. Bail out.
"""
conflict_tmpl = textwrap.dedent("""
The required version of setuptools (>={version}) is not available,
and can't be installed while this script is running. Please
install a more recent version first, using
'easy_install -U setuptools'.
(Currently using {VC_err.args[0]!r})
""")
msg = conflict_tmpl.format(**locals())
sys.stderr.write(msg)
sys.exit(2)
def _unload_pkg_resources():
sys.meta_path = [
importer
for importer in sys.meta_path
if importer.__class__.__module__ != 'pkg_resources.extern'
]
del_modules = [
name for name in sys.modules
if name.startswith('pkg_resources')
]
for mod_name in del_modules:
del sys.modules[mod_name]
def _clean_check(cmd, target):
"""
Run the command to download target.
If the command fails, clean up before re-raising the error.
"""
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError:
if os.access(target, os.F_OK):
os.unlink(target)
raise
def download_file_powershell(url, target):
"""
Download the file at url to target using Powershell.
Powershell will validate trust.
Raise an exception if the command cannot complete.
"""
target = os.path.abspath(target)
ps_cmd = (
"[System.Net.WebRequest]::DefaultWebProxy.Credentials = "
"[System.Net.CredentialCache]::DefaultCredentials; "
'(new-object System.Net.WebClient).DownloadFile("%(url)s", "%(target)s")'
% locals()
)
cmd = [
'powershell',
'-Command',
ps_cmd,
]
_clean_check(cmd, target)
def has_powershell():
"""Determine if Powershell is available."""
if platform.system() != 'Windows':
return False
cmd = ['powershell', '-Command', 'echo test']
with open(os.path.devnull, 'wb') as devnull:
try:
subprocess.check_call(cmd, stdout=devnull, stderr=devnull)
except Exception:
return False
return True
download_file_powershell.viable = has_powershell
def download_file_curl(url, target):
cmd = ['curl', url, '--location', '--silent', '--output', target]
_clean_check(cmd, target)
def has_curl():
cmd = ['curl', '--version']
with open(os.path.devnull, 'wb') as devnull:
try:
subprocess.check_call(cmd, stdout=devnull, stderr=devnull)
except Exception:
return False
return True
download_file_curl.viable = has_curl
def download_file_wget(url, target):
cmd = ['wget', url, '--quiet', '--output-document', target]
_clean_check(cmd, target)
def has_wget():
cmd = ['wget', '--version']
with open(os.path.devnull, 'wb') as devnull:
try:
subprocess.check_call(cmd, stdout=devnull, stderr=devnull)
except Exception:
return False
return True
download_file_wget.viable = has_wget
def download_file_insecure(url, target):
"""Use Python to download the file, without connection authentication."""
src = urlopen(url)
try:
# Read all the data in one block.
data = src.read()
finally:
src.close()
# Write all the data in one block to avoid creating a partial file.
with open(target, "wb") as dst:
dst.write(data)
download_file_insecure.viable = lambda: True
def get_best_downloader():
downloaders = (
download_file_powershell,
download_file_curl,
download_file_wget,
download_file_insecure,
)
viable_downloaders = (dl for dl in downloaders if dl.viable())
return next(viable_downloaders, None)
def download_setuptools(
version=DEFAULT_VERSION, download_base=DEFAULT_URL,
to_dir=DEFAULT_SAVE_DIR, delay=15,
downloader_factory=get_best_downloader):
"""
Download setuptools from a specified location and return its filename.
`version` should be a valid setuptools version number that is available
as an sdist for download under the `download_base` URL (which should end
with a '/'). `to_dir` is the directory where the egg will be downloaded.
`delay` is the number of seconds to pause before an actual download
attempt.
``downloader_factory`` should be a function taking no arguments and
returning a function for downloading a URL to a target.
"""
# making sure we use the absolute path
to_dir = os.path.abspath(to_dir)
zip_name = "setuptools-%s.zip" % version
url = download_base + zip_name
saveto = os.path.join(to_dir, zip_name)
if not os.path.exists(saveto): # Avoid repeated downloads
log.warn("Downloading %s", url)
downloader = downloader_factory()
downloader(url, saveto)
return os.path.realpath(saveto)
def _build_install_args(options):
"""
Build the arguments to 'python setup.py install' on the setuptools package.
Returns list of command line arguments.
"""
return ['--user'] if options.user_install else []
def _parse_args():
"""Parse the command line for options."""
parser = optparse.OptionParser()
parser.add_option(
'--user', dest='user_install', action='store_true', default=False,
help='install in user site package')
parser.add_option(
'--download-base', dest='download_base', metavar="URL",
default=DEFAULT_URL,
help='alternative URL from where to download the setuptools package')
parser.add_option(
'--insecure', dest='downloader_factory', action='store_const',
const=lambda: download_file_insecure, default=get_best_downloader,
help='Use internal, non-validating downloader'
)
parser.add_option(
'--version', help="Specify which version to download",
default=DEFAULT_VERSION,
)
parser.add_option(
'--to-dir',
help="Directory to save (and re-use) package",
default=DEFAULT_SAVE_DIR,
)
options, args = parser.parse_args()
# positional arguments are ignored
return options
def _download_args(options):
"""Return args for download_setuptools function from cmdline args."""
return dict(
version=options.version,
download_base=options.download_base,
downloader_factory=options.downloader_factory,
to_dir=options.to_dir,
)
def main():
"""Install or upgrade setuptools and EasyInstall."""
options = _parse_args()
archive = download_setuptools(**_download_args(options))
return _install(archive, _build_install_args(options))
if __name__ == '__main__':
sys.exit(main())

7
thirdparty/xmltodict/setup.cfg vendored Normal file
View File

@ -0,0 +1,7 @@
[bdist_wheel]
universal = 1
[egg_info]
tag_build =
tag_date = 0

44
thirdparty/xmltodict/setup.py vendored Executable file
View File

@ -0,0 +1,44 @@
#!/usr/bin/env python
try:
from setuptools import setup
except ImportError:
from ez_setup import use_setuptools
use_setuptools()
from setuptools import setup
import xmltodict
with open('README.md') as f:
long_description = f.read()
setup(name='xmltodict',
version=xmltodict.__version__,
description=xmltodict.__doc__,
long_description=long_description,
long_description_content_type='text/markdown',
author=xmltodict.__author__,
author_email='martinblech@gmail.com',
url='https://github.com/martinblech/xmltodict',
license=xmltodict.__license__,
platforms=['all'],
python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
classifiers=[
'Intended Audience :: Developers',
'License :: OSI Approved :: MIT License',
'Operating System :: OS Independent',
'Programming Language :: Python',
'Programming Language :: Python :: 2',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: Implementation :: Jython',
'Programming Language :: Python :: Implementation :: PyPy',
'Topic :: Text Processing :: Markup :: XML',
],
py_modules=['xmltodict'],
tests_require=['nose>=1.0', 'coverage'],
)

207
thirdparty/xmltodict/tests/test_dicttoxml.py vendored Normal file
View File

@ -0,0 +1,207 @@
import sys
from xmltodict import parse, unparse
from collections import OrderedDict
import unittest
import re
from textwrap import dedent
IS_JYTHON = sys.platform.startswith('java')
_HEADER_RE = re.compile(r'^[^\n]*\n')
def _strip(fullxml):
return _HEADER_RE.sub('', fullxml)
class DictToXMLTestCase(unittest.TestCase):
def test_root(self):
obj = {'a': None}
self.assertEqual(obj, parse(unparse(obj)))
self.assertEqual(unparse(obj), unparse(parse(unparse(obj))))
def test_simple_cdata(self):
obj = {'a': 'b'}
self.assertEqual(obj, parse(unparse(obj)))
self.assertEqual(unparse(obj), unparse(parse(unparse(obj))))
def test_cdata(self):
obj = {'a': {'#text': 'y'}}
self.assertEqual(obj, parse(unparse(obj), force_cdata=True))
self.assertEqual(unparse(obj), unparse(parse(unparse(obj))))
def test_attrib(self):
obj = {'a': {'@href': 'x'}}
self.assertEqual(obj, parse(unparse(obj)))
self.assertEqual(unparse(obj), unparse(parse(unparse(obj))))
def test_attrib_and_cdata(self):
obj = {'a': {'@href': 'x', '#text': 'y'}}
self.assertEqual(obj, parse(unparse(obj)))
self.assertEqual(unparse(obj), unparse(parse(unparse(obj))))
def test_list(self):
obj = {'a': {'b': ['1', '2', '3']}}
self.assertEqual(obj, parse(unparse(obj)))
self.assertEqual(unparse(obj), unparse(parse(unparse(obj))))
def test_generator(self):
obj = {'a': {'b': ['1', '2', '3']}}
def lazy_obj():
return {'a': {'b': (i for i in ('1', '2', '3'))}}
self.assertEqual(obj, parse(unparse(lazy_obj())))
self.assertEqual(unparse(lazy_obj()),
unparse(parse(unparse(lazy_obj()))))
def test_no_root(self):
self.assertRaises(ValueError, unparse, {})
def test_multiple_roots(self):
self.assertRaises(ValueError, unparse, {'a': '1', 'b': '2'})
self.assertRaises(ValueError, unparse, {'a': ['1', '2', '3']})
def test_no_root_nofulldoc(self):
self.assertEqual(unparse({}, full_document=False), '')
def test_multiple_roots_nofulldoc(self):
obj = OrderedDict((('a', 1), ('b', 2)))
xml = unparse(obj, full_document=False)
self.assertEqual(xml, '<a>1</a><b>2</b>')
obj = {'a': [1, 2]}
xml = unparse(obj, full_document=False)
self.assertEqual(xml, '<a>1</a><a>2</a>')
def test_nested(self):
obj = {'a': {'b': '1', 'c': '2'}}
self.assertEqual(obj, parse(unparse(obj)))
self.assertEqual(unparse(obj), unparse(parse(unparse(obj))))
obj = {'a': {'b': {'c': {'@a': 'x', '#text': 'y'}}}}
self.assertEqual(obj, parse(unparse(obj)))
self.assertEqual(unparse(obj), unparse(parse(unparse(obj))))
def test_semistructured(self):
xml = '<a>abc<d/>efg</a>'
self.assertEqual(_strip(unparse(parse(xml))),
'<a><d></d>abcefg</a>')
def test_preprocessor(self):
obj = {'a': OrderedDict((('b:int', [1, 2]), ('b', 'c')))}
def p(key, value):
try:
key, _ = key.split(':')
except ValueError:
pass
return key, value
self.assertEqual(_strip(unparse(obj, preprocessor=p)),
'<a><b>1</b><b>2</b><b>c</b></a>')
def test_preprocessor_skipkey(self):
obj = {'a': {'b': 1, 'c': 2}}
def p(key, value):
if key == 'b':
return None
return key, value
self.assertEqual(_strip(unparse(obj, preprocessor=p)),
'<a><c>2</c></a>')
if not IS_JYTHON:
# Jython's SAX does not preserve attribute order
def test_attr_order_roundtrip(self):
xml = '<root a="1" b="2" c="3"></root>'
self.assertEqual(xml, _strip(unparse(parse(xml))))
def test_pretty_print(self):
obj = {'a': OrderedDict((
('b', [{'c': [1, 2]}, 3]),
('x', 'y'),
))}
newl = '\n'
indent = '....'
xml = dedent('''\
<?xml version="1.0" encoding="utf-8"?>
<a>
....<b>
........<c>1</c>
........<c>2</c>
....</b>
....<b>3</b>
....<x>y</x>
</a>''')
self.assertEqual(xml, unparse(obj, pretty=True,
newl=newl, indent=indent))
def test_encoding(self):
try:
value = unichr(39321)
except NameError:
value = chr(39321)
obj = {'a': value}
utf8doc = unparse(obj, encoding='utf-8')
latin1doc = unparse(obj, encoding='iso-8859-1')
self.assertEqual(parse(utf8doc), parse(latin1doc))
self.assertEqual(parse(utf8doc), obj)
def test_fulldoc(self):
xml_declaration_re = re.compile(
'^' + re.escape('<?xml version="1.0" encoding="utf-8"?>'))
self.assertTrue(xml_declaration_re.match(unparse({'a': 1})))
self.assertFalse(
xml_declaration_re.match(unparse({'a': 1}, full_document=False)))
def test_non_string_value(self):
obj = {'a': 1}
self.assertEqual('<a>1</a>', _strip(unparse(obj)))
def test_non_string_attr(self):
obj = {'a': {'@attr': 1}}
self.assertEqual('<a attr="1"></a>', _strip(unparse(obj)))
def test_short_empty_elements(self):
if sys.version_info[0] < 3:
return
obj = {'a': None}
self.assertEqual('<a/>', _strip(unparse(obj, short_empty_elements=True)))
def test_namespace_support(self):
obj = OrderedDict((
('http://defaultns.com/:root', OrderedDict((
('@xmlns', OrderedDict((
('', 'http://defaultns.com/'),
('a', 'http://a.com/'),
('b', 'http://b.com/'),
))),
('http://defaultns.com/:x', OrderedDict((
('@http://a.com/:attr', 'val'),
('#text', '1'),
))),
('http://a.com/:y', '2'),
('http://b.com/:z', '3'),
))),
))
ns = {
'http://defaultns.com/': '',
'http://a.com/': 'a',
'http://b.com/': 'b',
}
expected_xml = '''<?xml version="1.0" encoding="utf-8"?>
<root xmlns="http://defaultns.com/" xmlns:a="http://a.com/" \
xmlns:b="http://b.com/"><x a:attr="val">1</x><a:y>2</a:y><b:z>3</b:z></root>'''
xml = unparse(obj, namespaces=ns)
self.assertEqual(xml, expected_xml)
def test_boolean_unparse(self):
expected_xml = '<?xml version="1.0" encoding="utf-8"?>\n<x>true</x>'
xml = unparse(dict(x=True))
self.assertEqual(xml, expected_xml)
expected_xml = '<?xml version="1.0" encoding="utf-8"?>\n<x>false</x>'
xml = unparse(dict(x=False))
self.assertEqual(xml, expected_xml)

382
thirdparty/xmltodict/tests/test_xmltodict.py vendored Normal file
View File

@ -0,0 +1,382 @@
from xmltodict import parse, ParsingInterrupted
import unittest
try:
from io import BytesIO as StringIO
except ImportError:
from xmltodict import StringIO
from xml.parsers.expat import ParserCreate
from xml.parsers import expat
def _encode(s):
try:
return bytes(s, 'ascii')
except (NameError, TypeError):
return s
class XMLToDictTestCase(unittest.TestCase):
def test_string_vs_file(self):
xml = '<a>data</a>'
self.assertEqual(parse(xml),
parse(StringIO(_encode(xml))))
def test_minimal(self):
self.assertEqual(parse('<a/>'),
{'a': None})
self.assertEqual(parse('<a/>', force_cdata=True),
{'a': None})
def test_simple(self):
self.assertEqual(parse('<a>data</a>'),
{'a': 'data'})
def test_force_cdata(self):
self.assertEqual(parse('<a>data</a>', force_cdata=True),
{'a': {'#text': 'data'}})
def test_custom_cdata(self):
self.assertEqual(parse('<a>data</a>',
force_cdata=True,
cdata_key='_CDATA_'),
{'a': {'_CDATA_': 'data'}})
def test_list(self):
self.assertEqual(parse('<a><b>1</b><b>2</b><b>3</b></a>'),
{'a': {'b': ['1', '2', '3']}})
def test_attrib(self):
self.assertEqual(parse('<a href="xyz"/>'),
{'a': {'@href': 'xyz'}})
def test_skip_attrib(self):
self.assertEqual(parse('<a href="xyz"/>', xml_attribs=False),
{'a': None})
def test_custom_attrib(self):
self.assertEqual(parse('<a href="xyz"/>',
attr_prefix='!'),
{'a': {'!href': 'xyz'}})
def test_attrib_and_cdata(self):
self.assertEqual(parse('<a href="xyz">123</a>'),
{'a': {'@href': 'xyz', '#text': '123'}})
def test_semi_structured(self):
self.assertEqual(parse('<a>abc<b/>def</a>'),
{'a': {'b': None, '#text': 'abcdef'}})
self.assertEqual(parse('<a>abc<b/>def</a>',
cdata_separator='\n'),
{'a': {'b': None, '#text': 'abc\ndef'}})
def test_nested_semi_structured(self):
self.assertEqual(parse('<a>abc<b>123<c/>456</b>def</a>'),
{'a': {'#text': 'abcdef', 'b': {
'#text': '123456', 'c': None}}})
def test_skip_whitespace(self):
xml = """
<root>
<emptya> </emptya>
<emptyb attr="attrvalue">
</emptyb>
<value>hello</value>
</root>
"""
self.assertEqual(
parse(xml),
{'root': {'emptya': None,
'emptyb': {'@attr': 'attrvalue'},
'value': 'hello'}})
def test_keep_whitespace(self):
xml = "<root> </root>"
self.assertEqual(parse(xml), dict(root=None))
self.assertEqual(parse(xml, strip_whitespace=False),
dict(root=' '))
def test_streaming(self):
def cb(path, item):
cb.count += 1
self.assertEqual(path, [('a', {'x': 'y'}), ('b', None)])
self.assertEqual(item, str(cb.count))
return True
cb.count = 0
parse('<a x="y"><b>1</b><b>2</b><b>3</b></a>',
item_depth=2, item_callback=cb)
self.assertEqual(cb.count, 3)
def test_streaming_interrupt(self):
cb = lambda path, item: False
self.assertRaises(ParsingInterrupted,
parse, '<a>x</a>',
item_depth=1, item_callback=cb)
def test_postprocessor(self):
def postprocessor(path, key, value):
try:
return key + ':int', int(value)
except (ValueError, TypeError):
return key, value
self.assertEqual({'a': {'b:int': [1, 2], 'b': 'x'}},
parse('<a><b>1</b><b>2</b><b>x</b></a>',
postprocessor=postprocessor))
def test_postprocessor_attribute(self):
def postprocessor(path, key, value):
try:
return key + ':int', int(value)
except (ValueError, TypeError):
return key, value
self.assertEqual({'a': {'@b:int': 1}},
parse('<a b="1"/>',
postprocessor=postprocessor))
def test_postprocessor_skip(self):
def postprocessor(path, key, value):
if key == 'b':
value = int(value)
if value == 3:
return None
return key, value
self.assertEqual({'a': {'b': [1, 2]}},
parse('<a><b>1</b><b>2</b><b>3</b></a>',
postprocessor=postprocessor))
def test_unicode(self):
try:
value = unichr(39321)
except NameError:
value = chr(39321)
self.assertEqual({'a': value},
parse('<a>%s</a>' % value))
def test_encoded_string(self):
try:
value = unichr(39321)
except NameError:
value = chr(39321)
xml = '<a>%s</a>' % value
self.assertEqual(parse(xml),
parse(xml.encode('utf-8')))
def test_namespace_support(self):
xml = """
<root xmlns="http://defaultns.com/"
xmlns:a="http://a.com/"
xmlns:b="http://b.com/">
<x a:attr="val">1</x>
<a:y>2</a:y>
<b:z>3</b:z>
</root>
"""
d = {
'http://defaultns.com/:root': {
'http://defaultns.com/:x': {
'@xmlns': {
'': 'http://defaultns.com/',
'a': 'http://a.com/',
'b': 'http://b.com/',
},
'@http://a.com/:attr': 'val',
'#text': '1',
},
'http://a.com/:y': '2',
'http://b.com/:z': '3',
}
}
res = parse(xml, process_namespaces=True)
self.assertEqual(res, d)
def test_namespace_collapse(self):
xml = """
<root xmlns="http://defaultns.com/"
xmlns:a="http://a.com/"
xmlns:b="http://b.com/">
<x a:attr="val">1</x>
<a:y>2</a:y>
<b:z>3</b:z>
</root>
"""
namespaces = {
'http://defaultns.com/': '',
'http://a.com/': 'ns_a',
}
d = {
'root': {
'x': {
'@xmlns': {
'': 'http://defaultns.com/',
'a': 'http://a.com/',
'b': 'http://b.com/',
},
'@ns_a:attr': 'val',
'#text': '1',
},
'ns_a:y': '2',
'http://b.com/:z': '3',
},
}
res = parse(xml, process_namespaces=True, namespaces=namespaces)
self.assertEqual(res, d)
def test_namespace_ignore(self):
xml = """
<root xmlns="http://defaultns.com/"
xmlns:a="http://a.com/"
xmlns:b="http://b.com/">
<x>1</x>
<a:y>2</a:y>
<b:z>3</b:z>
</root>
"""
d = {
'root': {
'@xmlns': 'http://defaultns.com/',
'@xmlns:a': 'http://a.com/',
'@xmlns:b': 'http://b.com/',
'x': '1',
'a:y': '2',
'b:z': '3',
},
}
self.assertEqual(parse(xml), d)
def test_force_list_basic(self):
xml = """
<servers>
<server>
<name>server1</name>
<os>os1</os>
</server>
</servers>
"""
expectedResult = {
'servers': {
'server': [
{
'name': 'server1',
'os': 'os1',
},
],
}
}
self.assertEqual(parse(xml, force_list=('server',)), expectedResult)
def test_force_list_callable(self):
xml = """
<config>
<servers>
<server>
<name>server1</name>
<os>os1</os>
</server>
</servers>
<skip>
<server></server>
</skip>
</config>
"""
def force_list(path, key, value):
"""Only return True for servers/server, but not for skip/server."""
if key != 'server':
return False
return path and path[-1][0] == 'servers'
expectedResult = {
'config': {
'servers': {
'server': [
{
'name': 'server1',
'os': 'os1',
},
],
},
'skip': {
'server': None,
},
},
}
self.assertEqual(parse(xml, force_list=force_list, dict_constructor=dict), expectedResult)
def test_disable_entities_true_ignores_xmlbomb(self):
xml = """
<!DOCTYPE xmlbomb [
<!ENTITY a "1234567890" >
<!ENTITY b "&a;&a;&a;&a;&a;&a;&a;&a;">
<!ENTITY c "&b;&b;&b;&b;&b;&b;&b;&b;">
]>
<bomb>&c;</bomb>
"""
expectedResult = {'bomb': None}
try:
parse_attempt = parse(xml, disable_entities=True)
except expat.ExpatError:
self.assertTrue(True)
else:
self.assertEqual(parse_attempt, expectedResult)
def test_disable_entities_false_returns_xmlbomb(self):
xml = """
<!DOCTYPE xmlbomb [
<!ENTITY a "1234567890" >
<!ENTITY b "&a;&a;&a;&a;&a;&a;&a;&a;">
<!ENTITY c "&b;&b;&b;&b;&b;&b;&b;&b;">
]>
<bomb>&c;</bomb>
"""
bomb = "1234567890" * 64
expectedResult = {'bomb': bomb}
self.assertEqual(parse(xml, disable_entities=False), expectedResult)
def test_disable_entities_true_ignores_external_dtd(self):
xml = """
<!DOCTYPE external [
<!ENTITY ee SYSTEM "http://www.python.org/">
]>
<root>&ee;</root>
"""
expectedResult = {'root': None}
try:
parse_attempt = parse(xml, disable_entities=True)
except expat.ExpatError:
self.assertTrue(True)
else:
self.assertEqual(parse_attempt, expectedResult)
def test_disable_entities_true_attempts_external_dtd(self):
xml = """
<!DOCTYPE external [
<!ENTITY ee SYSTEM "http://www.python.org/">
]>
<root>&ee;</root>
"""
def raising_external_ref_handler(*args, **kwargs):
parser = ParserCreate(*args, **kwargs)
parser.ExternalEntityRefHandler = lambda *x: 0
try:
feature = "http://apache.org/xml/features/disallow-doctype-decl"
parser._reader.setFeature(feature, True)
except AttributeError:
pass
return parser
expat.ParserCreate = raising_external_ref_handler
# Using this try/catch because a TypeError is thrown before
# the ExpatError, and Python 2.6 is confused by that.
try:
parse(xml, disable_entities=False, expat=expat)
except expat.ExpatError:
self.assertTrue(True)
else:
self.assertTrue(False)
expat.ParserCreate = ParserCreate

232
thirdparty/xmltodict/xmltodict.egg-info/PKG-INFO vendored Normal file
View File

@ -0,0 +1,232 @@
Metadata-Version: 2.1
Name: xmltodict
Version: 0.12.0
Summary: Makes working with XML feel like you are working with JSON
Home-page: https://github.com/martinblech/xmltodict
Author: Martin Blech
Author-email: martinblech@gmail.com
License: MIT
Description: # xmltodict
`xmltodict` is a Python module that makes working with XML feel like you are working with [JSON](http://docs.python.org/library/json.html), as in this ["spec"](http://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html):
[![Build Status](https://secure.travis-ci.org/martinblech/xmltodict.svg)](http://travis-ci.org/martinblech/xmltodict)
```python
>>> print(json.dumps(xmltodict.parse("""
... <mydocument has="an attribute">
... <and>
... <many>elements</many>
... <many>more elements</many>
... </and>
... <plus a="complex">
... element as well
... </plus>
... </mydocument>
... """), indent=4))
{
"mydocument": {
"@has": "an attribute",
"and": {
"many": [
"elements",
"more elements"
]
},
"plus": {
"@a": "complex",
"#text": "element as well"
}
}
}
```
## Namespace support
By default, `xmltodict` does no XML namespace processing (it just treats namespace declarations as regular node attributes), but passing `process_namespaces=True` will make it expand namespaces for you:
```python
>>> xml = """
... <root xmlns="http://defaultns.com/"
... xmlns:a="http://a.com/"
... xmlns:b="http://b.com/">
... <x>1</x>
... <a:y>2</a:y>
... <b:z>3</b:z>
... </root>
... """
>>> xmltodict.parse(xml, process_namespaces=True) == {
... 'http://defaultns.com/:root': {
... 'http://defaultns.com/:x': '1',
... 'http://a.com/:y': '2',
... 'http://b.com/:z': '3',
... }
... }
True
```
It also lets you collapse certain namespaces to shorthand prefixes, or skip them altogether:
```python
>>> namespaces = {
... 'http://defaultns.com/': None, # skip this namespace
... 'http://a.com/': 'ns_a', # collapse "http://a.com/" -> "ns_a"
... }
>>> xmltodict.parse(xml, process_namespaces=True, namespaces=namespaces) == {
... 'root': {
... 'x': '1',
... 'ns_a:y': '2',
... 'http://b.com/:z': '3',
... },
... }
True
```
## Streaming mode
`xmltodict` is very fast ([Expat](http://docs.python.org/library/pyexpat.html)-based) and has a streaming mode with a small memory footprint, suitable for big XML dumps like [Discogs](http://discogs.com/data/) or [Wikipedia](http://dumps.wikimedia.org/):
```python
>>> def handle_artist(_, artist):
... print(artist['name'])
... return True
>>>
>>> xmltodict.parse(GzipFile('discogs_artists.xml.gz'),
... item_depth=2, item_callback=handle_artist)
A Perfect Circle
Fantômas
King Crimson
Chris Potter
...
```
It can also be used from the command line to pipe objects to a script like this:
```python
import sys, marshal
while True:
_, article = marshal.load(sys.stdin)
print(article['title'])
```
```sh
$ bunzip2 enwiki-pages-articles.xml.bz2 | xmltodict.py 2 | myscript.py
AccessibleComputing
Anarchism
AfghanistanHistory
AfghanistanGeography
AfghanistanPeople
AfghanistanCommunications
Autism
...
```
Or just cache the dicts so you don't have to parse that big XML file again. You do this only once:
```sh
$ bunzip2 enwiki-pages-articles.xml.bz2 | xmltodict.py 2 | gzip > enwiki.dicts.gz
```
And you reuse the dicts with every script that needs them:
```sh
$ gunzip enwiki.dicts.gz | script1.py
$ gunzip enwiki.dicts.gz | script2.py
...
```
## Roundtripping
You can also convert in the other direction, using the `unparse()` method:
```python
>>> mydict = {
... 'response': {
... 'status': 'good',
... 'last_updated': '2014-02-16T23:10:12Z',
... }
... }
>>> print(unparse(mydict, pretty=True))
<?xml version="1.0" encoding="utf-8"?>
<response>
<status>good</status>
<last_updated>2014-02-16T23:10:12Z</last_updated>
</response>
```
Text values for nodes can be specified with the `cdata_key` key in the python dict, while node properties can be specified with the `attr_prefix` prefixed to the key name in the python dict. The default value for `attr_prefix` is `@` and the default value for `cdata_key` is `#text`.
```python
>>> import xmltodict
>>>
>>> mydict = {
... 'text': {
... '@color':'red',
... '@stroke':'2',
... '#text':'This is a test'
... }
... }
>>> print(xmltodict.unparse(mydict, pretty=True))
<?xml version="1.0" encoding="utf-8"?>
<text stroke="2" color="red">This is a test</text>
```
## Ok, how do I get it?
### Using pypi
You just need to
```sh
$ pip install xmltodict
```
### RPM-based distro (Fedora, RHEL, …)
There is an [official Fedora package for xmltodict](https://apps.fedoraproject.org/packages/python-xmltodict).
```sh
$ sudo yum install python-xmltodict
```
### Arch Linux
There is an [official Arch Linux package for xmltodict](https://www.archlinux.org/packages/community/any/python-xmltodict/).
```sh
$ sudo pacman -S python-xmltodict
```
### Debian-based distro (Debian, Ubuntu, …)
There is an [official Debian package for xmltodict](https://tracker.debian.org/pkg/python-xmltodict).
```sh
$ sudo apt install python-xmltodict
```
### FreeBSD
There is an [official FreeBSD port for xmltodict](https://svnweb.freebsd.org/ports/head/devel/py-xmltodict/).
```sh
$ pkg install py36-xmltodict
```
Platform: all
Classifier: Intended Audience :: Developers
Classifier: License :: OSI Approved :: MIT License
Classifier: Operating System :: OS Independent
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 2
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.4
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: Implementation :: Jython
Classifier: Programming Language :: Python :: Implementation :: PyPy
Classifier: Topic :: Text Processing :: Markup :: XML
Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
Description-Content-Type: text/markdown

13
thirdparty/xmltodict/xmltodict.egg-info/SOURCES.txt vendored Normal file
View File

@ -0,0 +1,13 @@
LICENSE
MANIFEST.in
README.md
ez_setup.py
setup.cfg
setup.py
xmltodict.py
tests/test_dicttoxml.py
tests/test_xmltodict.py
xmltodict.egg-info/PKG-INFO
xmltodict.egg-info/SOURCES.txt
xmltodict.egg-info/dependency_links.txt
xmltodict.egg-info/top_level.txt

1
thirdparty/xmltodict/xmltodict.egg-info/dependency_links.txt vendored Normal file
View File

@ -0,0 +1 @@

1
thirdparty/xmltodict/xmltodict.egg-info/top_level.txt vendored Normal file
View File

@ -0,0 +1 @@
xmltodict

488
thirdparty/xmltodict/xmltodict.py vendored Executable file
View File

@ -0,0 +1,488 @@
#!/usr/bin/env python
"Makes working with XML feel like you are working with JSON"
try:
from defusedexpat import pyexpat as expat
except ImportError:
from xml.parsers import expat
from xml.sax.saxutils import XMLGenerator
from xml.sax.xmlreader import AttributesImpl
try: # pragma no cover
from cStringIO import StringIO
except ImportError: # pragma no cover
try:
from StringIO import StringIO
except ImportError:
from io import StringIO
from collections import OrderedDict
try: # pragma no cover
_basestring = basestring
except NameError: # pragma no cover
_basestring = str
try: # pragma no cover
_unicode = unicode
except NameError: # pragma no cover
_unicode = str
__author__ = 'Martin Blech'
__version__ = '0.12.0'
__license__ = 'MIT'
class ParsingInterrupted(Exception):
pass
class _DictSAXHandler(object):
def __init__(self,
item_depth=0,
item_callback=lambda *args: True,
xml_attribs=True,
attr_prefix='@',
cdata_key='#text',
force_cdata=False,
cdata_separator='',
postprocessor=None,
dict_constructor=OrderedDict,
strip_whitespace=True,
namespace_separator=':',
namespaces=None,
force_list=None):
self.path = []
self.stack = []
self.data = []
self.item = None
self.item_depth = item_depth
self.xml_attribs = xml_attribs
self.item_callback = item_callback
self.attr_prefix = attr_prefix
self.cdata_key = cdata_key
self.force_cdata = force_cdata
self.cdata_separator = cdata_separator
self.postprocessor = postprocessor
self.dict_constructor = dict_constructor
self.strip_whitespace = strip_whitespace
self.namespace_separator = namespace_separator
self.namespaces = namespaces
self.namespace_declarations = OrderedDict()
self.force_list = force_list
def _build_name(self, full_name):
if not self.namespaces:
return full_name
i = full_name.rfind(self.namespace_separator)
if i == -1:
return full_name
namespace, name = full_name[:i], full_name[i+1:]
short_namespace = self.namespaces.get(namespace, namespace)
if not short_namespace:
return name
else:
return self.namespace_separator.join((short_namespace, name))
def _attrs_to_dict(self, attrs):
if isinstance(attrs, dict):
return attrs
return self.dict_constructor(zip(attrs[0::2], attrs[1::2]))
def startNamespaceDecl(self, prefix, uri):
self.namespace_declarations[prefix or ''] = uri
def startElement(self, full_name, attrs):
name = self._build_name(full_name)
attrs = self._attrs_to_dict(attrs)
if attrs and self.namespace_declarations:
attrs['xmlns'] = self.namespace_declarations
self.namespace_declarations = OrderedDict()
self.path.append((name, attrs or None))
if len(self.path) > self.item_depth:
self.stack.append((self.item, self.data))
if self.xml_attribs:
attr_entries = []
for key, value in attrs.items():
key = self.attr_prefix+self._build_name(key)
if self.postprocessor:
entry = self.postprocessor(self.path, key, value)
else:
entry = (key, value)
if entry:
attr_entries.append(entry)
attrs = self.dict_constructor(attr_entries)
else:
attrs = None
self.item = attrs or None
self.data = []
def endElement(self, full_name):
name = self._build_name(full_name)
if len(self.path) == self.item_depth:
item = self.item
if item is None:
item = (None if not self.data
else self.cdata_separator.join(self.data))
should_continue = self.item_callback(self.path, item)
if not should_continue:
raise ParsingInterrupted()
if len(self.stack):
data = (None if not self.data
else self.cdata_separator.join(self.data))
item = self.item
self.item, self.data = self.stack.pop()
if self.strip_whitespace and data:
data = data.strip() or None
if data and self.force_cdata and item is None:
item = self.dict_constructor()
if item is not None:
if data:
self.push_data(item, self.cdata_key, data)
self.item = self.push_data(self.item, name, item)
else:
self.item = self.push_data(self.item, name, data)
else:
self.item = None
self.data = []
self.path.pop()
def characters(self, data):
if not self.data:
self.data = [data]
else:
self.data.append(data)
def push_data(self, item, key, data):
if self.postprocessor is not None:
result = self.postprocessor(self.path, key, data)
if result is None:
return item
key, data = result
if item is None:
item = self.dict_constructor()
try:
value = item[key]
if isinstance(value, list):
value.append(data)
else:
item[key] = [value, data]
except KeyError:
if self._should_force_list(key, data):
item[key] = [data]
else:
item[key] = data
return item
def _should_force_list(self, key, value):
if not self.force_list:
return False
if isinstance(self.force_list, bool):
return self.force_list
try:
return key in self.force_list
except TypeError:
return self.force_list(self.path[:-1], key, value)
def parse(xml_input, encoding=None, expat=expat, process_namespaces=False,
namespace_separator=':', disable_entities=True, **kwargs):
"""Parse the given XML input and convert it into a dictionary.
`xml_input` can either be a `string` or a file-like object.
If `xml_attribs` is `True`, element attributes are put in the dictionary
among regular child elements, using `@` as a prefix to avoid collisions. If
set to `False`, they are just ignored.
Simple example::
>>> import xmltodict
>>> doc = xmltodict.parse(\"\"\"
... <a prop="x">
... <b>1</b>
... <b>2</b>
... </a>
... \"\"\")
>>> doc['a']['@prop']
u'x'
>>> doc['a']['b']
[u'1', u'2']
If `item_depth` is `0`, the function returns a dictionary for the root
element (default behavior). Otherwise, it calls `item_callback` every time
an item at the specified depth is found and returns `None` in the end
(streaming mode).
The callback function receives two parameters: the `path` from the document
root to the item (name-attribs pairs), and the `item` (dict). If the
callback's return value is false-ish, parsing will be stopped with the
:class:`ParsingInterrupted` exception.
Streaming example::
>>> def handle(path, item):
... print('path:%s item:%s' % (path, item))
... return True
...
>>> xmltodict.parse(\"\"\"
... <a prop="x">
... <b>1</b>
... <b>2</b>
... </a>\"\"\", item_depth=2, item_callback=handle)
path:[(u'a', {u'prop': u'x'}), (u'b', None)] item:1
path:[(u'a', {u'prop': u'x'}), (u'b', None)] item:2
The optional argument `postprocessor` is a function that takes `path`,
`key` and `value` as positional arguments and returns a new `(key, value)`
pair where both `key` and `value` may have changed. Usage example::
>>> def postprocessor(path, key, value):
... try:
... return key + ':int', int(value)
... except (ValueError, TypeError):
... return key, value
>>> xmltodict.parse('<a><b>1</b><b>2</b><b>x</b></a>',
... postprocessor=postprocessor)
OrderedDict([(u'a', OrderedDict([(u'b:int', [1, 2]), (u'b', u'x')]))])
You can pass an alternate version of `expat` (such as `defusedexpat`) by
using the `expat` parameter. E.g:
>>> import defusedexpat
>>> xmltodict.parse('<a>hello</a>', expat=defusedexpat.pyexpat)
OrderedDict([(u'a', u'hello')])
You can use the force_list argument to force lists to be created even
when there is only a single child of a given level of hierarchy. The
force_list argument is a tuple of keys. If the key for a given level
of hierarchy is in the force_list argument, that level of hierarchy
will have a list as a child (even if there is only one sub-element).
The index_keys operation takes precendence over this. This is applied
after any user-supplied postprocessor has already run.
For example, given this input:
<servers>
<server>
<name>host1</name>
<os>Linux</os>
<interfaces>
<interface>
<name>em0</name>
<ip_address>10.0.0.1</ip_address>
</interface>
</interfaces>
</server>
</servers>
If called with force_list=('interface',), it will produce
this dictionary:
{'servers':
{'server':
{'name': 'host1',
'os': 'Linux'},
'interfaces':
{'interface':
[ {'name': 'em0', 'ip_address': '10.0.0.1' } ] } } }
`force_list` can also be a callable that receives `path`, `key` and
`value`. This is helpful in cases where the logic that decides whether
a list should be forced is more complex.
"""
handler = _DictSAXHandler(namespace_separator=namespace_separator,
**kwargs)
if isinstance(xml_input, _unicode):
if not encoding:
encoding = 'utf-8'
xml_input = xml_input.encode(encoding)
if not process_namespaces:
namespace_separator = None
parser = expat.ParserCreate(
encoding,
namespace_separator
)
try:
parser.ordered_attributes = True
except AttributeError:
# Jython's expat does not support ordered_attributes
pass
parser.StartNamespaceDeclHandler = handler.startNamespaceDecl
parser.StartElementHandler = handler.startElement
parser.EndElementHandler = handler.endElement
parser.CharacterDataHandler = handler.characters
parser.buffer_text = True
if disable_entities:
try:
# Attempt to disable DTD in Jython's expat parser (Xerces-J).
feature = "http://apache.org/xml/features/disallow-doctype-decl"
parser._reader.setFeature(feature, True)
except AttributeError:
# For CPython / expat parser.
# Anything not handled ends up here and entities aren't expanded.
parser.DefaultHandler = lambda x: None
# Expects an integer return; zero means failure -> expat.ExpatError.
parser.ExternalEntityRefHandler = lambda *x: 1
if hasattr(xml_input, 'read'):
parser.ParseFile(xml_input)
else:
parser.Parse(xml_input, True)
return handler.item
def _process_namespace(name, namespaces, ns_sep=':', attr_prefix='@'):
if not namespaces:
return name
try:
ns, name = name.rsplit(ns_sep, 1)
except ValueError:
pass
else:
ns_res = namespaces.get(ns.strip(attr_prefix))
name = '{}{}{}{}'.format(
attr_prefix if ns.startswith(attr_prefix) else '',
ns_res, ns_sep, name) if ns_res else name
return name
def _emit(key, value, content_handler,
attr_prefix='@',
cdata_key='#text',
depth=0,
preprocessor=None,
pretty=False,
newl='\n',
indent='\t',
namespace_separator=':',
namespaces=None,
full_document=True):
key = _process_namespace(key, namespaces, namespace_separator, attr_prefix)
if preprocessor is not None:
result = preprocessor(key, value)
if result is None:
return
key, value = result
if (not hasattr(value, '__iter__')
or isinstance(value, _basestring)
or isinstance(value, dict)):
value = [value]
for index, v in enumerate(value):
if full_document and depth == 0 and index > 0:
raise ValueError('document with multiple roots')
if v is None:
v = OrderedDict()
elif isinstance(v, bool):
if v:
v = _unicode('true')
else:
v = _unicode('false')
elif not isinstance(v, dict):
v = _unicode(v)
if isinstance(v, _basestring):
v = OrderedDict(((cdata_key, v),))
cdata = None
attrs = OrderedDict()
children = []
for ik, iv in v.items():
if ik == cdata_key:
cdata = iv
continue
if ik.startswith(attr_prefix):
ik = _process_namespace(ik, namespaces, namespace_separator,
attr_prefix)
if ik == '@xmlns' and isinstance(iv, dict):
for k, v in iv.items():
attr = 'xmlns{}'.format(':{}'.format(k) if k else '')
attrs[attr] = _unicode(v)
continue
if not isinstance(iv, _unicode):
iv = _unicode(iv)
attrs[ik[len(attr_prefix):]] = iv
continue
children.append((ik, iv))
if pretty:
content_handler.ignorableWhitespace(depth * indent)
content_handler.startElement(key, AttributesImpl(attrs))
if pretty and children:
content_handler.ignorableWhitespace(newl)
for child_key, child_value in children:
_emit(child_key, child_value, content_handler,
attr_prefix, cdata_key, depth+1, preprocessor,
pretty, newl, indent, namespaces=namespaces,
namespace_separator=namespace_separator)
if cdata is not None:
content_handler.characters(cdata)
if pretty and children:
content_handler.ignorableWhitespace(depth * indent)
content_handler.endElement(key)
if pretty and depth:
content_handler.ignorableWhitespace(newl)
def unparse(input_dict, output=None, encoding='utf-8', full_document=True,
short_empty_elements=False,
**kwargs):
"""Emit an XML document for the given `input_dict` (reverse of `parse`).
The resulting XML document is returned as a string, but if `output` (a
file-like object) is specified, it is written there instead.
Dictionary keys prefixed with `attr_prefix` (default=`'@'`) are interpreted
as XML node attributes, whereas keys equal to `cdata_key`
(default=`'#text'`) are treated as character data.
The `pretty` parameter (default=`False`) enables pretty-printing. In this
mode, lines are terminated with `'\n'` and indented with `'\t'`, but this
can be customized with the `newl` and `indent` parameters.
"""
if full_document and len(input_dict) != 1:
raise ValueError('Document must have exactly one root.')
must_return = False
if output is None:
output = StringIO()
must_return = True
if short_empty_elements:
content_handler = XMLGenerator(output, encoding, True)
else:
content_handler = XMLGenerator(output, encoding)
if full_document:
content_handler.startDocument()
for key, value in input_dict.items():
_emit(key, value, content_handler, full_document=full_document,
**kwargs)
if full_document:
content_handler.endDocument()
if must_return:
value = output.getvalue()
try: # pragma no cover
value = value.decode(encoding)
except AttributeError: # pragma no cover
pass
return value
if __name__ == '__main__': # pragma: no cover
import sys
import marshal
try:
stdin = sys.stdin.buffer
stdout = sys.stdout.buffer
except AttributeError:
stdin = sys.stdin
stdout = sys.stdout
(item_depth,) = sys.argv[1:]
item_depth = int(item_depth)
def handle_item(path, item):
marshal.dump((path, item), stdout)
return True
try:
root = parse(stdin,
item_depth=item_depth,
item_callback=handle_item,
dict_constructor=dict)
if item_depth == 0:
handle_item([], root)
except KeyboardInterrupt:
pass