mirror of
https://github.com/hexated/cloudstream-extensions-hexated.git
synced 2024-08-15 00:03:22 +00:00
2285 lines
80 KiB
Python
2285 lines
80 KiB
Python
"""Small, fast HTTP client library for Python.
|
|
|
|
Features persistent connections, cache, and Google App Engine Standard
|
|
Environment support.
|
|
"""
|
|
|
|
from __future__ import print_function
|
|
|
|
__author__ = "Joe Gregorio (joe@bitworking.org)"
|
|
__copyright__ = "Copyright 2006, Joe Gregorio"
|
|
__contributors__ = [
|
|
"Thomas Broyer (t.broyer@ltgt.net)",
|
|
"James Antill",
|
|
"Xavier Verges Farrero",
|
|
"Jonathan Feinberg",
|
|
"Blair Zajac",
|
|
"Sam Ruby",
|
|
"Louis Nyffenegger",
|
|
"Alex Yu",
|
|
]
|
|
__license__ = "MIT"
|
|
__version__ = '0.17.0'
|
|
|
|
import base64
|
|
import calendar
|
|
import copy
|
|
import email
|
|
import email.FeedParser
|
|
import email.Message
|
|
import email.Utils
|
|
import errno
|
|
import gzip
|
|
import httplib
|
|
import os
|
|
import random
|
|
import re
|
|
import StringIO
|
|
import sys
|
|
import time
|
|
import urllib
|
|
import urlparse
|
|
import zlib
|
|
|
|
try:
|
|
from hashlib import sha1 as _sha, md5 as _md5
|
|
except ImportError:
|
|
# prior to Python 2.5, these were separate modules
|
|
import sha
|
|
import md5
|
|
|
|
_sha = sha.new
|
|
_md5 = md5.new
|
|
import hmac
|
|
from gettext import gettext as _
|
|
import socket
|
|
|
|
try:
|
|
import socks
|
|
except ImportError:
|
|
try:
|
|
import socks
|
|
except (ImportError, AttributeError):
|
|
socks = None
|
|
|
|
# Build the appropriate socket wrapper for ssl
|
|
ssl = None
|
|
ssl_SSLError = None
|
|
ssl_CertificateError = None
|
|
try:
|
|
import ssl # python 2.6
|
|
except ImportError:
|
|
pass
|
|
if ssl is not None:
|
|
ssl_SSLError = getattr(ssl, "SSLError", None)
|
|
ssl_CertificateError = getattr(ssl, "CertificateError", None)
|
|
|
|
|
|
def _ssl_wrap_socket(
|
|
sock, key_file, cert_file, disable_validation, ca_certs, ssl_version, hostname, key_password
|
|
):
|
|
if disable_validation:
|
|
cert_reqs = ssl.CERT_NONE
|
|
else:
|
|
cert_reqs = ssl.CERT_REQUIRED
|
|
if ssl_version is None:
|
|
ssl_version = ssl.PROTOCOL_SSLv23
|
|
|
|
if hasattr(ssl, "SSLContext"): # Python 2.7.9
|
|
context = ssl.SSLContext(ssl_version)
|
|
context.verify_mode = cert_reqs
|
|
context.check_hostname = cert_reqs != ssl.CERT_NONE
|
|
if cert_file:
|
|
if key_password:
|
|
context.load_cert_chain(cert_file, key_file, key_password)
|
|
else:
|
|
context.load_cert_chain(cert_file, key_file)
|
|
if ca_certs:
|
|
context.load_verify_locations(ca_certs)
|
|
return context.wrap_socket(sock, server_hostname=hostname)
|
|
else:
|
|
if key_password:
|
|
raise NotSupportedOnThisPlatform("Certificate with password is not supported.")
|
|
return ssl.wrap_socket(
|
|
sock,
|
|
keyfile=key_file,
|
|
certfile=cert_file,
|
|
cert_reqs=cert_reqs,
|
|
ca_certs=ca_certs,
|
|
ssl_version=ssl_version,
|
|
)
|
|
|
|
|
|
def _ssl_wrap_socket_unsupported(
|
|
sock, key_file, cert_file, disable_validation, ca_certs, ssl_version, hostname, key_password
|
|
):
|
|
if not disable_validation:
|
|
raise CertificateValidationUnsupported(
|
|
"SSL certificate validation is not supported without "
|
|
"the ssl module installed. To avoid this error, install "
|
|
"the ssl module, or explicity disable validation."
|
|
)
|
|
if key_password:
|
|
raise NotSupportedOnThisPlatform("Certificate with password is not supported.")
|
|
ssl_sock = socket.ssl(sock, key_file, cert_file)
|
|
return httplib.FakeSocket(sock, ssl_sock)
|
|
|
|
|
|
if ssl is None:
|
|
_ssl_wrap_socket = _ssl_wrap_socket_unsupported
|
|
|
|
if sys.version_info >= (2, 3):
|
|
from iri2uri import iri2uri
|
|
else:
|
|
|
|
def iri2uri(uri):
|
|
return uri
|
|
|
|
|
|
def has_timeout(timeout): # python 2.6
|
|
if hasattr(socket, "_GLOBAL_DEFAULT_TIMEOUT"):
|
|
return timeout is not None and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT
|
|
return timeout is not None
|
|
|
|
|
|
__all__ = [
|
|
"Http",
|
|
"Response",
|
|
"ProxyInfo",
|
|
"HttpLib2Error",
|
|
"RedirectMissingLocation",
|
|
"RedirectLimit",
|
|
"FailedToDecompressContent",
|
|
"UnimplementedDigestAuthOptionError",
|
|
"UnimplementedHmacDigestAuthOptionError",
|
|
"debuglevel",
|
|
"ProxiesUnavailableError",
|
|
]
|
|
|
|
# The httplib debug level, set to a non-zero value to get debug output
|
|
debuglevel = 0
|
|
|
|
# A request will be tried 'RETRIES' times if it fails at the socket/connection level.
|
|
RETRIES = 2
|
|
|
|
# Python 2.3 support
|
|
if sys.version_info < (2, 4):
|
|
|
|
def sorted(seq):
|
|
seq.sort()
|
|
return seq
|
|
|
|
|
|
# Python 2.3 support
|
|
def HTTPResponse__getheaders(self):
|
|
"""Return list of (header, value) tuples."""
|
|
if self.msg is None:
|
|
raise httplib.ResponseNotReady()
|
|
return self.msg.items()
|
|
|
|
|
|
if not hasattr(httplib.HTTPResponse, "getheaders"):
|
|
httplib.HTTPResponse.getheaders = HTTPResponse__getheaders
|
|
|
|
|
|
# All exceptions raised here derive from HttpLib2Error
|
|
class HttpLib2Error(Exception):
|
|
pass
|
|
|
|
|
|
# Some exceptions can be caught and optionally
|
|
# be turned back into responses.
|
|
class HttpLib2ErrorWithResponse(HttpLib2Error):
|
|
def __init__(self, desc, response, content):
|
|
self.response = response
|
|
self.content = content
|
|
HttpLib2Error.__init__(self, desc)
|
|
|
|
|
|
class RedirectMissingLocation(HttpLib2ErrorWithResponse):
|
|
pass
|
|
|
|
|
|
class RedirectLimit(HttpLib2ErrorWithResponse):
|
|
pass
|
|
|
|
|
|
class FailedToDecompressContent(HttpLib2ErrorWithResponse):
|
|
pass
|
|
|
|
|
|
class UnimplementedDigestAuthOptionError(HttpLib2ErrorWithResponse):
|
|
pass
|
|
|
|
|
|
class UnimplementedHmacDigestAuthOptionError(HttpLib2ErrorWithResponse):
|
|
pass
|
|
|
|
|
|
class MalformedHeader(HttpLib2Error):
|
|
pass
|
|
|
|
|
|
class RelativeURIError(HttpLib2Error):
|
|
pass
|
|
|
|
|
|
class ServerNotFoundError(HttpLib2Error):
|
|
pass
|
|
|
|
|
|
class ProxiesUnavailableError(HttpLib2Error):
|
|
pass
|
|
|
|
|
|
class CertificateValidationUnsupported(HttpLib2Error):
|
|
pass
|
|
|
|
|
|
class SSLHandshakeError(HttpLib2Error):
|
|
pass
|
|
|
|
|
|
class NotSupportedOnThisPlatform(HttpLib2Error):
|
|
pass
|
|
|
|
|
|
class CertificateHostnameMismatch(SSLHandshakeError):
|
|
def __init__(self, desc, host, cert):
|
|
HttpLib2Error.__init__(self, desc)
|
|
self.host = host
|
|
self.cert = cert
|
|
|
|
|
|
class NotRunningAppEngineEnvironment(HttpLib2Error):
|
|
pass
|
|
|
|
|
|
# Open Items:
|
|
# -----------
|
|
# Proxy support
|
|
|
|
# Are we removing the cached content too soon on PUT (only delete on 200 Maybe?)
|
|
|
|
# Pluggable cache storage (supports storing the cache in
|
|
# flat files by default. We need a plug-in architecture
|
|
# that can support Berkeley DB and Squid)
|
|
|
|
# == Known Issues ==
|
|
# Does not handle a resource that uses conneg and Last-Modified but no ETag as a cache validator.
|
|
# Does not handle Cache-Control: max-stale
|
|
# Does not use Age: headers when calculating cache freshness.
|
|
|
|
# The number of redirections to follow before giving up.
|
|
# Note that only GET redirects are automatically followed.
|
|
# Will also honor 301 requests by saving that info and never
|
|
# requesting that URI again.
|
|
DEFAULT_MAX_REDIRECTS = 5
|
|
|
|
import certs
|
|
CA_CERTS = certs.where()
|
|
|
|
# Which headers are hop-by-hop headers by default
|
|
HOP_BY_HOP = [
|
|
"connection",
|
|
"keep-alive",
|
|
"proxy-authenticate",
|
|
"proxy-authorization",
|
|
"te",
|
|
"trailers",
|
|
"transfer-encoding",
|
|
"upgrade",
|
|
]
|
|
|
|
# https://tools.ietf.org/html/rfc7231#section-8.1.3
|
|
SAFE_METHODS = ("GET", "HEAD") # TODO add "OPTIONS", "TRACE"
|
|
|
|
# To change, assign to `Http().redirect_codes`
|
|
REDIRECT_CODES = frozenset((300, 301, 302, 303, 307, 308))
|
|
|
|
|
|
def _get_end2end_headers(response):
|
|
hopbyhop = list(HOP_BY_HOP)
|
|
hopbyhop.extend([x.strip() for x in response.get("connection", "").split(",")])
|
|
return [header for header in response.keys() if header not in hopbyhop]
|
|
|
|
|
|
URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?")
|
|
|
|
|
|
def parse_uri(uri):
|
|
"""Parses a URI using the regex given in Appendix B of RFC 3986.
|
|
|
|
(scheme, authority, path, query, fragment) = parse_uri(uri)
|
|
"""
|
|
groups = URI.match(uri).groups()
|
|
return (groups[1], groups[3], groups[4], groups[6], groups[8])
|
|
|
|
|
|
def urlnorm(uri):
|
|
(scheme, authority, path, query, fragment) = parse_uri(uri)
|
|
if not scheme or not authority:
|
|
raise RelativeURIError("Only absolute URIs are allowed. uri = %s" % uri)
|
|
authority = authority.lower()
|
|
scheme = scheme.lower()
|
|
if not path:
|
|
path = "/"
|
|
# Could do syntax based normalization of the URI before
|
|
# computing the digest. See Section 6.2.2 of Std 66.
|
|
request_uri = query and "?".join([path, query]) or path
|
|
scheme = scheme.lower()
|
|
defrag_uri = scheme + "://" + authority + request_uri
|
|
return scheme, authority, request_uri, defrag_uri
|
|
|
|
|
|
# Cache filename construction (original borrowed from Venus http://intertwingly.net/code/venus/)
|
|
re_url_scheme = re.compile(r"^\w+://")
|
|
re_unsafe = re.compile(r"[^\w\-_.()=!]+")
|
|
|
|
|
|
def safename(filename):
|
|
"""Return a filename suitable for the cache.
|
|
Strips dangerous and common characters to create a filename we
|
|
can use to store the cache in.
|
|
"""
|
|
if isinstance(filename, str):
|
|
filename_bytes = filename
|
|
filename = filename.decode("utf-8")
|
|
else:
|
|
filename_bytes = filename.encode("utf-8")
|
|
filemd5 = _md5(filename_bytes).hexdigest()
|
|
filename = re_url_scheme.sub("", filename)
|
|
filename = re_unsafe.sub("", filename)
|
|
|
|
# limit length of filename (vital for Windows)
|
|
# https://github.com/httplib2/httplib2/pull/74
|
|
# C:\Users\ <username> \AppData\Local\Temp\ <safe_filename> , <md5>
|
|
# 9 chars + max 104 chars + 20 chars + x + 1 + 32 = max 259 chars
|
|
# Thus max safe filename x = 93 chars. Let it be 90 to make a round sum:
|
|
filename = filename[:90]
|
|
|
|
return ",".join((filename, filemd5))
|
|
|
|
|
|
NORMALIZE_SPACE = re.compile(r"(?:\r\n)?[ \t]+")
|
|
|
|
|
|
def _normalize_headers(headers):
|
|
return dict(
|
|
[
|
|
(key.lower(), NORMALIZE_SPACE.sub(value, " ").strip())
|
|
for (key, value) in headers.iteritems()
|
|
]
|
|
)
|
|
|
|
|
|
def _parse_cache_control(headers):
|
|
retval = {}
|
|
if "cache-control" in headers:
|
|
parts = headers["cache-control"].split(",")
|
|
parts_with_args = [
|
|
tuple([x.strip().lower() for x in part.split("=", 1)])
|
|
for part in parts
|
|
if -1 != part.find("=")
|
|
]
|
|
parts_wo_args = [
|
|
(name.strip().lower(), 1) for name in parts if -1 == name.find("=")
|
|
]
|
|
retval = dict(parts_with_args + parts_wo_args)
|
|
return retval
|
|
|
|
|
|
# Whether to use a strict mode to parse WWW-Authenticate headers
|
|
# Might lead to bad results in case of ill-formed header value,
|
|
# so disabled by default, falling back to relaxed parsing.
|
|
# Set to true to turn on, usefull for testing servers.
|
|
USE_WWW_AUTH_STRICT_PARSING = 0
|
|
|
|
# In regex below:
|
|
# [^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+ matches a "token" as defined by HTTP
|
|
# "(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?" matches a "quoted-string" as defined by HTTP, when LWS have already been replaced by a single space
|
|
# Actually, as an auth-param value can be either a token or a quoted-string, they are combined in a single pattern which matches both:
|
|
# \"?((?<=\")(?:[^\0-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?<!\")[^\0-\x08\x0A-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+(?!\"))\"?
|
|
WWW_AUTH_STRICT = re.compile(
|
|
r"^(?:\s*(?:,\s*)?([^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+)\s*=\s*\"?((?<=\")(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?<!\")[^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+(?!\"))\"?)(.*)$"
|
|
)
|
|
WWW_AUTH_RELAXED = re.compile(
|
|
r"^(?:\s*(?:,\s*)?([^ \t\r\n=]+)\s*=\s*\"?((?<=\")(?:[^\\\"]|\\.)*?(?=\")|(?<!\")[^ \t\r\n,]+(?!\"))\"?)(.*)$"
|
|
)
|
|
UNQUOTE_PAIRS = re.compile(r"\\(.)")
|
|
|
|
|
|
def _parse_www_authenticate(headers, headername="www-authenticate"):
|
|
"""Returns a dictionary of dictionaries, one dict
|
|
per auth_scheme."""
|
|
retval = {}
|
|
if headername in headers:
|
|
try:
|
|
|
|
authenticate = headers[headername].strip()
|
|
www_auth = (
|
|
USE_WWW_AUTH_STRICT_PARSING and WWW_AUTH_STRICT or WWW_AUTH_RELAXED
|
|
)
|
|
while authenticate:
|
|
# Break off the scheme at the beginning of the line
|
|
if headername == "authentication-info":
|
|
(auth_scheme, the_rest) = ("digest", authenticate)
|
|
else:
|
|
(auth_scheme, the_rest) = authenticate.split(" ", 1)
|
|
# Now loop over all the key value pairs that come after the scheme,
|
|
# being careful not to roll into the next scheme
|
|
match = www_auth.search(the_rest)
|
|
auth_params = {}
|
|
while match:
|
|
if match and len(match.groups()) == 3:
|
|
(key, value, the_rest) = match.groups()
|
|
auth_params[key.lower()] = UNQUOTE_PAIRS.sub(
|
|
r"\1", value
|
|
) # '\\'.join([x.replace('\\', '') for x in value.split('\\\\')])
|
|
match = www_auth.search(the_rest)
|
|
retval[auth_scheme.lower()] = auth_params
|
|
authenticate = the_rest.strip()
|
|
|
|
except ValueError:
|
|
raise MalformedHeader("WWW-Authenticate")
|
|
return retval
|
|
|
|
|
|
# TODO: add current time as _entry_disposition argument to avoid sleep in tests
|
|
def _entry_disposition(response_headers, request_headers):
|
|
"""Determine freshness from the Date, Expires and Cache-Control headers.
|
|
|
|
We don't handle the following:
|
|
|
|
1. Cache-Control: max-stale
|
|
2. Age: headers are not used in the calculations.
|
|
|
|
Not that this algorithm is simpler than you might think
|
|
because we are operating as a private (non-shared) cache.
|
|
This lets us ignore 's-maxage'. We can also ignore
|
|
'proxy-invalidate' since we aren't a proxy.
|
|
We will never return a stale document as
|
|
fresh as a design decision, and thus the non-implementation
|
|
of 'max-stale'. This also lets us safely ignore 'must-revalidate'
|
|
since we operate as if every server has sent 'must-revalidate'.
|
|
Since we are private we get to ignore both 'public' and
|
|
'private' parameters. We also ignore 'no-transform' since
|
|
we don't do any transformations.
|
|
The 'no-store' parameter is handled at a higher level.
|
|
So the only Cache-Control parameters we look at are:
|
|
|
|
no-cache
|
|
only-if-cached
|
|
max-age
|
|
min-fresh
|
|
"""
|
|
|
|
retval = "STALE"
|
|
cc = _parse_cache_control(request_headers)
|
|
cc_response = _parse_cache_control(response_headers)
|
|
|
|
if (
|
|
"pragma" in request_headers
|
|
and request_headers["pragma"].lower().find("no-cache") != -1
|
|
):
|
|
retval = "TRANSPARENT"
|
|
if "cache-control" not in request_headers:
|
|
request_headers["cache-control"] = "no-cache"
|
|
elif "no-cache" in cc:
|
|
retval = "TRANSPARENT"
|
|
elif "no-cache" in cc_response:
|
|
retval = "STALE"
|
|
elif "only-if-cached" in cc:
|
|
retval = "FRESH"
|
|
elif "date" in response_headers:
|
|
date = calendar.timegm(email.Utils.parsedate_tz(response_headers["date"]))
|
|
now = time.time()
|
|
current_age = max(0, now - date)
|
|
if "max-age" in cc_response:
|
|
try:
|
|
freshness_lifetime = int(cc_response["max-age"])
|
|
except ValueError:
|
|
freshness_lifetime = 0
|
|
elif "expires" in response_headers:
|
|
expires = email.Utils.parsedate_tz(response_headers["expires"])
|
|
if None == expires:
|
|
freshness_lifetime = 0
|
|
else:
|
|
freshness_lifetime = max(0, calendar.timegm(expires) - date)
|
|
else:
|
|
freshness_lifetime = 0
|
|
if "max-age" in cc:
|
|
try:
|
|
freshness_lifetime = int(cc["max-age"])
|
|
except ValueError:
|
|
freshness_lifetime = 0
|
|
if "min-fresh" in cc:
|
|
try:
|
|
min_fresh = int(cc["min-fresh"])
|
|
except ValueError:
|
|
min_fresh = 0
|
|
current_age += min_fresh
|
|
if freshness_lifetime > current_age:
|
|
retval = "FRESH"
|
|
return retval
|
|
|
|
|
|
def _decompressContent(response, new_content):
|
|
content = new_content
|
|
try:
|
|
encoding = response.get("content-encoding", None)
|
|
if encoding in ["gzip", "deflate"]:
|
|
if encoding == "gzip":
|
|
content = gzip.GzipFile(fileobj=StringIO.StringIO(new_content)).read()
|
|
if encoding == "deflate":
|
|
content = zlib.decompress(content, -zlib.MAX_WBITS)
|
|
response["content-length"] = str(len(content))
|
|
# Record the historical presence of the encoding in a way the won't interfere.
|
|
response["-content-encoding"] = response["content-encoding"]
|
|
del response["content-encoding"]
|
|
except (IOError, zlib.error):
|
|
content = ""
|
|
raise FailedToDecompressContent(
|
|
_("Content purported to be compressed with %s but failed to decompress.")
|
|
% response.get("content-encoding"),
|
|
response,
|
|
content,
|
|
)
|
|
return content
|
|
|
|
|
|
def _updateCache(request_headers, response_headers, content, cache, cachekey):
|
|
if cachekey:
|
|
cc = _parse_cache_control(request_headers)
|
|
cc_response = _parse_cache_control(response_headers)
|
|
if "no-store" in cc or "no-store" in cc_response:
|
|
cache.delete(cachekey)
|
|
else:
|
|
info = email.Message.Message()
|
|
for key, value in response_headers.iteritems():
|
|
if key not in ["status", "content-encoding", "transfer-encoding"]:
|
|
info[key] = value
|
|
|
|
# Add annotations to the cache to indicate what headers
|
|
# are variant for this request.
|
|
vary = response_headers.get("vary", None)
|
|
if vary:
|
|
vary_headers = vary.lower().replace(" ", "").split(",")
|
|
for header in vary_headers:
|
|
key = "-varied-%s" % header
|
|
try:
|
|
info[key] = request_headers[header]
|
|
except KeyError:
|
|
pass
|
|
|
|
status = response_headers.status
|
|
if status == 304:
|
|
status = 200
|
|
|
|
status_header = "status: %d\r\n" % status
|
|
|
|
header_str = info.as_string()
|
|
|
|
header_str = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", header_str)
|
|
text = "".join([status_header, header_str, content])
|
|
|
|
cache.set(cachekey, text)
|
|
|
|
|
|
def _cnonce():
|
|
dig = _md5(
|
|
"%s:%s"
|
|
% (time.ctime(), ["0123456789"[random.randrange(0, 9)] for i in range(20)])
|
|
).hexdigest()
|
|
return dig[:16]
|
|
|
|
|
|
def _wsse_username_token(cnonce, iso_now, password):
|
|
return base64.b64encode(
|
|
_sha("%s%s%s" % (cnonce, iso_now, password)).digest()
|
|
).strip()
|
|
|
|
|
|
# For credentials we need two things, first
|
|
# a pool of credential to try (not necesarily tied to BAsic, Digest, etc.)
|
|
# Then we also need a list of URIs that have already demanded authentication
|
|
# That list is tricky since sub-URIs can take the same auth, or the
|
|
# auth scheme may change as you descend the tree.
|
|
# So we also need each Auth instance to be able to tell us
|
|
# how close to the 'top' it is.
|
|
|
|
|
|
class Authentication(object):
|
|
def __init__(
|
|
self, credentials, host, request_uri, headers, response, content, http
|
|
):
|
|
(scheme, authority, path, query, fragment) = parse_uri(request_uri)
|
|
self.path = path
|
|
self.host = host
|
|
self.credentials = credentials
|
|
self.http = http
|
|
|
|
def depth(self, request_uri):
|
|
(scheme, authority, path, query, fragment) = parse_uri(request_uri)
|
|
return request_uri[len(self.path) :].count("/")
|
|
|
|
def inscope(self, host, request_uri):
|
|
# XXX Should we normalize the request_uri?
|
|
(scheme, authority, path, query, fragment) = parse_uri(request_uri)
|
|
return (host == self.host) and path.startswith(self.path)
|
|
|
|
def request(self, method, request_uri, headers, content):
|
|
"""Modify the request headers to add the appropriate
|
|
Authorization header. Over-ride this in sub-classes."""
|
|
pass
|
|
|
|
def response(self, response, content):
|
|
"""Gives us a chance to update with new nonces
|
|
or such returned from the last authorized response.
|
|
Over-rise this in sub-classes if necessary.
|
|
|
|
Return TRUE is the request is to be retried, for
|
|
example Digest may return stale=true.
|
|
"""
|
|
return False
|
|
|
|
|
|
class BasicAuthentication(Authentication):
|
|
def __init__(
|
|
self, credentials, host, request_uri, headers, response, content, http
|
|
):
|
|
Authentication.__init__(
|
|
self, credentials, host, request_uri, headers, response, content, http
|
|
)
|
|
|
|
def request(self, method, request_uri, headers, content):
|
|
"""Modify the request headers to add the appropriate
|
|
Authorization header."""
|
|
headers["authorization"] = (
|
|
"Basic " + base64.b64encode("%s:%s" % self.credentials).strip()
|
|
)
|
|
|
|
|
|
class DigestAuthentication(Authentication):
|
|
"""Only do qop='auth' and MD5, since that
|
|
is all Apache currently implements"""
|
|
|
|
def __init__(
|
|
self, credentials, host, request_uri, headers, response, content, http
|
|
):
|
|
Authentication.__init__(
|
|
self, credentials, host, request_uri, headers, response, content, http
|
|
)
|
|
challenge = _parse_www_authenticate(response, "www-authenticate")
|
|
self.challenge = challenge["digest"]
|
|
qop = self.challenge.get("qop", "auth")
|
|
self.challenge["qop"] = (
|
|
("auth" in [x.strip() for x in qop.split()]) and "auth" or None
|
|
)
|
|
if self.challenge["qop"] is None:
|
|
raise UnimplementedDigestAuthOptionError(
|
|
_("Unsupported value for qop: %s." % qop)
|
|
)
|
|
self.challenge["algorithm"] = self.challenge.get("algorithm", "MD5").upper()
|
|
if self.challenge["algorithm"] != "MD5":
|
|
raise UnimplementedDigestAuthOptionError(
|
|
_("Unsupported value for algorithm: %s." % self.challenge["algorithm"])
|
|
)
|
|
self.A1 = "".join(
|
|
[
|
|
self.credentials[0],
|
|
":",
|
|
self.challenge["realm"],
|
|
":",
|
|
self.credentials[1],
|
|
]
|
|
)
|
|
self.challenge["nc"] = 1
|
|
|
|
def request(self, method, request_uri, headers, content, cnonce=None):
|
|
"""Modify the request headers"""
|
|
H = lambda x: _md5(x).hexdigest()
|
|
KD = lambda s, d: H("%s:%s" % (s, d))
|
|
A2 = "".join([method, ":", request_uri])
|
|
self.challenge["cnonce"] = cnonce or _cnonce()
|
|
request_digest = '"%s"' % KD(
|
|
H(self.A1),
|
|
"%s:%s:%s:%s:%s"
|
|
% (
|
|
self.challenge["nonce"],
|
|
"%08x" % self.challenge["nc"],
|
|
self.challenge["cnonce"],
|
|
self.challenge["qop"],
|
|
H(A2),
|
|
),
|
|
)
|
|
headers["authorization"] = (
|
|
'Digest username="%s", realm="%s", nonce="%s", '
|
|
'uri="%s", algorithm=%s, response=%s, qop=%s, '
|
|
'nc=%08x, cnonce="%s"'
|
|
) % (
|
|
self.credentials[0],
|
|
self.challenge["realm"],
|
|
self.challenge["nonce"],
|
|
request_uri,
|
|
self.challenge["algorithm"],
|
|
request_digest,
|
|
self.challenge["qop"],
|
|
self.challenge["nc"],
|
|
self.challenge["cnonce"],
|
|
)
|
|
if self.challenge.get("opaque"):
|
|
headers["authorization"] += ', opaque="%s"' % self.challenge["opaque"]
|
|
self.challenge["nc"] += 1
|
|
|
|
def response(self, response, content):
|
|
if "authentication-info" not in response:
|
|
challenge = _parse_www_authenticate(response, "www-authenticate").get(
|
|
"digest", {}
|
|
)
|
|
if "true" == challenge.get("stale"):
|
|
self.challenge["nonce"] = challenge["nonce"]
|
|
self.challenge["nc"] = 1
|
|
return True
|
|
else:
|
|
updated_challenge = _parse_www_authenticate(
|
|
response, "authentication-info"
|
|
).get("digest", {})
|
|
|
|
if "nextnonce" in updated_challenge:
|
|
self.challenge["nonce"] = updated_challenge["nextnonce"]
|
|
self.challenge["nc"] = 1
|
|
return False
|
|
|
|
|
|
class HmacDigestAuthentication(Authentication):
|
|
"""Adapted from Robert Sayre's code and DigestAuthentication above."""
|
|
|
|
__author__ = "Thomas Broyer (t.broyer@ltgt.net)"
|
|
|
|
def __init__(
|
|
self, credentials, host, request_uri, headers, response, content, http
|
|
):
|
|
Authentication.__init__(
|
|
self, credentials, host, request_uri, headers, response, content, http
|
|
)
|
|
challenge = _parse_www_authenticate(response, "www-authenticate")
|
|
self.challenge = challenge["hmacdigest"]
|
|
# TODO: self.challenge['domain']
|
|
self.challenge["reason"] = self.challenge.get("reason", "unauthorized")
|
|
if self.challenge["reason"] not in ["unauthorized", "integrity"]:
|
|
self.challenge["reason"] = "unauthorized"
|
|
self.challenge["salt"] = self.challenge.get("salt", "")
|
|
if not self.challenge.get("snonce"):
|
|
raise UnimplementedHmacDigestAuthOptionError(
|
|
_("The challenge doesn't contain a server nonce, or this one is empty.")
|
|
)
|
|
self.challenge["algorithm"] = self.challenge.get("algorithm", "HMAC-SHA-1")
|
|
if self.challenge["algorithm"] not in ["HMAC-SHA-1", "HMAC-MD5"]:
|
|
raise UnimplementedHmacDigestAuthOptionError(
|
|
_("Unsupported value for algorithm: %s." % self.challenge["algorithm"])
|
|
)
|
|
self.challenge["pw-algorithm"] = self.challenge.get("pw-algorithm", "SHA-1")
|
|
if self.challenge["pw-algorithm"] not in ["SHA-1", "MD5"]:
|
|
raise UnimplementedHmacDigestAuthOptionError(
|
|
_(
|
|
"Unsupported value for pw-algorithm: %s."
|
|
% self.challenge["pw-algorithm"]
|
|
)
|
|
)
|
|
if self.challenge["algorithm"] == "HMAC-MD5":
|
|
self.hashmod = _md5
|
|
else:
|
|
self.hashmod = _sha
|
|
if self.challenge["pw-algorithm"] == "MD5":
|
|
self.pwhashmod = _md5
|
|
else:
|
|
self.pwhashmod = _sha
|
|
self.key = "".join(
|
|
[
|
|
self.credentials[0],
|
|
":",
|
|
self.pwhashmod.new(
|
|
"".join([self.credentials[1], self.challenge["salt"]])
|
|
)
|
|
.hexdigest()
|
|
.lower(),
|
|
":",
|
|
self.challenge["realm"],
|
|
]
|
|
)
|
|
self.key = self.pwhashmod.new(self.key).hexdigest().lower()
|
|
|
|
def request(self, method, request_uri, headers, content):
|
|
"""Modify the request headers"""
|
|
keys = _get_end2end_headers(headers)
|
|
keylist = "".join(["%s " % k for k in keys])
|
|
headers_val = "".join([headers[k] for k in keys])
|
|
created = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
cnonce = _cnonce()
|
|
request_digest = "%s:%s:%s:%s:%s" % (
|
|
method,
|
|
request_uri,
|
|
cnonce,
|
|
self.challenge["snonce"],
|
|
headers_val,
|
|
)
|
|
request_digest = (
|
|
hmac.new(self.key, request_digest, self.hashmod).hexdigest().lower()
|
|
)
|
|
headers["authorization"] = (
|
|
'HMACDigest username="%s", realm="%s", snonce="%s",'
|
|
' cnonce="%s", uri="%s", created="%s", '
|
|
'response="%s", headers="%s"'
|
|
) % (
|
|
self.credentials[0],
|
|
self.challenge["realm"],
|
|
self.challenge["snonce"],
|
|
cnonce,
|
|
request_uri,
|
|
created,
|
|
request_digest,
|
|
keylist,
|
|
)
|
|
|
|
def response(self, response, content):
|
|
challenge = _parse_www_authenticate(response, "www-authenticate").get(
|
|
"hmacdigest", {}
|
|
)
|
|
if challenge.get("reason") in ["integrity", "stale"]:
|
|
return True
|
|
return False
|
|
|
|
|
|
class WsseAuthentication(Authentication):
|
|
"""This is thinly tested and should not be relied upon.
|
|
At this time there isn't any third party server to test against.
|
|
Blogger and TypePad implemented this algorithm at one point
|
|
but Blogger has since switched to Basic over HTTPS and
|
|
TypePad has implemented it wrong, by never issuing a 401
|
|
challenge but instead requiring your client to telepathically know that
|
|
their endpoint is expecting WSSE profile="UsernameToken"."""
|
|
|
|
def __init__(
|
|
self, credentials, host, request_uri, headers, response, content, http
|
|
):
|
|
Authentication.__init__(
|
|
self, credentials, host, request_uri, headers, response, content, http
|
|
)
|
|
|
|
def request(self, method, request_uri, headers, content):
|
|
"""Modify the request headers to add the appropriate
|
|
Authorization header."""
|
|
headers["authorization"] = 'WSSE profile="UsernameToken"'
|
|
iso_now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
cnonce = _cnonce()
|
|
password_digest = _wsse_username_token(cnonce, iso_now, self.credentials[1])
|
|
headers["X-WSSE"] = (
|
|
'UsernameToken Username="%s", PasswordDigest="%s", '
|
|
'Nonce="%s", Created="%s"'
|
|
) % (self.credentials[0], password_digest, cnonce, iso_now)
|
|
|
|
|
|
class GoogleLoginAuthentication(Authentication):
|
|
def __init__(
|
|
self, credentials, host, request_uri, headers, response, content, http
|
|
):
|
|
from urllib import urlencode
|
|
|
|
Authentication.__init__(
|
|
self, credentials, host, request_uri, headers, response, content, http
|
|
)
|
|
challenge = _parse_www_authenticate(response, "www-authenticate")
|
|
service = challenge["googlelogin"].get("service", "xapi")
|
|
# Bloggger actually returns the service in the challenge
|
|
# For the rest we guess based on the URI
|
|
if service == "xapi" and request_uri.find("calendar") > 0:
|
|
service = "cl"
|
|
# No point in guessing Base or Spreadsheet
|
|
# elif request_uri.find("spreadsheets") > 0:
|
|
# service = "wise"
|
|
|
|
auth = dict(
|
|
Email=credentials[0],
|
|
Passwd=credentials[1],
|
|
service=service,
|
|
source=headers["user-agent"],
|
|
)
|
|
resp, content = self.http.request(
|
|
"https://www.google.com/accounts/ClientLogin",
|
|
method="POST",
|
|
body=urlencode(auth),
|
|
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
|
)
|
|
lines = content.split("\n")
|
|
d = dict([tuple(line.split("=", 1)) for line in lines if line])
|
|
if resp.status == 403:
|
|
self.Auth = ""
|
|
else:
|
|
self.Auth = d["Auth"]
|
|
|
|
def request(self, method, request_uri, headers, content):
|
|
"""Modify the request headers to add the appropriate
|
|
Authorization header."""
|
|
headers["authorization"] = "GoogleLogin Auth=" + self.Auth
|
|
|
|
|
|
AUTH_SCHEME_CLASSES = {
|
|
"basic": BasicAuthentication,
|
|
"wsse": WsseAuthentication,
|
|
"digest": DigestAuthentication,
|
|
"hmacdigest": HmacDigestAuthentication,
|
|
"googlelogin": GoogleLoginAuthentication,
|
|
}
|
|
|
|
AUTH_SCHEME_ORDER = ["hmacdigest", "googlelogin", "digest", "wsse", "basic"]
|
|
|
|
|
|
class FileCache(object):
|
|
"""Uses a local directory as a store for cached files.
|
|
Not really safe to use if multiple threads or processes are going to
|
|
be running on the same cache.
|
|
"""
|
|
|
|
def __init__(
|
|
self, cache, safe=safename
|
|
): # use safe=lambda x: md5.new(x).hexdigest() for the old behavior
|
|
self.cache = cache
|
|
self.safe = safe
|
|
if not os.path.exists(cache):
|
|
os.makedirs(self.cache)
|
|
|
|
def get(self, key):
|
|
retval = None
|
|
cacheFullPath = os.path.join(self.cache, self.safe(key))
|
|
try:
|
|
f = file(cacheFullPath, "rb")
|
|
retval = f.read()
|
|
f.close()
|
|
except IOError:
|
|
pass
|
|
return retval
|
|
|
|
def set(self, key, value):
|
|
cacheFullPath = os.path.join(self.cache, self.safe(key))
|
|
f = file(cacheFullPath, "wb")
|
|
f.write(value)
|
|
f.close()
|
|
|
|
def delete(self, key):
|
|
cacheFullPath = os.path.join(self.cache, self.safe(key))
|
|
if os.path.exists(cacheFullPath):
|
|
os.remove(cacheFullPath)
|
|
|
|
|
|
class Credentials(object):
|
|
def __init__(self):
|
|
self.credentials = []
|
|
|
|
def add(self, name, password, domain=""):
|
|
self.credentials.append((domain.lower(), name, password))
|
|
|
|
def clear(self):
|
|
self.credentials = []
|
|
|
|
def iter(self, domain):
|
|
for (cdomain, name, password) in self.credentials:
|
|
if cdomain == "" or domain == cdomain:
|
|
yield (name, password)
|
|
|
|
|
|
class KeyCerts(Credentials):
|
|
"""Identical to Credentials except that
|
|
name/password are mapped to key/cert."""
|
|
def add(self, key, cert, domain, password):
|
|
self.credentials.append((domain.lower(), key, cert, password))
|
|
|
|
def iter(self, domain):
|
|
for (cdomain, key, cert, password) in self.credentials:
|
|
if cdomain == "" or domain == cdomain:
|
|
yield (key, cert, password)
|
|
|
|
|
|
class AllHosts(object):
|
|
pass
|
|
|
|
|
|
class ProxyInfo(object):
|
|
"""Collect information required to use a proxy."""
|
|
|
|
bypass_hosts = ()
|
|
|
|
def __init__(
|
|
self,
|
|
proxy_type,
|
|
proxy_host,
|
|
proxy_port,
|
|
proxy_rdns=True,
|
|
proxy_user=None,
|
|
proxy_pass=None,
|
|
proxy_headers=None,
|
|
):
|
|
"""Args:
|
|
|
|
proxy_type: The type of proxy server. This must be set to one of
|
|
socks.PROXY_TYPE_XXX constants. For example: p =
|
|
ProxyInfo(proxy_type=socks.PROXY_TYPE_HTTP, proxy_host='localhost',
|
|
proxy_port=8000)
|
|
proxy_host: The hostname or IP address of the proxy server.
|
|
proxy_port: The port that the proxy server is running on.
|
|
proxy_rdns: If True (default), DNS queries will not be performed
|
|
locally, and instead, handed to the proxy to resolve. This is useful
|
|
if the network does not allow resolution of non-local names. In
|
|
httplib2 0.9 and earlier, this defaulted to False.
|
|
proxy_user: The username used to authenticate with the proxy server.
|
|
proxy_pass: The password used to authenticate with the proxy server.
|
|
proxy_headers: Additional or modified headers for the proxy connect
|
|
request.
|
|
"""
|
|
self.proxy_type = proxy_type
|
|
self.proxy_host = proxy_host
|
|
self.proxy_port = proxy_port
|
|
self.proxy_rdns = proxy_rdns
|
|
self.proxy_user = proxy_user
|
|
self.proxy_pass = proxy_pass
|
|
self.proxy_headers = proxy_headers
|
|
|
|
def astuple(self):
|
|
return (
|
|
self.proxy_type,
|
|
self.proxy_host,
|
|
self.proxy_port,
|
|
self.proxy_rdns,
|
|
self.proxy_user,
|
|
self.proxy_pass,
|
|
self.proxy_headers,
|
|
)
|
|
|
|
def isgood(self):
|
|
return (self.proxy_host != None) and (self.proxy_port != None)
|
|
|
|
def applies_to(self, hostname):
|
|
return not self.bypass_host(hostname)
|
|
|
|
def bypass_host(self, hostname):
|
|
"""Has this host been excluded from the proxy config"""
|
|
if self.bypass_hosts is AllHosts:
|
|
return True
|
|
|
|
hostname = "." + hostname.lstrip(".")
|
|
for skip_name in self.bypass_hosts:
|
|
# *.suffix
|
|
if skip_name.startswith(".") and hostname.endswith(skip_name):
|
|
return True
|
|
# exact match
|
|
if hostname == "." + skip_name:
|
|
return True
|
|
return False
|
|
|
|
def __repr__(self):
|
|
return (
|
|
"<ProxyInfo type={p.proxy_type} "
|
|
"host:port={p.proxy_host}:{p.proxy_port} rdns={p.proxy_rdns}"
|
|
+ " user={p.proxy_user} headers={p.proxy_headers}>"
|
|
).format(p=self)
|
|
|
|
|
|
def proxy_info_from_environment(method="http"):
|
|
"""Read proxy info from the environment variables.
|
|
"""
|
|
if method not in ["http", "https"]:
|
|
return
|
|
|
|
env_var = method + "_proxy"
|
|
url = os.environ.get(env_var, os.environ.get(env_var.upper()))
|
|
if not url:
|
|
return
|
|
return proxy_info_from_url(url, method, None)
|
|
|
|
|
|
def proxy_info_from_url(url, method="http", noproxy=None):
|
|
"""Construct a ProxyInfo from a URL (such as http_proxy env var)
|
|
"""
|
|
url = urlparse.urlparse(url)
|
|
username = None
|
|
password = None
|
|
port = None
|
|
if "@" in url[1]:
|
|
ident, host_port = url[1].split("@", 1)
|
|
if ":" in ident:
|
|
username, password = ident.split(":", 1)
|
|
else:
|
|
password = ident
|
|
else:
|
|
host_port = url[1]
|
|
if ":" in host_port:
|
|
host, port = host_port.split(":", 1)
|
|
else:
|
|
host = host_port
|
|
|
|
if port:
|
|
port = int(port)
|
|
else:
|
|
port = dict(https=443, http=80)[method]
|
|
|
|
proxy_type = 3 # socks.PROXY_TYPE_HTTP
|
|
pi = ProxyInfo(
|
|
proxy_type=proxy_type,
|
|
proxy_host=host,
|
|
proxy_port=port,
|
|
proxy_user=username or None,
|
|
proxy_pass=password or None,
|
|
proxy_headers=None,
|
|
)
|
|
|
|
bypass_hosts = []
|
|
# If not given an explicit noproxy value, respect values in env vars.
|
|
if noproxy is None:
|
|
noproxy = os.environ.get("no_proxy", os.environ.get("NO_PROXY", ""))
|
|
# Special case: A single '*' character means all hosts should be bypassed.
|
|
if noproxy == "*":
|
|
bypass_hosts = AllHosts
|
|
elif noproxy.strip():
|
|
bypass_hosts = noproxy.split(",")
|
|
bypass_hosts = filter(bool, bypass_hosts) # To exclude empty string.
|
|
|
|
pi.bypass_hosts = bypass_hosts
|
|
return pi
|
|
|
|
|
|
class HTTPConnectionWithTimeout(httplib.HTTPConnection):
|
|
"""HTTPConnection subclass that supports timeouts
|
|
|
|
All timeouts are in seconds. If None is passed for timeout then
|
|
Python's default timeout for sockets will be used. See for example
|
|
the docs of socket.setdefaulttimeout():
|
|
http://docs.python.org/library/socket.html#socket.setdefaulttimeout
|
|
"""
|
|
|
|
def __init__(self, host, port=None, strict=None, timeout=None, proxy_info=None):
|
|
httplib.HTTPConnection.__init__(self, host, port, strict)
|
|
self.timeout = timeout
|
|
self.proxy_info = proxy_info
|
|
|
|
def connect(self):
|
|
"""Connect to the host and port specified in __init__."""
|
|
# Mostly verbatim from httplib.py.
|
|
if self.proxy_info and socks is None:
|
|
raise ProxiesUnavailableError(
|
|
"Proxy support missing but proxy use was requested!"
|
|
)
|
|
if self.proxy_info and self.proxy_info.isgood():
|
|
use_proxy = True
|
|
proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers = (
|
|
self.proxy_info.astuple()
|
|
)
|
|
|
|
host = proxy_host
|
|
port = proxy_port
|
|
else:
|
|
use_proxy = False
|
|
|
|
host = self.host
|
|
port = self.port
|
|
|
|
socket_err = None
|
|
|
|
for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
|
|
af, socktype, proto, canonname, sa = res
|
|
try:
|
|
if use_proxy:
|
|
self.sock = socks.socksocket(af, socktype, proto)
|
|
self.sock.setproxy(
|
|
proxy_type,
|
|
proxy_host,
|
|
proxy_port,
|
|
proxy_rdns,
|
|
proxy_user,
|
|
proxy_pass,
|
|
proxy_headers,
|
|
)
|
|
else:
|
|
self.sock = socket.socket(af, socktype, proto)
|
|
self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
|
# Different from httplib: support timeouts.
|
|
if has_timeout(self.timeout):
|
|
self.sock.settimeout(self.timeout)
|
|
# End of difference from httplib.
|
|
if self.debuglevel > 0:
|
|
print("connect: (%s, %s) ************" % (self.host, self.port))
|
|
if use_proxy:
|
|
print(
|
|
"proxy: %s ************"
|
|
% str(
|
|
(
|
|
proxy_host,
|
|
proxy_port,
|
|
proxy_rdns,
|
|
proxy_user,
|
|
proxy_pass,
|
|
proxy_headers,
|
|
)
|
|
)
|
|
)
|
|
if use_proxy:
|
|
self.sock.connect((self.host, self.port) + sa[2:])
|
|
else:
|
|
self.sock.connect(sa)
|
|
except socket.error as e:
|
|
socket_err = e
|
|
if self.debuglevel > 0:
|
|
print("connect fail: (%s, %s)" % (self.host, self.port))
|
|
if use_proxy:
|
|
print(
|
|
"proxy: %s"
|
|
% str(
|
|
(
|
|
proxy_host,
|
|
proxy_port,
|
|
proxy_rdns,
|
|
proxy_user,
|
|
proxy_pass,
|
|
proxy_headers,
|
|
)
|
|
)
|
|
)
|
|
if self.sock:
|
|
self.sock.close()
|
|
self.sock = None
|
|
continue
|
|
break
|
|
if not self.sock:
|
|
raise socket_err or socket.error("getaddrinfo returns an empty list")
|
|
|
|
|
|
class HTTPSConnectionWithTimeout(httplib.HTTPSConnection):
|
|
"""This class allows communication via SSL.
|
|
|
|
All timeouts are in seconds. If None is passed for timeout then
|
|
Python's default timeout for sockets will be used. See for example
|
|
the docs of socket.setdefaulttimeout():
|
|
http://docs.python.org/library/socket.html#socket.setdefaulttimeout
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
host,
|
|
port=None,
|
|
key_file=None,
|
|
cert_file=None,
|
|
strict=None,
|
|
timeout=None,
|
|
proxy_info=None,
|
|
ca_certs=None,
|
|
disable_ssl_certificate_validation=False,
|
|
ssl_version=None,
|
|
key_password=None,
|
|
):
|
|
if key_password:
|
|
httplib.HTTPSConnection.__init__(self, host, port=port, strict=strict)
|
|
self._context.load_cert_chain(cert_file, key_file, key_password)
|
|
self.key_file = key_file
|
|
self.cert_file = cert_file
|
|
self.key_password = key_password
|
|
else:
|
|
httplib.HTTPSConnection.__init__(
|
|
self, host, port=port, key_file=key_file, cert_file=cert_file, strict=strict
|
|
)
|
|
self.key_password = None
|
|
self.timeout = timeout
|
|
self.proxy_info = proxy_info
|
|
if ca_certs is None:
|
|
ca_certs = CA_CERTS
|
|
self.ca_certs = ca_certs
|
|
self.disable_ssl_certificate_validation = disable_ssl_certificate_validation
|
|
self.ssl_version = ssl_version
|
|
|
|
# The following two methods were adapted from https_wrapper.py, released
|
|
# with the Google Appengine SDK at
|
|
# http://googleappengine.googlecode.com/svn-history/r136/trunk/python/google/appengine/tools/https_wrapper.py
|
|
# under the following license:
|
|
#
|
|
# Copyright 2007 Google Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
def _GetValidHostsForCert(self, cert):
|
|
"""Returns a list of valid host globs for an SSL certificate.
|
|
|
|
Args:
|
|
cert: A dictionary representing an SSL certificate.
|
|
Returns:
|
|
list: A list of valid host globs.
|
|
"""
|
|
if "subjectAltName" in cert:
|
|
return [x[1] for x in cert["subjectAltName"] if x[0].lower() == "dns"]
|
|
else:
|
|
return [x[0][1] for x in cert["subject"] if x[0][0].lower() == "commonname"]
|
|
|
|
def _ValidateCertificateHostname(self, cert, hostname):
|
|
"""Validates that a given hostname is valid for an SSL certificate.
|
|
|
|
Args:
|
|
cert: A dictionary representing an SSL certificate.
|
|
hostname: The hostname to test.
|
|
Returns:
|
|
bool: Whether or not the hostname is valid for this certificate.
|
|
"""
|
|
hosts = self._GetValidHostsForCert(cert)
|
|
for host in hosts:
|
|
host_re = host.replace(".", "\.").replace("*", "[^.]*")
|
|
if re.search("^%s$" % (host_re,), hostname, re.I):
|
|
return True
|
|
return False
|
|
|
|
def connect(self):
|
|
"Connect to a host on a given (SSL) port."
|
|
|
|
if self.proxy_info and self.proxy_info.isgood():
|
|
use_proxy = True
|
|
proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers = (
|
|
self.proxy_info.astuple()
|
|
)
|
|
|
|
host = proxy_host
|
|
port = proxy_port
|
|
else:
|
|
use_proxy = False
|
|
|
|
host = self.host
|
|
port = self.port
|
|
|
|
socket_err = None
|
|
|
|
address_info = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
|
|
for family, socktype, proto, canonname, sockaddr in address_info:
|
|
try:
|
|
if use_proxy:
|
|
sock = socks.socksocket(family, socktype, proto)
|
|
|
|
sock.setproxy(
|
|
proxy_type,
|
|
proxy_host,
|
|
proxy_port,
|
|
proxy_rdns,
|
|
proxy_user,
|
|
proxy_pass,
|
|
proxy_headers,
|
|
)
|
|
else:
|
|
sock = socket.socket(family, socktype, proto)
|
|
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
|
|
|
if has_timeout(self.timeout):
|
|
sock.settimeout(self.timeout)
|
|
|
|
if use_proxy:
|
|
sock.connect((self.host, self.port) + sockaddr[:2])
|
|
else:
|
|
sock.connect(sockaddr)
|
|
self.sock = _ssl_wrap_socket(
|
|
sock,
|
|
self.key_file,
|
|
self.cert_file,
|
|
self.disable_ssl_certificate_validation,
|
|
self.ca_certs,
|
|
self.ssl_version,
|
|
self.host,
|
|
self.key_password,
|
|
)
|
|
if self.debuglevel > 0:
|
|
print("connect: (%s, %s)" % (self.host, self.port))
|
|
if use_proxy:
|
|
print(
|
|
"proxy: %s"
|
|
% str(
|
|
(
|
|
proxy_host,
|
|
proxy_port,
|
|
proxy_rdns,
|
|
proxy_user,
|
|
proxy_pass,
|
|
proxy_headers,
|
|
)
|
|
)
|
|
)
|
|
if not self.disable_ssl_certificate_validation:
|
|
cert = self.sock.getpeercert()
|
|
hostname = self.host.split(":", 0)[0]
|
|
if not self._ValidateCertificateHostname(cert, hostname):
|
|
raise CertificateHostnameMismatch(
|
|
"Server presented certificate that does not match "
|
|
"host %s: %s" % (hostname, cert),
|
|
hostname,
|
|
cert,
|
|
)
|
|
except (
|
|
ssl_SSLError,
|
|
ssl_CertificateError,
|
|
CertificateHostnameMismatch,
|
|
) as e:
|
|
if sock:
|
|
sock.close()
|
|
if self.sock:
|
|
self.sock.close()
|
|
self.sock = None
|
|
# Unfortunately the ssl module doesn't seem to provide any way
|
|
# to get at more detailed error information, in particular
|
|
# whether the error is due to certificate validation or
|
|
# something else (such as SSL protocol mismatch).
|
|
if getattr(e, "errno", None) == ssl.SSL_ERROR_SSL:
|
|
raise SSLHandshakeError(e)
|
|
else:
|
|
raise
|
|
except (socket.timeout, socket.gaierror):
|
|
raise
|
|
except socket.error as e:
|
|
socket_err = e
|
|
if self.debuglevel > 0:
|
|
print("connect fail: (%s, %s)" % (self.host, self.port))
|
|
if use_proxy:
|
|
print(
|
|
"proxy: %s"
|
|
% str(
|
|
(
|
|
proxy_host,
|
|
proxy_port,
|
|
proxy_rdns,
|
|
proxy_user,
|
|
proxy_pass,
|
|
proxy_headers,
|
|
)
|
|
)
|
|
)
|
|
if self.sock:
|
|
self.sock.close()
|
|
self.sock = None
|
|
continue
|
|
break
|
|
if not self.sock:
|
|
raise socket_err or socket.error("getaddrinfo returns an empty list")
|
|
|
|
|
|
SCHEME_TO_CONNECTION = {
|
|
"http": HTTPConnectionWithTimeout,
|
|
"https": HTTPSConnectionWithTimeout,
|
|
}
|
|
|
|
|
|
def _new_fixed_fetch(validate_certificate):
|
|
|
|
def fixed_fetch(
|
|
url,
|
|
payload=None,
|
|
method="GET",
|
|
headers={},
|
|
allow_truncated=False,
|
|
follow_redirects=True,
|
|
deadline=None,
|
|
):
|
|
return fetch(
|
|
url,
|
|
payload=payload,
|
|
method=method,
|
|
headers=headers,
|
|
allow_truncated=allow_truncated,
|
|
follow_redirects=follow_redirects,
|
|
deadline=deadline,
|
|
validate_certificate=validate_certificate,
|
|
)
|
|
|
|
return fixed_fetch
|
|
|
|
|
|
class AppEngineHttpConnection(httplib.HTTPConnection):
|
|
"""Use httplib on App Engine, but compensate for its weirdness.
|
|
|
|
The parameters key_file, cert_file, proxy_info, ca_certs,
|
|
disable_ssl_certificate_validation, and ssl_version are all dropped on
|
|
the ground.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
host,
|
|
port=None,
|
|
key_file=None,
|
|
cert_file=None,
|
|
strict=None,
|
|
timeout=None,
|
|
proxy_info=None,
|
|
ca_certs=None,
|
|
disable_ssl_certificate_validation=False,
|
|
ssl_version=None,
|
|
):
|
|
httplib.HTTPConnection.__init__(
|
|
self, host, port=port, strict=strict, timeout=timeout
|
|
)
|
|
|
|
|
|
class AppEngineHttpsConnection(httplib.HTTPSConnection):
|
|
"""Same as AppEngineHttpConnection, but for HTTPS URIs.
|
|
|
|
The parameters proxy_info, ca_certs, disable_ssl_certificate_validation,
|
|
and ssl_version are all dropped on the ground.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
host,
|
|
port=None,
|
|
key_file=None,
|
|
cert_file=None,
|
|
strict=None,
|
|
timeout=None,
|
|
proxy_info=None,
|
|
ca_certs=None,
|
|
disable_ssl_certificate_validation=False,
|
|
ssl_version=None,
|
|
key_password=None,
|
|
):
|
|
if key_password:
|
|
raise NotSupportedOnThisPlatform("Certificate with password is not supported.")
|
|
httplib.HTTPSConnection.__init__(
|
|
self,
|
|
host,
|
|
port=port,
|
|
key_file=key_file,
|
|
cert_file=cert_file,
|
|
strict=strict,
|
|
timeout=timeout,
|
|
)
|
|
self._fetch = _new_fixed_fetch(not disable_ssl_certificate_validation)
|
|
|
|
|
|
# Use a different connection object for Google App Engine Standard Environment.
|
|
def is_gae_instance():
|
|
server_software = os.environ.get('SERVER_SOFTWARE', '')
|
|
if (server_software.startswith('Google App Engine/') or
|
|
server_software.startswith('Development/') or
|
|
server_software.startswith('testutil/')):
|
|
return True
|
|
return False
|
|
|
|
|
|
try:
|
|
if not is_gae_instance():
|
|
raise NotRunningAppEngineEnvironment()
|
|
|
|
from google.appengine.api import apiproxy_stub_map
|
|
if apiproxy_stub_map.apiproxy.GetStub("urlfetch") is None:
|
|
raise ImportError
|
|
|
|
from google.appengine.api.urlfetch import fetch
|
|
|
|
# Update the connection classes to use the Googel App Engine specific ones.
|
|
SCHEME_TO_CONNECTION = {
|
|
"http": AppEngineHttpConnection,
|
|
"https": AppEngineHttpsConnection,
|
|
}
|
|
except (ImportError, NotRunningAppEngineEnvironment):
|
|
pass
|
|
|
|
|
|
class Http(object):
|
|
"""An HTTP client that handles:
|
|
|
|
- all methods
|
|
- caching
|
|
- ETags
|
|
- compression,
|
|
- HTTPS
|
|
- Basic
|
|
- Digest
|
|
- WSSE
|
|
|
|
and more.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
cache=None,
|
|
timeout=None,
|
|
proxy_info=proxy_info_from_environment,
|
|
ca_certs=None,
|
|
disable_ssl_certificate_validation=False,
|
|
ssl_version=None,
|
|
):
|
|
"""If 'cache' is a string then it is used as a directory name for
|
|
a disk cache. Otherwise it must be an object that supports the
|
|
same interface as FileCache.
|
|
|
|
All timeouts are in seconds. If None is passed for timeout
|
|
then Python's default timeout for sockets will be used. See
|
|
for example the docs of socket.setdefaulttimeout():
|
|
http://docs.python.org/library/socket.html#socket.setdefaulttimeout
|
|
|
|
`proxy_info` may be:
|
|
- a callable that takes the http scheme ('http' or 'https') and
|
|
returns a ProxyInfo instance per request. By default, uses
|
|
proxy_nfo_from_environment.
|
|
- a ProxyInfo instance (static proxy config).
|
|
- None (proxy disabled).
|
|
|
|
ca_certs is the path of a file containing root CA certificates for SSL
|
|
server certificate validation. By default, a CA cert file bundled with
|
|
httplib2 is used.
|
|
|
|
If disable_ssl_certificate_validation is true, SSL cert validation will
|
|
not be performed.
|
|
|
|
By default, ssl.PROTOCOL_SSLv23 will be used for the ssl version.
|
|
"""
|
|
self.proxy_info = proxy_info
|
|
self.ca_certs = ca_certs
|
|
self.disable_ssl_certificate_validation = disable_ssl_certificate_validation
|
|
self.ssl_version = ssl_version
|
|
|
|
# Map domain name to an httplib connection
|
|
self.connections = {}
|
|
# The location of the cache, for now a directory
|
|
# where cached responses are held.
|
|
if cache and isinstance(cache, basestring):
|
|
self.cache = FileCache(cache)
|
|
else:
|
|
self.cache = cache
|
|
|
|
# Name/password
|
|
self.credentials = Credentials()
|
|
|
|
# Key/cert
|
|
self.certificates = KeyCerts()
|
|
|
|
# authorization objects
|
|
self.authorizations = []
|
|
|
|
# If set to False then no redirects are followed, even safe ones.
|
|
self.follow_redirects = True
|
|
|
|
self.redirect_codes = REDIRECT_CODES
|
|
|
|
# Which HTTP methods do we apply optimistic concurrency to, i.e.
|
|
# which methods get an "if-match:" etag header added to them.
|
|
self.optimistic_concurrency_methods = ["PUT", "PATCH"]
|
|
|
|
self.safe_methods = list(SAFE_METHODS)
|
|
|
|
# If 'follow_redirects' is True, and this is set to True then
|
|
# all redirecs are followed, including unsafe ones.
|
|
self.follow_all_redirects = False
|
|
|
|
self.ignore_etag = False
|
|
|
|
self.force_exception_to_status_code = False
|
|
|
|
self.timeout = timeout
|
|
|
|
# Keep Authorization: headers on a redirect.
|
|
self.forward_authorization_headers = False
|
|
|
|
def close(self):
|
|
"""Close persistent connections, clear sensitive data.
|
|
Not thread-safe, requires external synchronization against concurrent requests.
|
|
"""
|
|
existing, self.connections = self.connections, {}
|
|
for _, c in existing.iteritems():
|
|
c.close()
|
|
self.certificates.clear()
|
|
self.clear_credentials()
|
|
|
|
def __getstate__(self):
|
|
state_dict = copy.copy(self.__dict__)
|
|
# In case request is augmented by some foreign object such as
|
|
# credentials which handle auth
|
|
if "request" in state_dict:
|
|
del state_dict["request"]
|
|
if "connections" in state_dict:
|
|
del state_dict["connections"]
|
|
return state_dict
|
|
|
|
def __setstate__(self, state):
|
|
self.__dict__.update(state)
|
|
self.connections = {}
|
|
|
|
def _auth_from_challenge(self, host, request_uri, headers, response, content):
|
|
"""A generator that creates Authorization objects
|
|
that can be applied to requests.
|
|
"""
|
|
challenges = _parse_www_authenticate(response, "www-authenticate")
|
|
for cred in self.credentials.iter(host):
|
|
for scheme in AUTH_SCHEME_ORDER:
|
|
if scheme in challenges:
|
|
yield AUTH_SCHEME_CLASSES[scheme](
|
|
cred, host, request_uri, headers, response, content, self
|
|
)
|
|
|
|
def add_credentials(self, name, password, domain=""):
|
|
"""Add a name and password that will be used
|
|
any time a request requires authentication."""
|
|
self.credentials.add(name, password, domain)
|
|
|
|
def add_certificate(self, key, cert, domain, password=None):
|
|
"""Add a key and cert that will be used
|
|
any time a request requires authentication."""
|
|
self.certificates.add(key, cert, domain, password)
|
|
|
|
def clear_credentials(self):
|
|
"""Remove all the names and passwords
|
|
that are used for authentication"""
|
|
self.credentials.clear()
|
|
self.authorizations = []
|
|
|
|
def _conn_request(self, conn, request_uri, method, body, headers):
|
|
i = 0
|
|
seen_bad_status_line = False
|
|
while i < RETRIES:
|
|
i += 1
|
|
try:
|
|
if hasattr(conn, "sock") and conn.sock is None:
|
|
conn.connect()
|
|
conn.request(method, request_uri, body, headers)
|
|
except socket.timeout:
|
|
raise
|
|
except socket.gaierror:
|
|
conn.close()
|
|
raise ServerNotFoundError("Unable to find the server at %s" % conn.host)
|
|
except ssl_SSLError:
|
|
conn.close()
|
|
raise
|
|
except socket.error as e:
|
|
err = 0
|
|
if hasattr(e, "args"):
|
|
err = getattr(e, "args")[0]
|
|
else:
|
|
err = e.errno
|
|
if err == errno.ECONNREFUSED: # Connection refused
|
|
raise
|
|
if err in (errno.ENETUNREACH, errno.EADDRNOTAVAIL) and i < RETRIES:
|
|
continue # retry on potentially transient socket errors
|
|
except httplib.HTTPException:
|
|
# Just because the server closed the connection doesn't apparently mean
|
|
# that the server didn't send a response.
|
|
if hasattr(conn, "sock") and conn.sock is None:
|
|
if i < RETRIES - 1:
|
|
conn.close()
|
|
conn.connect()
|
|
continue
|
|
else:
|
|
conn.close()
|
|
raise
|
|
if i < RETRIES - 1:
|
|
conn.close()
|
|
conn.connect()
|
|
continue
|
|
try:
|
|
response = conn.getresponse()
|
|
except httplib.BadStatusLine:
|
|
# If we get a BadStatusLine on the first try then that means
|
|
# the connection just went stale, so retry regardless of the
|
|
# number of RETRIES set.
|
|
if not seen_bad_status_line and i == 1:
|
|
i = 0
|
|
seen_bad_status_line = True
|
|
conn.close()
|
|
conn.connect()
|
|
continue
|
|
else:
|
|
conn.close()
|
|
raise
|
|
except (socket.error, httplib.HTTPException):
|
|
if i < RETRIES - 1:
|
|
conn.close()
|
|
conn.connect()
|
|
continue
|
|
else:
|
|
conn.close()
|
|
raise
|
|
else:
|
|
content = ""
|
|
if method == "HEAD":
|
|
conn.close()
|
|
else:
|
|
content = response.read()
|
|
response = Response(response)
|
|
if method != "HEAD":
|
|
content = _decompressContent(response, content)
|
|
break
|
|
return (response, content)
|
|
|
|
def _request(
|
|
self,
|
|
conn,
|
|
host,
|
|
absolute_uri,
|
|
request_uri,
|
|
method,
|
|
body,
|
|
headers,
|
|
redirections,
|
|
cachekey,
|
|
):
|
|
"""Do the actual request using the connection object
|
|
and also follow one level of redirects if necessary"""
|
|
|
|
auths = [
|
|
(auth.depth(request_uri), auth)
|
|
for auth in self.authorizations
|
|
if auth.inscope(host, request_uri)
|
|
]
|
|
auth = auths and sorted(auths)[0][1] or None
|
|
if auth:
|
|
auth.request(method, request_uri, headers, body)
|
|
|
|
(response, content) = self._conn_request(
|
|
conn, request_uri, method, body, headers
|
|
)
|
|
|
|
if auth:
|
|
if auth.response(response, body):
|
|
auth.request(method, request_uri, headers, body)
|
|
(response, content) = self._conn_request(
|
|
conn, request_uri, method, body, headers
|
|
)
|
|
response._stale_digest = 1
|
|
|
|
if response.status == 401:
|
|
for authorization in self._auth_from_challenge(
|
|
host, request_uri, headers, response, content
|
|
):
|
|
authorization.request(method, request_uri, headers, body)
|
|
(response, content) = self._conn_request(
|
|
conn, request_uri, method, body, headers
|
|
)
|
|
if response.status != 401:
|
|
self.authorizations.append(authorization)
|
|
authorization.response(response, body)
|
|
break
|
|
|
|
if (
|
|
self.follow_all_redirects
|
|
or method in self.safe_methods
|
|
or response.status in (303, 308)
|
|
):
|
|
if self.follow_redirects and response.status in self.redirect_codes:
|
|
# Pick out the location header and basically start from the beginning
|
|
# remembering first to strip the ETag header and decrement our 'depth'
|
|
if redirections:
|
|
if "location" not in response and response.status != 300:
|
|
raise RedirectMissingLocation(
|
|
_(
|
|
"Redirected but the response is missing a Location: header."
|
|
),
|
|
response,
|
|
content,
|
|
)
|
|
# Fix-up relative redirects (which violate an RFC 2616 MUST)
|
|
if "location" in response:
|
|
location = response["location"]
|
|
(scheme, authority, path, query, fragment) = parse_uri(location)
|
|
if authority == None:
|
|
response["location"] = urlparse.urljoin(
|
|
absolute_uri, location
|
|
)
|
|
if response.status == 308 or (response.status == 301 and method in self.safe_methods):
|
|
response["-x-permanent-redirect-url"] = response["location"]
|
|
if "content-location" not in response:
|
|
response["content-location"] = absolute_uri
|
|
_updateCache(headers, response, content, self.cache, cachekey)
|
|
if "if-none-match" in headers:
|
|
del headers["if-none-match"]
|
|
if "if-modified-since" in headers:
|
|
del headers["if-modified-since"]
|
|
if (
|
|
"authorization" in headers
|
|
and not self.forward_authorization_headers
|
|
):
|
|
del headers["authorization"]
|
|
if "location" in response:
|
|
location = response["location"]
|
|
old_response = copy.deepcopy(response)
|
|
if "content-location" not in old_response:
|
|
old_response["content-location"] = absolute_uri
|
|
redirect_method = method
|
|
if response.status in [302, 303]:
|
|
redirect_method = "GET"
|
|
body = None
|
|
(response, content) = self.request(
|
|
location,
|
|
method=redirect_method,
|
|
body=body,
|
|
headers=headers,
|
|
redirections=redirections - 1,
|
|
)
|
|
response.previous = old_response
|
|
else:
|
|
raise RedirectLimit(
|
|
"Redirected more times than rediection_limit allows.",
|
|
response,
|
|
content,
|
|
)
|
|
elif response.status in [200, 203] and method in self.safe_methods:
|
|
# Don't cache 206's since we aren't going to handle byte range requests
|
|
if "content-location" not in response:
|
|
response["content-location"] = absolute_uri
|
|
_updateCache(headers, response, content, self.cache, cachekey)
|
|
|
|
return (response, content)
|
|
|
|
def _normalize_headers(self, headers):
|
|
return _normalize_headers(headers)
|
|
|
|
# Need to catch and rebrand some exceptions
|
|
# Then need to optionally turn all exceptions into status codes
|
|
# including all socket.* and httplib.* exceptions.
|
|
|
|
def request(
|
|
self,
|
|
uri,
|
|
method="GET",
|
|
body=None,
|
|
headers=None,
|
|
redirections=DEFAULT_MAX_REDIRECTS,
|
|
connection_type=None,
|
|
):
|
|
""" Performs a single HTTP request.
|
|
|
|
The 'uri' is the URI of the HTTP resource and can begin with either
|
|
'http' or 'https'. The value of 'uri' must be an absolute URI.
|
|
|
|
The 'method' is the HTTP method to perform, such as GET, POST, DELETE,
|
|
etc. There is no restriction on the methods allowed.
|
|
|
|
The 'body' is the entity body to be sent with the request. It is a
|
|
string object.
|
|
|
|
Any extra headers that are to be sent with the request should be
|
|
provided in the 'headers' dictionary.
|
|
|
|
The maximum number of redirect to follow before raising an
|
|
exception is 'redirections. The default is 5.
|
|
|
|
The return value is a tuple of (response, content), the first
|
|
being and instance of the 'Response' class, the second being
|
|
a string that contains the response entity body.
|
|
"""
|
|
conn_key = ''
|
|
|
|
try:
|
|
if headers is None:
|
|
headers = {}
|
|
else:
|
|
headers = self._normalize_headers(headers)
|
|
|
|
if "user-agent" not in headers:
|
|
headers["user-agent"] = "Python-httplib2/%s (gzip)" % __version__
|
|
|
|
uri = iri2uri(uri)
|
|
|
|
(scheme, authority, request_uri, defrag_uri) = urlnorm(uri)
|
|
|
|
proxy_info = self._get_proxy_info(scheme, authority)
|
|
|
|
conn_key = scheme + ":" + authority
|
|
conn = self.connections.get(conn_key)
|
|
if conn is None:
|
|
if not connection_type:
|
|
connection_type = SCHEME_TO_CONNECTION[scheme]
|
|
certs = list(self.certificates.iter(authority))
|
|
if scheme == "https":
|
|
if certs:
|
|
conn = self.connections[conn_key] = connection_type(
|
|
authority,
|
|
key_file=certs[0][0],
|
|
cert_file=certs[0][1],
|
|
timeout=self.timeout,
|
|
proxy_info=proxy_info,
|
|
ca_certs=self.ca_certs,
|
|
disable_ssl_certificate_validation=self.disable_ssl_certificate_validation,
|
|
ssl_version=self.ssl_version,
|
|
key_password=certs[0][2],
|
|
)
|
|
else:
|
|
conn = self.connections[conn_key] = connection_type(
|
|
authority,
|
|
timeout=self.timeout,
|
|
proxy_info=proxy_info,
|
|
ca_certs=self.ca_certs,
|
|
disable_ssl_certificate_validation=self.disable_ssl_certificate_validation,
|
|
ssl_version=self.ssl_version,
|
|
)
|
|
else:
|
|
conn = self.connections[conn_key] = connection_type(
|
|
authority, timeout=self.timeout, proxy_info=proxy_info
|
|
)
|
|
conn.set_debuglevel(debuglevel)
|
|
|
|
if "range" not in headers and "accept-encoding" not in headers:
|
|
headers["accept-encoding"] = "gzip, deflate"
|
|
|
|
info = email.Message.Message()
|
|
cachekey = None
|
|
cached_value = None
|
|
if self.cache:
|
|
cachekey = defrag_uri.encode("utf-8")
|
|
cached_value = self.cache.get(cachekey)
|
|
if cached_value:
|
|
# info = email.message_from_string(cached_value)
|
|
#
|
|
# Need to replace the line above with the kludge below
|
|
# to fix the non-existent bug not fixed in this
|
|
# bug report: http://mail.python.org/pipermail/python-bugs-list/2005-September/030289.html
|
|
try:
|
|
info, content = cached_value.split("\r\n\r\n", 1)
|
|
feedparser = email.FeedParser.FeedParser()
|
|
feedparser.feed(info)
|
|
info = feedparser.close()
|
|
feedparser._parse = None
|
|
except (IndexError, ValueError):
|
|
self.cache.delete(cachekey)
|
|
cachekey = None
|
|
cached_value = None
|
|
|
|
if (
|
|
method in self.optimistic_concurrency_methods
|
|
and self.cache
|
|
and "etag" in info
|
|
and not self.ignore_etag
|
|
and "if-match" not in headers
|
|
):
|
|
# http://www.w3.org/1999/04/Editing/
|
|
headers["if-match"] = info["etag"]
|
|
|
|
# https://tools.ietf.org/html/rfc7234
|
|
# A cache MUST invalidate the effective Request URI as well as [...] Location and Content-Location
|
|
# when a non-error status code is received in response to an unsafe request method.
|
|
if self.cache and cachekey and method not in self.safe_methods:
|
|
self.cache.delete(cachekey)
|
|
|
|
# Check the vary header in the cache to see if this request
|
|
# matches what varies in the cache.
|
|
if method in self.safe_methods and "vary" in info:
|
|
vary = info["vary"]
|
|
vary_headers = vary.lower().replace(" ", "").split(",")
|
|
for header in vary_headers:
|
|
key = "-varied-%s" % header
|
|
value = info[key]
|
|
if headers.get(header, None) != value:
|
|
cached_value = None
|
|
break
|
|
|
|
if (
|
|
self.cache
|
|
and cached_value
|
|
and (method in self.safe_methods or info["status"] == "308")
|
|
and "range" not in headers
|
|
):
|
|
redirect_method = method
|
|
if info["status"] not in ("307", "308"):
|
|
redirect_method = "GET"
|
|
if "-x-permanent-redirect-url" in info:
|
|
# Should cached permanent redirects be counted in our redirection count? For now, yes.
|
|
if redirections <= 0:
|
|
raise RedirectLimit(
|
|
"Redirected more times than rediection_limit allows.",
|
|
{},
|
|
"",
|
|
)
|
|
(response, new_content) = self.request(
|
|
info["-x-permanent-redirect-url"],
|
|
method=redirect_method,
|
|
headers=headers,
|
|
redirections=redirections - 1,
|
|
)
|
|
response.previous = Response(info)
|
|
response.previous.fromcache = True
|
|
else:
|
|
# Determine our course of action:
|
|
# Is the cached entry fresh or stale?
|
|
# Has the client requested a non-cached response?
|
|
#
|
|
# There seems to be three possible answers:
|
|
# 1. [FRESH] Return the cache entry w/o doing a GET
|
|
# 2. [STALE] Do the GET (but add in cache validators if available)
|
|
# 3. [TRANSPARENT] Do a GET w/o any cache validators (Cache-Control: no-cache) on the request
|
|
entry_disposition = _entry_disposition(info, headers)
|
|
|
|
if entry_disposition == "FRESH":
|
|
if not cached_value:
|
|
info["status"] = "504"
|
|
content = ""
|
|
response = Response(info)
|
|
if cached_value:
|
|
response.fromcache = True
|
|
return (response, content)
|
|
|
|
if entry_disposition == "STALE":
|
|
if (
|
|
"etag" in info
|
|
and not self.ignore_etag
|
|
and not "if-none-match" in headers
|
|
):
|
|
headers["if-none-match"] = info["etag"]
|
|
if "last-modified" in info and not "last-modified" in headers:
|
|
headers["if-modified-since"] = info["last-modified"]
|
|
elif entry_disposition == "TRANSPARENT":
|
|
pass
|
|
|
|
(response, new_content) = self._request(
|
|
conn,
|
|
authority,
|
|
uri,
|
|
request_uri,
|
|
method,
|
|
body,
|
|
headers,
|
|
redirections,
|
|
cachekey,
|
|
)
|
|
|
|
if response.status == 304 and method == "GET":
|
|
# Rewrite the cache entry with the new end-to-end headers
|
|
# Take all headers that are in response
|
|
# and overwrite their values in info.
|
|
# unless they are hop-by-hop, or are listed in the connection header.
|
|
|
|
for key in _get_end2end_headers(response):
|
|
info[key] = response[key]
|
|
merged_response = Response(info)
|
|
if hasattr(response, "_stale_digest"):
|
|
merged_response._stale_digest = response._stale_digest
|
|
_updateCache(
|
|
headers, merged_response, content, self.cache, cachekey
|
|
)
|
|
response = merged_response
|
|
response.status = 200
|
|
response.fromcache = True
|
|
|
|
elif response.status == 200:
|
|
content = new_content
|
|
else:
|
|
self.cache.delete(cachekey)
|
|
content = new_content
|
|
else:
|
|
cc = _parse_cache_control(headers)
|
|
if "only-if-cached" in cc:
|
|
info["status"] = "504"
|
|
response = Response(info)
|
|
content = ""
|
|
else:
|
|
(response, content) = self._request(
|
|
conn,
|
|
authority,
|
|
uri,
|
|
request_uri,
|
|
method,
|
|
body,
|
|
headers,
|
|
redirections,
|
|
cachekey,
|
|
)
|
|
except Exception as e:
|
|
is_timeout = isinstance(e, socket.timeout)
|
|
if is_timeout:
|
|
conn = self.connections.pop(conn_key, None)
|
|
if conn:
|
|
conn.close()
|
|
|
|
if self.force_exception_to_status_code:
|
|
if isinstance(e, HttpLib2ErrorWithResponse):
|
|
response = e.response
|
|
content = e.content
|
|
response.status = 500
|
|
response.reason = str(e)
|
|
elif is_timeout:
|
|
content = "Request Timeout"
|
|
response = Response(
|
|
{
|
|
"content-type": "text/plain",
|
|
"status": "408",
|
|
"content-length": len(content),
|
|
}
|
|
)
|
|
response.reason = "Request Timeout"
|
|
else:
|
|
content = str(e)
|
|
response = Response(
|
|
{
|
|
"content-type": "text/plain",
|
|
"status": "400",
|
|
"content-length": len(content),
|
|
}
|
|
)
|
|
response.reason = "Bad Request"
|
|
else:
|
|
raise
|
|
|
|
return (response, content)
|
|
|
|
def _get_proxy_info(self, scheme, authority):
|
|
"""Return a ProxyInfo instance (or None) based on the scheme
|
|
and authority.
|
|
"""
|
|
hostname, port = urllib.splitport(authority)
|
|
proxy_info = self.proxy_info
|
|
if callable(proxy_info):
|
|
proxy_info = proxy_info(scheme)
|
|
|
|
if hasattr(proxy_info, "applies_to") and not proxy_info.applies_to(hostname):
|
|
proxy_info = None
|
|
return proxy_info
|
|
|
|
|
|
class Response(dict):
|
|
"""An object more like email.Message than httplib.HTTPResponse."""
|
|
|
|
"""Is this response from our local cache"""
|
|
fromcache = False
|
|
"""HTTP protocol version used by server.
|
|
|
|
10 for HTTP/1.0, 11 for HTTP/1.1.
|
|
"""
|
|
version = 11
|
|
|
|
"Status code returned by server. "
|
|
status = 200
|
|
"""Reason phrase returned by server."""
|
|
reason = "Ok"
|
|
|
|
previous = None
|
|
|
|
def __init__(self, info):
|
|
# info is either an email.Message or
|
|
# an httplib.HTTPResponse object.
|
|
if isinstance(info, httplib.HTTPResponse):
|
|
for key, value in info.getheaders():
|
|
self[key.lower()] = value
|
|
self.status = info.status
|
|
self["status"] = str(self.status)
|
|
self.reason = info.reason
|
|
self.version = info.version
|
|
elif isinstance(info, email.Message.Message):
|
|
for key, value in info.items():
|
|
self[key.lower()] = value
|
|
self.status = int(self["status"])
|
|
else:
|
|
for key, value in info.iteritems():
|
|
self[key.lower()] = value
|
|
self.status = int(self.get("status", self.status))
|
|
self.reason = self.get("reason", self.reason)
|
|
|
|
def __getattr__(self, name):
|
|
if name == "dict":
|
|
return self
|
|
else:
|
|
raise AttributeError(name)
|