598 lines
21 KiB
Python
598 lines
21 KiB
Python
# urllib3/connectionpool.py
|
|
# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
|
#
|
|
# This module is part of urllib3 and is released under
|
|
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
|
|
|
import logging
|
|
import socket
|
|
import errno
|
|
|
|
from socket import error as SocketError, timeout as SocketTimeout
|
|
from .util import resolve_cert_reqs, resolve_ssl_version, assert_fingerprint
|
|
|
|
try: # Python 3
|
|
from http.client import HTTPConnection, HTTPException
|
|
from http.client import HTTP_PORT, HTTPS_PORT
|
|
except ImportError:
|
|
from httplib import HTTPConnection, HTTPException
|
|
from httplib import HTTP_PORT, HTTPS_PORT
|
|
|
|
try: # Python 3
|
|
from queue import LifoQueue, Empty, Full
|
|
except ImportError:
|
|
from Queue import LifoQueue, Empty, Full
|
|
|
|
|
|
try: # Compiled with SSL?
|
|
HTTPSConnection = object
|
|
BaseSSLError = None
|
|
ssl = None
|
|
|
|
try: # Python 3
|
|
from http.client import HTTPSConnection
|
|
except ImportError:
|
|
from httplib import HTTPSConnection
|
|
|
|
import ssl
|
|
BaseSSLError = ssl.SSLError
|
|
|
|
except (ImportError, AttributeError): # Platform-specific: No SSL.
|
|
pass
|
|
|
|
|
|
from .request import RequestMethods
|
|
from .response import HTTPResponse
|
|
from .util import get_host, is_connection_dropped, ssl_wrap_socket
|
|
from .exceptions import (
|
|
ClosedPoolError,
|
|
EmptyPoolError,
|
|
HostChangedError,
|
|
MaxRetryError,
|
|
SSLError,
|
|
TimeoutError,
|
|
)
|
|
|
|
from .packages.ssl_match_hostname import match_hostname, CertificateError
|
|
from .packages import six
|
|
|
|
|
|
xrange = six.moves.xrange
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
_Default = object()
|
|
|
|
port_by_scheme = {
|
|
'http': HTTP_PORT,
|
|
'https': HTTPS_PORT,
|
|
}
|
|
|
|
|
|
## Connection objects (extension of httplib)
|
|
|
|
class VerifiedHTTPSConnection(HTTPSConnection):
|
|
"""
|
|
Based on httplib.HTTPSConnection but wraps the socket with
|
|
SSL certification.
|
|
"""
|
|
cert_reqs = None
|
|
ca_certs = None
|
|
ssl_version = None
|
|
|
|
def set_cert(self, key_file=None, cert_file=None,
|
|
cert_reqs=None, ca_certs=None,
|
|
assert_hostname=None, assert_fingerprint=None):
|
|
|
|
self.key_file = key_file
|
|
self.cert_file = cert_file
|
|
self.cert_reqs = cert_reqs
|
|
self.ca_certs = ca_certs
|
|
self.assert_hostname = assert_hostname
|
|
self.assert_fingerprint = assert_fingerprint
|
|
|
|
def connect(self):
|
|
# Add certificate verification
|
|
sock = socket.create_connection((self.host, self.port), self.timeout)
|
|
|
|
resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs)
|
|
resolved_ssl_version = resolve_ssl_version(self.ssl_version)
|
|
|
|
# Wrap socket using verification with the root certs in
|
|
# trusted_root_certs
|
|
self.sock = ssl_wrap_socket(sock, self.key_file, self.cert_file,
|
|
cert_reqs=resolved_cert_reqs,
|
|
ca_certs=self.ca_certs,
|
|
server_hostname=self.host,
|
|
ssl_version=resolved_ssl_version)
|
|
|
|
if resolved_cert_reqs != ssl.CERT_NONE:
|
|
if self.assert_fingerprint:
|
|
assert_fingerprint(self.sock.getpeercert(binary_form=True),
|
|
self.assert_fingerprint)
|
|
else:
|
|
match_hostname(self.sock.getpeercert(),
|
|
self.assert_hostname or self.host)
|
|
|
|
## Pool objects
|
|
|
|
class ConnectionPool(object):
|
|
"""
|
|
Base class for all connection pools, such as
|
|
:class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.
|
|
"""
|
|
|
|
scheme = None
|
|
QueueCls = LifoQueue
|
|
|
|
def __init__(self, host, port=None):
|
|
self.host = host
|
|
self.port = port
|
|
|
|
def __str__(self):
|
|
return '%s(host=%r, port=%r)' % (type(self).__name__,
|
|
self.host, self.port)
|
|
|
|
|
|
class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|
"""
|
|
Thread-safe connection pool for one host.
|
|
|
|
:param host:
|
|
Host used for this HTTP Connection (e.g. "localhost"), passed into
|
|
:class:`httplib.HTTPConnection`.
|
|
|
|
:param port:
|
|
Port used for this HTTP Connection (None is equivalent to 80), passed
|
|
into :class:`httplib.HTTPConnection`.
|
|
|
|
:param strict:
|
|
Causes BadStatusLine to be raised if the status line can't be parsed
|
|
as a valid HTTP/1.0 or 1.1 status line, passed into
|
|
:class:`httplib.HTTPConnection`.
|
|
|
|
:param timeout:
|
|
Socket timeout for each individual connection, can be a float. None
|
|
disables timeout.
|
|
|
|
:param maxsize:
|
|
Number of connections to save that can be reused. More than 1 is useful
|
|
in multithreaded situations. If ``block`` is set to false, more
|
|
connections will be created but they will not be saved once they've
|
|
been used.
|
|
|
|
:param block:
|
|
If set to True, no more than ``maxsize`` connections will be used at
|
|
a time. When no free connections are available, the call will block
|
|
until a connection has been released. This is a useful side effect for
|
|
particular multithreaded situations where one does not want to use more
|
|
than maxsize connections per host to prevent flooding.
|
|
|
|
:param headers:
|
|
Headers to include with all requests, unless other headers are given
|
|
explicitly.
|
|
"""
|
|
|
|
scheme = 'http'
|
|
|
|
def __init__(self, host, port=None, strict=False, timeout=None, maxsize=1,
|
|
block=False, headers=None):
|
|
ConnectionPool.__init__(self, host, port)
|
|
RequestMethods.__init__(self, headers)
|
|
|
|
self.strict = strict
|
|
self.timeout = timeout
|
|
self.pool = self.QueueCls(maxsize)
|
|
self.block = block
|
|
|
|
# Fill the queue up so that doing get() on it will block properly
|
|
for _ in xrange(maxsize):
|
|
self.pool.put(None)
|
|
|
|
# These are mostly for testing and debugging purposes.
|
|
self.num_connections = 0
|
|
self.num_requests = 0
|
|
|
|
def _new_conn(self):
|
|
"""
|
|
Return a fresh :class:`httplib.HTTPConnection`.
|
|
"""
|
|
self.num_connections += 1
|
|
log.info("Starting new HTTP connection (%d): %s" %
|
|
(self.num_connections, self.host))
|
|
return HTTPConnection(host=self.host,
|
|
port=self.port,
|
|
strict=self.strict)
|
|
|
|
def _get_conn(self, timeout=None):
|
|
"""
|
|
Get a connection. Will return a pooled connection if one is available.
|
|
|
|
If no connections are available and :prop:`.block` is ``False``, then a
|
|
fresh connection is returned.
|
|
|
|
:param timeout:
|
|
Seconds to wait before giving up and raising
|
|
:class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and
|
|
:prop:`.block` is ``True``.
|
|
"""
|
|
conn = None
|
|
try:
|
|
conn = self.pool.get(block=self.block, timeout=timeout)
|
|
|
|
except AttributeError: # self.pool is None
|
|
raise ClosedPoolError(self, "Pool is closed.")
|
|
|
|
except Empty:
|
|
if self.block:
|
|
raise EmptyPoolError(self,
|
|
"Pool reached maximum size and no more "
|
|
"connections are allowed.")
|
|
pass # Oh well, we'll create a new connection then
|
|
|
|
# If this is a persistent connection, check if it got disconnected
|
|
if conn and is_connection_dropped(conn):
|
|
log.info("Resetting dropped connection: %s" % self.host)
|
|
conn.close()
|
|
|
|
return conn or self._new_conn()
|
|
|
|
def _put_conn(self, conn):
|
|
"""
|
|
Put a connection back into the pool.
|
|
|
|
:param conn:
|
|
Connection object for the current host and port as returned by
|
|
:meth:`._new_conn` or :meth:`._get_conn`.
|
|
|
|
If the pool is already full, the connection is closed and discarded
|
|
because we exceeded maxsize. If connections are discarded frequently,
|
|
then maxsize should be increased.
|
|
|
|
If the pool is closed, then the connection will be closed and discarded.
|
|
"""
|
|
try:
|
|
self.pool.put(conn, block=False)
|
|
return # Everything is dandy, done.
|
|
except AttributeError:
|
|
# self.pool is None.
|
|
pass
|
|
except Full:
|
|
# This should never happen if self.block == True
|
|
log.warning("HttpConnectionPool is full, discarding connection: %s"
|
|
% self.host)
|
|
|
|
# Connection never got put back into the pool, close it.
|
|
conn.close()
|
|
|
|
def _make_request(self, conn, method, url, timeout=_Default,
|
|
**httplib_request_kw):
|
|
"""
|
|
Perform a request on a given httplib connection object taken from our
|
|
pool.
|
|
"""
|
|
self.num_requests += 1
|
|
|
|
if timeout is _Default:
|
|
timeout = self.timeout
|
|
|
|
conn.timeout = timeout # This only does anything in Py26+
|
|
conn.request(method, url, **httplib_request_kw)
|
|
|
|
# Set timeout
|
|
sock = getattr(conn, 'sock', False) # AppEngine doesn't have sock attr.
|
|
if sock:
|
|
sock.settimeout(timeout)
|
|
|
|
try: # Python 2.7+, use buffering of HTTP responses
|
|
httplib_response = conn.getresponse(buffering=True)
|
|
except TypeError: # Python 2.6 and older
|
|
httplib_response = conn.getresponse()
|
|
|
|
# AppEngine doesn't have a version attr.
|
|
http_version = getattr(conn, '_http_vsn_str', 'HTTP/?')
|
|
log.debug("\"%s %s %s\" %s %s" % (method, url, http_version,
|
|
httplib_response.status,
|
|
httplib_response.length))
|
|
return httplib_response
|
|
|
|
def close(self):
|
|
"""
|
|
Close all pooled connections and disable the pool.
|
|
"""
|
|
# Disable access to the pool
|
|
old_pool, self.pool = self.pool, None
|
|
|
|
try:
|
|
while True:
|
|
conn = old_pool.get(block=False)
|
|
if conn:
|
|
conn.close()
|
|
|
|
except Empty:
|
|
pass # Done.
|
|
|
|
def is_same_host(self, url):
|
|
"""
|
|
Check if the given ``url`` is a member of the same host as this
|
|
connection pool.
|
|
"""
|
|
if url.startswith('/'):
|
|
return True
|
|
|
|
# TODO: Add optional support for socket.gethostbyname checking.
|
|
scheme, host, port = get_host(url)
|
|
|
|
if self.port and not port:
|
|
# Use explicit default port for comparison when none is given.
|
|
port = port_by_scheme.get(scheme)
|
|
|
|
return (scheme, host, port) == (self.scheme, self.host, self.port)
|
|
|
|
def urlopen(self, method, url, body=None, headers=None, retries=3,
|
|
redirect=True, assert_same_host=True, timeout=_Default,
|
|
pool_timeout=None, release_conn=None, **response_kw):
|
|
"""
|
|
Get a connection from the pool and perform an HTTP request. This is the
|
|
lowest level call for making a request, so you'll need to specify all
|
|
the raw details.
|
|
|
|
.. note::
|
|
|
|
More commonly, it's appropriate to use a convenience method provided
|
|
by :class:`.RequestMethods`, such as :meth:`request`.
|
|
|
|
.. note::
|
|
|
|
`release_conn` will only behave as expected if
|
|
`preload_content=False` because we want to make
|
|
`preload_content=False` the default behaviour someday soon without
|
|
breaking backwards compatibility.
|
|
|
|
:param method:
|
|
HTTP request method (such as GET, POST, PUT, etc.)
|
|
|
|
:param body:
|
|
Data to send in the request body (useful for creating
|
|
POST requests, see HTTPConnectionPool.post_url for
|
|
more convenience).
|
|
|
|
:param headers:
|
|
Dictionary of custom headers to send, such as User-Agent,
|
|
If-None-Match, etc. If None, pool headers are used. If provided,
|
|
these headers completely replace any pool-specific headers.
|
|
|
|
:param retries:
|
|
Number of retries to allow before raising a MaxRetryError exception.
|
|
|
|
:param redirect:
|
|
If True, automatically handle redirects (status codes 301, 302,
|
|
303, 307). Each redirect counts as a retry.
|
|
|
|
:param assert_same_host:
|
|
If ``True``, will make sure that the host of the pool requests is
|
|
consistent else will raise HostChangedError. When False, you can
|
|
use the pool on an HTTP proxy and request foreign hosts.
|
|
|
|
:param timeout:
|
|
If specified, overrides the default timeout for this one request.
|
|
|
|
:param pool_timeout:
|
|
If set and the pool is set to block=True, then this method will
|
|
block for ``pool_timeout`` seconds and raise EmptyPoolError if no
|
|
connection is available within the time period.
|
|
|
|
:param release_conn:
|
|
If False, then the urlopen call will not release the connection
|
|
back into the pool once a response is received (but will release if
|
|
you read the entire contents of the response such as when
|
|
`preload_content=True`). This is useful if you're not preloading
|
|
the response's content immediately. You will need to call
|
|
``r.release_conn()`` on the response ``r`` to return the connection
|
|
back into the pool. If None, it takes the value of
|
|
``response_kw.get('preload_content', True)``.
|
|
|
|
:param \**response_kw:
|
|
Additional parameters are passed to
|
|
:meth:`urllib3.response.HTTPResponse.from_httplib`
|
|
"""
|
|
if headers is None:
|
|
headers = self.headers
|
|
|
|
if retries < 0:
|
|
raise MaxRetryError(self, url)
|
|
|
|
if timeout is _Default:
|
|
timeout = self.timeout
|
|
|
|
if release_conn is None:
|
|
release_conn = response_kw.get('preload_content', True)
|
|
|
|
# Check host
|
|
if assert_same_host and not self.is_same_host(url):
|
|
host = "%s://%s" % (self.scheme, self.host)
|
|
if self.port:
|
|
host = "%s:%d" % (host, self.port)
|
|
|
|
raise HostChangedError(self, url, retries - 1)
|
|
|
|
conn = None
|
|
|
|
try:
|
|
# Request a connection from the queue
|
|
conn = self._get_conn(timeout=pool_timeout)
|
|
|
|
# Make the request on the httplib connection object
|
|
httplib_response = self._make_request(conn, method, url,
|
|
timeout=timeout,
|
|
body=body, headers=headers)
|
|
|
|
# If we're going to release the connection in ``finally:``, then
|
|
# the request doesn't need to know about the connection. Otherwise
|
|
# it will also try to release it and we'll have a double-release
|
|
# mess.
|
|
response_conn = not release_conn and conn
|
|
|
|
# Import httplib's response into our own wrapper object
|
|
response = HTTPResponse.from_httplib(httplib_response,
|
|
pool=self,
|
|
connection=response_conn,
|
|
**response_kw)
|
|
|
|
# else:
|
|
# The connection will be put back into the pool when
|
|
# ``response.release_conn()`` is called (implicitly by
|
|
# ``response.read()``)
|
|
|
|
except Empty as e:
|
|
# Timed out by queue
|
|
raise TimeoutError(self, url,
|
|
"Request timed out. (pool_timeout=%s)" %
|
|
pool_timeout)
|
|
|
|
except SocketTimeout as e:
|
|
# Timed out by socket
|
|
raise TimeoutError(self, url,
|
|
"Request timed out. (timeout=%s)" %
|
|
timeout)
|
|
|
|
except BaseSSLError as e:
|
|
# SSL certificate error
|
|
raise SSLError(e)
|
|
|
|
except CertificateError as e:
|
|
# Name mismatch
|
|
raise SSLError(e)
|
|
|
|
except (HTTPException, SocketError) as e:
|
|
# Connection broken, discard. It will be replaced next _get_conn().
|
|
conn = None
|
|
# This is necessary so we can access e below
|
|
err = e
|
|
|
|
if retries == 0:
|
|
raise MaxRetryError(self, url, e)
|
|
|
|
finally:
|
|
if release_conn:
|
|
# Put the connection back to be reused. If the connection is
|
|
# expired then it will be None, which will get replaced with a
|
|
# fresh connection during _get_conn.
|
|
self._put_conn(conn)
|
|
|
|
if not conn:
|
|
# Try again
|
|
log.warn("Retrying (%d attempts remain) after connection "
|
|
"broken by '%r': %s" % (retries, err, url))
|
|
return self.urlopen(method, url, body, headers, retries - 1,
|
|
redirect, assert_same_host,
|
|
timeout=timeout, pool_timeout=pool_timeout,
|
|
release_conn=release_conn, **response_kw)
|
|
|
|
# Handle redirect?
|
|
redirect_location = redirect and response.get_redirect_location()
|
|
if redirect_location:
|
|
if response.status == 303:
|
|
method = 'GET'
|
|
log.info("Redirecting %s -> %s" % (url, redirect_location))
|
|
return self.urlopen(method, redirect_location, body, headers,
|
|
retries - 1, redirect, assert_same_host,
|
|
timeout=timeout, pool_timeout=pool_timeout,
|
|
release_conn=release_conn, **response_kw)
|
|
|
|
return response
|
|
|
|
|
|
class HTTPSConnectionPool(HTTPConnectionPool):
|
|
"""
|
|
Same as :class:`.HTTPConnectionPool`, but HTTPS.
|
|
|
|
When Python is compiled with the :mod:`ssl` module, then
|
|
:class:`.VerifiedHTTPSConnection` is used, which *can* verify certificates,
|
|
instead of :class:`httplib.HTTPSConnection`.
|
|
|
|
:class:`.VerifiedHTTPSConnection` uses one of ``assert_fingerprint``,
|
|
``assert_hostname`` and ``host`` in this order to verify connections.
|
|
|
|
The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs`` and
|
|
``ssl_version`` are only used if :mod:`ssl` is available and are fed into
|
|
:meth:`urllib3.util.ssl_wrap_socket` to upgrade the connection socket
|
|
into an SSL socket.
|
|
"""
|
|
|
|
scheme = 'https'
|
|
|
|
def __init__(self, host, port=None,
|
|
strict=False, timeout=None, maxsize=1,
|
|
block=False, headers=None,
|
|
key_file=None, cert_file=None, cert_reqs=None,
|
|
ca_certs=None, ssl_version=None,
|
|
assert_hostname=None, assert_fingerprint=None):
|
|
|
|
HTTPConnectionPool.__init__(self, host, port,
|
|
strict, timeout, maxsize,
|
|
block, headers)
|
|
self.key_file = key_file
|
|
self.cert_file = cert_file
|
|
self.cert_reqs = cert_reqs
|
|
self.ca_certs = ca_certs
|
|
self.ssl_version = ssl_version
|
|
self.assert_hostname = assert_hostname
|
|
self.assert_fingerprint = assert_fingerprint
|
|
|
|
def _new_conn(self):
|
|
"""
|
|
Return a fresh :class:`httplib.HTTPSConnection`.
|
|
"""
|
|
self.num_connections += 1
|
|
log.info("Starting new HTTPS connection (%d): %s"
|
|
% (self.num_connections, self.host))
|
|
|
|
if not ssl: # Platform-specific: Python compiled without +ssl
|
|
if not HTTPSConnection or HTTPSConnection is object:
|
|
raise SSLError("Can't connect to HTTPS URL because the SSL "
|
|
"module is not available.")
|
|
|
|
return HTTPSConnection(host=self.host,
|
|
port=self.port,
|
|
strict=self.strict)
|
|
|
|
connection = VerifiedHTTPSConnection(host=self.host,
|
|
port=self.port,
|
|
strict=self.strict)
|
|
connection.set_cert(key_file=self.key_file, cert_file=self.cert_file,
|
|
cert_reqs=self.cert_reqs, ca_certs=self.ca_certs,
|
|
assert_hostname=self.assert_hostname,
|
|
assert_fingerprint=self.assert_fingerprint)
|
|
|
|
connection.ssl_version = self.ssl_version
|
|
|
|
return connection
|
|
|
|
|
|
def connection_from_url(url, **kw):
|
|
"""
|
|
Given a url, return an :class:`.ConnectionPool` instance of its host.
|
|
|
|
This is a shortcut for not having to parse out the scheme, host, and port
|
|
of the url before creating an :class:`.ConnectionPool` instance.
|
|
|
|
:param url:
|
|
Absolute URL string that must include the scheme. Port is optional.
|
|
|
|
:param \**kw:
|
|
Passes additional parameters to the constructor of the appropriate
|
|
:class:`.ConnectionPool`. Useful for specifying things like
|
|
timeout, maxsize, headers, etc.
|
|
|
|
Example: ::
|
|
|
|
>>> conn = connection_from_url('http://google.com/')
|
|
>>> r = conn.request('GET', '/')
|
|
"""
|
|
scheme, host, port = get_host(url)
|
|
if scheme == 'https':
|
|
return HTTPSConnectionPool(host, port=port, **kw)
|
|
else:
|
|
return HTTPConnectionPool(host, port=port, **kw)
|