upgrade wakatime-cli to latest master version

This commit is contained in:
Alan Hamlett 2016-07-06 17:11:03 +02:00
parent 2701f8a381
commit df5bb015ff
13 changed files with 262 additions and 67 deletions

View File

@ -73,10 +73,9 @@ class JsonFormatter(logging.Formatter):
def traceback_formatter(*args, **kwargs): def traceback_formatter(*args, **kwargs):
try: level = kwargs.get('level', args[0] if len(args) else None)
level = kwargs.get('level', args[0]).lower() if level:
except: level = level.lower()
level = None
if level == 'warn' or level == 'warning': if level == 'warn' or level == 'warning':
logging.getLogger('WakaTime').warning(traceback.format_exc()) logging.getLogger('WakaTime').warning(traceback.format_exc())
elif level == 'info': elif level == 'info':

View File

@ -185,6 +185,9 @@ def parseArguments():
return args, configs return args, configs
# update args from configs # update args from configs
if not args.hostname:
if configs.has_option('settings', 'hostname'):
args.hostname = configs.get('settings', 'hostname')
if not args.key: if not args.key:
default_key = None default_key = None
if configs.has_option('settings', 'api_key'): if configs.has_option('settings', 'api_key'):

View File

@ -44,8 +44,9 @@ from .packages.ssl_match_hostname import match_hostname, CertificateError
from .util.ssl_ import ( from .util.ssl_ import (
resolve_cert_reqs, resolve_cert_reqs,
resolve_ssl_version, resolve_ssl_version,
ssl_wrap_socket,
assert_fingerprint, assert_fingerprint,
create_urllib3_context,
ssl_wrap_socket
) )
@ -203,14 +204,18 @@ class HTTPConnection(_HTTPConnection, object):
class HTTPSConnection(HTTPConnection): class HTTPSConnection(HTTPConnection):
default_port = port_by_scheme['https'] default_port = port_by_scheme['https']
ssl_version = None
def __init__(self, host, port=None, key_file=None, cert_file=None, def __init__(self, host, port=None, key_file=None, cert_file=None,
strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, **kw): strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
ssl_context=None, **kw):
HTTPConnection.__init__(self, host, port, strict=strict, HTTPConnection.__init__(self, host, port, strict=strict,
timeout=timeout, **kw) timeout=timeout, **kw)
self.key_file = key_file self.key_file = key_file
self.cert_file = cert_file self.cert_file = cert_file
self.ssl_context = ssl_context
# Required property for Google AppEngine 1.9.0 which otherwise causes # Required property for Google AppEngine 1.9.0 which otherwise causes
# HTTPS requests to go out as HTTP. (See Issue #356) # HTTPS requests to go out as HTTP. (See Issue #356)
@ -219,7 +224,19 @@ class HTTPSConnection(HTTPConnection):
def connect(self): def connect(self):
conn = self._new_conn() conn = self._new_conn()
self._prepare_conn(conn) self._prepare_conn(conn)
self.sock = ssl.wrap_socket(conn, self.key_file, self.cert_file)
if self.ssl_context is None:
self.ssl_context = create_urllib3_context(
ssl_version=resolve_ssl_version(None),
cert_reqs=resolve_cert_reqs(None),
)
self.sock = ssl_wrap_socket(
sock=conn,
keyfile=self.key_file,
certfile=self.cert_file,
ssl_context=self.ssl_context,
)
class VerifiedHTTPSConnection(HTTPSConnection): class VerifiedHTTPSConnection(HTTPSConnection):
@ -237,9 +254,18 @@ class VerifiedHTTPSConnection(HTTPSConnection):
cert_reqs=None, ca_certs=None, cert_reqs=None, ca_certs=None,
assert_hostname=None, assert_fingerprint=None, assert_hostname=None, assert_fingerprint=None,
ca_cert_dir=None): ca_cert_dir=None):
"""
if (ca_certs or ca_cert_dir) and cert_reqs is None: This method should only be called once, before the connection is used.
cert_reqs = 'CERT_REQUIRED' """
# If cert_reqs is not provided, we can try to guess. If the user gave
# us a cert database, we assume they want to use it: otherwise, if
# they gave us an SSL Context object we should use whatever is set for
# it.
if cert_reqs is None:
if ca_certs or ca_cert_dir:
cert_reqs = 'CERT_REQUIRED'
elif self.ssl_context is not None:
cert_reqs = self.ssl_context.verify_mode
self.key_file = key_file self.key_file = key_file
self.cert_file = cert_file self.cert_file = cert_file
@ -253,9 +279,6 @@ class VerifiedHTTPSConnection(HTTPSConnection):
# Add certificate verification # Add certificate verification
conn = self._new_conn() conn = self._new_conn()
resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs)
resolved_ssl_version = resolve_ssl_version(self.ssl_version)
hostname = self.host hostname = self.host
if getattr(self, '_tunnel_host', None): if getattr(self, '_tunnel_host', None):
# _tunnel_host was added in Python 2.6.3 # _tunnel_host was added in Python 2.6.3
@ -281,17 +304,27 @@ class VerifiedHTTPSConnection(HTTPSConnection):
# Wrap socket using verification with the root certs in # Wrap socket using verification with the root certs in
# trusted_root_certs # trusted_root_certs
self.sock = ssl_wrap_socket(conn, self.key_file, self.cert_file, if self.ssl_context is None:
cert_reqs=resolved_cert_reqs, self.ssl_context = create_urllib3_context(
ca_certs=self.ca_certs, ssl_version=resolve_ssl_version(self.ssl_version),
ca_cert_dir=self.ca_cert_dir, cert_reqs=resolve_cert_reqs(self.cert_reqs),
server_hostname=hostname, )
ssl_version=resolved_ssl_version)
context = self.ssl_context
context.verify_mode = resolve_cert_reqs(self.cert_reqs)
self.sock = ssl_wrap_socket(
sock=conn,
keyfile=self.key_file,
certfile=self.cert_file,
ca_certs=self.ca_certs,
ca_cert_dir=self.ca_cert_dir,
server_hostname=hostname,
ssl_context=context)
if self.assert_fingerprint: if self.assert_fingerprint:
assert_fingerprint(self.sock.getpeercert(binary_form=True), assert_fingerprint(self.sock.getpeercert(binary_form=True),
self.assert_fingerprint) self.assert_fingerprint)
elif resolved_cert_reqs != ssl.CERT_NONE \ elif context.verify_mode != ssl.CERT_NONE \
and self.assert_hostname is not False: and self.assert_hostname is not False:
cert = self.sock.getpeercert() cert = self.sock.getpeercert()
if not cert.get('subjectAltName', ()): if not cert.get('subjectAltName', ()):
@ -304,8 +337,10 @@ class VerifiedHTTPSConnection(HTTPSConnection):
) )
_match_hostname(cert, self.assert_hostname or hostname) _match_hostname(cert, self.assert_hostname or hostname)
self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED or self.is_verified = (
self.assert_fingerprint is not None) context.verify_mode == ssl.CERT_REQUIRED or
self.assert_fingerprint is not None
)
def _match_hostname(cert, asserted_hostname): def _match_hostname(cert, asserted_hostname):

View File

@ -397,8 +397,9 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
# AppEngine doesn't have a version attr. # AppEngine doesn't have a version attr.
http_version = getattr(conn, '_http_vsn_str', 'HTTP/?') http_version = getattr(conn, '_http_vsn_str', 'HTTP/?')
log.debug("\"%s %s %s\" %s %s", method, url, http_version, log.debug("%s://%s:%s \"%s %s %s\" %s %s", self.scheme, self.host, self.port,
httplib_response.status, httplib_response.length) method, url, http_version, httplib_response.status,
httplib_response.length)
try: try:
assert_header_parsing(httplib_response.msg) assert_header_parsing(httplib_response.msg)
@ -604,6 +605,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
response = self.ResponseCls.from_httplib(httplib_response, response = self.ResponseCls.from_httplib(httplib_response,
pool=self, pool=self,
connection=response_conn, connection=response_conn,
retries=retries,
**response_kw) **response_kw)
# Everything went great! # Everything went great!
@ -775,7 +777,6 @@ class HTTPSConnectionPool(HTTPConnectionPool):
assert_hostname=self.assert_hostname, assert_hostname=self.assert_hostname,
assert_fingerprint=self.assert_fingerprint) assert_fingerprint=self.assert_fingerprint)
conn.ssl_version = self.ssl_version conn.ssl_version = self.ssl_version
return conn return conn
def _prepare_proxy(self, conn): def _prepare_proxy(self, conn):

View File

@ -1,3 +1,43 @@
"""
This module provides a pool manager that uses Google App Engine's
`URLFetch Service <https://cloud.google.com/appengine/docs/python/urlfetch>`_.
Example usage::
from urllib3 import PoolManager
from urllib3.contrib.appengine import AppEngineManager, is_appengine_sandbox
if is_appengine_sandbox():
# AppEngineManager uses AppEngine's URLFetch API behind the scenes
http = AppEngineManager()
else:
# PoolManager uses a socket-level API behind the scenes
http = PoolManager()
r = http.request('GET', 'https://google.com/')
There are `limitations <https://cloud.google.com/appengine/docs/python/\
urlfetch/#Python_Quotas_and_limits>`_ to the URLFetch service and it may not be
the best choice for your application. There are three options for using
urllib3 on Google App Engine:
1. You can use :class:`AppEngineManager` with URLFetch. URLFetch is
cost-effective in many circumstances as long as your usage is within the
limitations.
2. You can use a normal :class:`~urllib3.PoolManager` by enabling sockets.
Sockets also have `limitations and restrictions
<https://cloud.google.com/appengine/docs/python/sockets/\
#limitations-and-restrictions>`_ and have a lower free quota than URLFetch.
To use sockets, be sure to specify the following in your ``app.yaml``::
env_variables:
GAE_USE_SOCKETS_HTTPLIB : 'true'
3. If you are using `App Engine Flexible
<https://cloud.google.com/appengine/docs/flexible/>`_, you can use the standard
:class:`PoolManager` without any configuration or special environment variables.
"""
from __future__ import absolute_import from __future__ import absolute_import
import logging import logging
import os import os
@ -41,13 +81,12 @@ class AppEngineManager(RequestMethods):
This manager uses the URLFetch service directly instead of using the This manager uses the URLFetch service directly instead of using the
emulated httplib, and is subject to URLFetch limitations as described in emulated httplib, and is subject to URLFetch limitations as described in
the App Engine documentation here: the App Engine documentation `here
<https://cloud.google.com/appengine/docs/python/urlfetch>`_.
https://cloud.google.com/appengine/docs/python/urlfetch Notably it will raise an :class:`AppEnginePlatformError` if:
Notably it will raise an AppEnginePlatformError if:
* URLFetch is not available. * URLFetch is not available.
* If you attempt to use this on GAEv2 (Managed VMs), as full socket * If you attempt to use this on App Engine Flexible, as full socket
support is available. support is available.
* If a request size is more than 10 megabytes. * If a request size is more than 10 megabytes.
* If a response size is more than 32 megabtyes. * If a response size is more than 32 megabtyes.
@ -133,7 +172,7 @@ class AppEngineManager(RequestMethods):
"URLFetch does not support method: %s" % method, e) "URLFetch does not support method: %s" % method, e)
http_response = self._urlfetch_response_to_http_response( http_response = self._urlfetch_response_to_http_response(
response, **response_kw) response, retries=retries, **response_kw)
# Check for redirect response # Check for redirect response
if (http_response.get_redirect_location() and if (http_response.get_redirect_location() and
@ -183,12 +222,13 @@ class AppEngineManager(RequestMethods):
def _get_absolute_timeout(self, timeout): def _get_absolute_timeout(self, timeout):
if timeout is Timeout.DEFAULT_TIMEOUT: if timeout is Timeout.DEFAULT_TIMEOUT:
return 5 # 5s is the default timeout for URLFetch. return None # Defer to URLFetch's default.
if isinstance(timeout, Timeout): if isinstance(timeout, Timeout):
if timeout._read is not timeout._connect: if timeout._read is not None or timeout._connect is not None:
warnings.warn( warnings.warn(
"URLFetch does not support granular timeout settings, " "URLFetch does not support granular timeout settings, "
"reverting to total timeout.", AppEnginePlatformWarning) "reverting to total or default URLFetch timeout.",
AppEnginePlatformWarning)
return timeout.total return timeout.total
return timeout return timeout

View File

@ -12,7 +12,7 @@ except ImportError:
from logging import getLogger from logging import getLogger
from ntlm import ntlm from ntlm import ntlm
from urllib3 import HTTPSConnectionPool from .. import HTTPSConnectionPool
log = getLogger(__name__) log = getLogger(__name__)

View File

@ -1,4 +1,5 @@
'''SSL with SNI_-support for Python 2. Follow these instructions if you would """
SSL with SNI_-support for Python 2. Follow these instructions if you would
like to verify SSL certificates in Python 2. Note, the default libraries do like to verify SSL certificates in Python 2. Note, the default libraries do
*not* do certificate checking; you need to do additional work to validate *not* do certificate checking; you need to do additional work to validate
certificates yourself. certificates yourself.
@ -34,15 +35,9 @@ compression in Python 2 (see `CRIME attack`_).
If you want to configure the default list of supported cipher suites, you can If you want to configure the default list of supported cipher suites, you can
set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable. set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable.
Module Variables
----------------
:var DEFAULT_SSL_CIPHER_LIST: The list of supported SSL/TLS cipher suites.
.. _sni: https://en.wikipedia.org/wiki/Server_Name_Indication .. _sni: https://en.wikipedia.org/wiki/Server_Name_Indication
.. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit) .. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit)
"""
'''
from __future__ import absolute_import from __future__ import absolute_import
try: try:
@ -55,18 +50,18 @@ import OpenSSL.SSL
from pyasn1.codec.der import decoder as der_decoder from pyasn1.codec.der import decoder as der_decoder
from pyasn1.type import univ, constraint from pyasn1.type import univ, constraint
from socket import timeout, error as SocketError from socket import timeout, error as SocketError
from io import BytesIO
try: # Platform-specific: Python 2 try: # Platform-specific: Python 2
from socket import _fileobject from socket import _fileobject
except ImportError: # Platform-specific: Python 3 except ImportError: # Platform-specific: Python 3
_fileobject = None _fileobject = None
from urllib3.packages.backports.makefile import backport_makefile from ..packages.backports.makefile import backport_makefile
import ssl import ssl
import select import select
import six import six
from .. import connection
from .. import util from .. import util
__all__ = ['inject_into_urllib3', 'extract_from_urllib3'] __all__ = ['inject_into_urllib3', 'extract_from_urllib3']
@ -98,29 +93,34 @@ _openssl_verify = {
OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
} }
#: The list of supported SSL/TLS cipher suites.
DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS.encode('ascii') DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS.encode('ascii')
# OpenSSL will only write 16K at a time # OpenSSL will only write 16K at a time
SSL_WRITE_BLOCKSIZE = 16384 SSL_WRITE_BLOCKSIZE = 16384
orig_util_HAS_SNI = util.HAS_SNI orig_util_HAS_SNI = util.HAS_SNI
orig_connection_ssl_wrap_socket = connection.ssl_wrap_socket orig_util_SSLContext = util.ssl_.SSLContext
def inject_into_urllib3(): def inject_into_urllib3():
'Monkey-patch urllib3 with PyOpenSSL-backed SSL-support.' 'Monkey-patch urllib3 with PyOpenSSL-backed SSL-support.'
connection.ssl_wrap_socket = ssl_wrap_socket util.ssl_.SSLContext = PyOpenSSLContext
util.HAS_SNI = HAS_SNI util.HAS_SNI = HAS_SNI
util.ssl_.HAS_SNI = HAS_SNI
util.IS_PYOPENSSL = True util.IS_PYOPENSSL = True
util.ssl_.IS_PYOPENSSL = True
def extract_from_urllib3(): def extract_from_urllib3():
'Undo monkey-patching by :func:`inject_into_urllib3`.' 'Undo monkey-patching by :func:`inject_into_urllib3`.'
connection.ssl_wrap_socket = orig_connection_ssl_wrap_socket util.ssl_.SSLContext = orig_util_SSLContext
util.HAS_SNI = orig_util_HAS_SNI util.HAS_SNI = orig_util_HAS_SNI
util.ssl_.HAS_SNI = orig_util_HAS_SNI
util.IS_PYOPENSSL = False util.IS_PYOPENSSL = False
util.ssl_.IS_PYOPENSSL = False
# Note: This is a slightly bug-fixed version of same from ndg-httpsclient. # Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
@ -308,6 +308,86 @@ else: # Platform-specific: Python 3
WrappedSocket.makefile = makefile WrappedSocket.makefile = makefile
class PyOpenSSLContext(object):
"""
I am a wrapper class for the PyOpenSSL ``Context`` object. I am responsible
for translating the interface of the standard library ``SSLContext`` object
to calls into PyOpenSSL.
"""
def __init__(self, protocol):
self.protocol = _openssl_versions[protocol]
self._ctx = OpenSSL.SSL.Context(self.protocol)
self._options = 0
self.check_hostname = False
@property
def options(self):
return self._options
@options.setter
def options(self, value):
self._options = value
self._ctx.set_options(value)
@property
def verify_mode(self):
return self._ctx.get_verify_mode()
@verify_mode.setter
def verify_mode(self, value):
self._ctx.set_verify(value, _verify_callback)
def set_default_verify_paths(self):
self._ctx.set_default_verify_paths()
def set_ciphers(self, ciphers):
if isinstance(ciphers, six.text_type):
ciphers = ciphers.encode('utf-8')
self._ctx.set_cipher_list(ciphers)
def load_verify_locations(self, cafile=None, capath=None, cadata=None):
if cafile is not None:
cafile = cafile.encode('utf-8')
if capath is not None:
capath = capath.encode('utf-8')
self._ctx.load_verify_locations(cafile, capath)
if cadata is not None:
self._ctx.load_verify_locations(BytesIO(cadata))
def load_cert_chain(self, certfile, keyfile=None, password=None):
self._ctx.use_certificate_file(certfile)
if password is not None:
self._ctx.set_passwd_cb(lambda max_length, prompt_twice, userdata: password)
self._ctx.use_privatekey_file(keyfile or certfile)
def wrap_socket(self, sock, server_side=False,
do_handshake_on_connect=True, suppress_ragged_eofs=True,
server_hostname=None):
cnx = OpenSSL.SSL.Connection(self._ctx, sock)
if isinstance(server_hostname, six.text_type): # Platform-specific: Python 3
server_hostname = server_hostname.encode('utf-8')
if server_hostname is not None:
cnx.set_tlsext_host_name(server_hostname)
cnx.set_connect_state()
while True:
try:
cnx.do_handshake()
except OpenSSL.SSL.WantReadError:
rd, _, _ = select.select([sock], [], [], sock.gettimeout())
if not rd:
raise timeout('select timed out')
continue
except OpenSSL.SSL.Error as e:
raise ssl.SSLError('bad handshake: %r' % e)
break
return WrappedSocket(cnx, sock)
def _verify_callback(cnx, x509, err_no, err_depth, return_code): def _verify_callback(cnx, x509, err_no, err_depth, return_code):
return err_no == 0 return err_no == 0
@ -315,6 +395,8 @@ def _verify_callback(cnx, x509, err_no, err_depth, return_code):
def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
ca_certs=None, server_hostname=None, ca_certs=None, server_hostname=None,
ssl_version=None, ca_cert_dir=None): ssl_version=None, ca_cert_dir=None):
# This function is no longer used by urllib3. We should strongly consider
# removing it.
ctx = OpenSSL.SSL.Context(_openssl_versions[ssl_version]) ctx = OpenSSL.SSL.Context(_openssl_versions[ssl_version])
if certfile: if certfile:
keyfile = keyfile or certfile # Match behaviour of the normal python ssl library keyfile = keyfile or certfile # Match behaviour of the normal python ssl library

View File

@ -1,17 +1,23 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
""" """
SOCKS support for urllib3 This module contains provisional support for SOCKS proxies from within
~~~~~~~~~~~~~~~~~~~~~~~~~
This contrib module contains provisional support for SOCKS proxies from within
urllib3. This module supports SOCKS4 (specifically the SOCKS4A variant) and urllib3. This module supports SOCKS4 (specifically the SOCKS4A variant) and
SOCKS5. To enable its functionality, either install PySocks or install this SOCKS5. To enable its functionality, either install PySocks or install this
module with the ``socks`` extra. module with the ``socks`` extra.
The SOCKS implementation supports the full range of urllib3 features. It also
supports the following SOCKS features:
- SOCKS4
- SOCKS4a
- SOCKS5
- Usernames and passwords for the SOCKS proxy
Known Limitations: Known Limitations:
- Currently PySocks does not support contacting remote websites via literal - Currently PySocks does not support contacting remote websites via literal
IPv6 addresses. Any such connection attempt will fail. IPv6 addresses. Any such connection attempt will fail. You must use a domain
name.
- Currently PySocks does not support IPv6 connections to the SOCKS proxy. Any - Currently PySocks does not support IPv6 connections to the SOCKS proxy. Any
such connection attempt will fail. such connection attempt will fail.
""" """

View File

@ -23,7 +23,7 @@ __all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url']
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs', SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs',
'ssl_version', 'ca_cert_dir') 'ssl_version', 'ca_cert_dir', 'ssl_context')
# The base fields to use when determining what pool to get a connection from; # The base fields to use when determining what pool to get a connection from;
# these do not rely on the ``connection_pool_kw`` and can be determined by the # these do not rely on the ``connection_pool_kw`` and can be determined by the

View File

@ -89,6 +89,10 @@ class HTTPResponse(io.IOBase):
When this HTTPResponse wrapper is generated from an httplib.HTTPResponse When this HTTPResponse wrapper is generated from an httplib.HTTPResponse
object, it's convenient to include the original for debug purposes. It's object, it's convenient to include the original for debug purposes. It's
otherwise unused. otherwise unused.
:param retries:
The retries contains the last :class:`~urllib3.util.retry.Retry` that
was used during the request.
""" """
CONTENT_DECODERS = ['gzip', 'deflate'] CONTENT_DECODERS = ['gzip', 'deflate']
@ -96,7 +100,7 @@ class HTTPResponse(io.IOBase):
def __init__(self, body='', headers=None, status=0, version=0, reason=None, def __init__(self, body='', headers=None, status=0, version=0, reason=None,
strict=0, preload_content=True, decode_content=True, strict=0, preload_content=True, decode_content=True,
original_response=None, pool=None, connection=None): original_response=None, pool=None, connection=None, retries=None):
if isinstance(headers, HTTPHeaderDict): if isinstance(headers, HTTPHeaderDict):
self.headers = headers self.headers = headers
@ -107,6 +111,7 @@ class HTTPResponse(io.IOBase):
self.reason = reason self.reason = reason
self.strict = strict self.strict = strict
self.decode_content = decode_content self.decode_content = decode_content
self.retries = retries
self._decoder = None self._decoder = None
self._body = None self._body = None

View File

@ -1,6 +1,8 @@
from __future__ import absolute_import from __future__ import absolute_import
import time import time
import logging import logging
from collections import namedtuple
from itertools import takewhile
from ..exceptions import ( from ..exceptions import (
ConnectTimeoutError, ConnectTimeoutError,
@ -14,6 +16,10 @@ from ..packages import six
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
# Data structure for representing the metadata of requests that result in a retry.
RequestHistory = namedtuple('RequestHistory', ["method", "url", "error",
"status", "redirect_location"])
class Retry(object): class Retry(object):
""" Retry configuration. """ Retry configuration.
@ -113,6 +119,10 @@ class Retry(object):
whether we should raise an exception, or return a response, whether we should raise an exception, or return a response,
if status falls in ``status_forcelist`` range and retries have if status falls in ``status_forcelist`` range and retries have
been exhausted. been exhausted.
:param tuple history: The history of the request encountered during
each call to :meth:`~Retry.increment`. The list is in the order
the requests occurred. Each list item is of class :class:`RequestHistory`.
""" """
DEFAULT_METHOD_WHITELIST = frozenset([ DEFAULT_METHOD_WHITELIST = frozenset([
@ -124,7 +134,7 @@ class Retry(object):
def __init__(self, total=10, connect=None, read=None, redirect=None, def __init__(self, total=10, connect=None, read=None, redirect=None,
method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None, method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None,
backoff_factor=0, raise_on_redirect=True, raise_on_status=True, backoff_factor=0, raise_on_redirect=True, raise_on_status=True,
_observed_errors=0): history=None):
self.total = total self.total = total
self.connect = connect self.connect = connect
@ -140,7 +150,7 @@ class Retry(object):
self.backoff_factor = backoff_factor self.backoff_factor = backoff_factor
self.raise_on_redirect = raise_on_redirect self.raise_on_redirect = raise_on_redirect
self.raise_on_status = raise_on_status self.raise_on_status = raise_on_status
self._observed_errors = _observed_errors # TODO: use .history instead? self.history = history or tuple()
def new(self, **kw): def new(self, **kw):
params = dict( params = dict(
@ -151,7 +161,7 @@ class Retry(object):
backoff_factor=self.backoff_factor, backoff_factor=self.backoff_factor,
raise_on_redirect=self.raise_on_redirect, raise_on_redirect=self.raise_on_redirect,
raise_on_status=self.raise_on_status, raise_on_status=self.raise_on_status,
_observed_errors=self._observed_errors, history=self.history,
) )
params.update(kw) params.update(kw)
return type(self)(**params) return type(self)(**params)
@ -175,10 +185,13 @@ class Retry(object):
:rtype: float :rtype: float
""" """
if self._observed_errors <= 1: # We want to consider only the last consecutive errors sequence (Ignore redirects).
consecutive_errors_len = len(list(takewhile(lambda x: x.redirect_location is None,
reversed(self.history))))
if consecutive_errors_len <= 1:
return 0 return 0
backoff_value = self.backoff_factor * (2 ** (self._observed_errors - 1)) backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1))
return min(self.BACKOFF_MAX, backoff_value) return min(self.BACKOFF_MAX, backoff_value)
def sleep(self): def sleep(self):
@ -241,11 +254,12 @@ class Retry(object):
if total is not None: if total is not None:
total -= 1 total -= 1
_observed_errors = self._observed_errors
connect = self.connect connect = self.connect
read = self.read read = self.read
redirect = self.redirect redirect = self.redirect
cause = 'unknown' cause = 'unknown'
status = None
redirect_location = None
if error and self._is_connection_error(error): if error and self._is_connection_error(error):
# Connect retry? # Connect retry?
@ -253,7 +267,6 @@ class Retry(object):
raise six.reraise(type(error), error, _stacktrace) raise six.reraise(type(error), error, _stacktrace)
elif connect is not None: elif connect is not None:
connect -= 1 connect -= 1
_observed_errors += 1
elif error and self._is_read_error(error): elif error and self._is_read_error(error):
# Read retry? # Read retry?
@ -261,27 +274,30 @@ class Retry(object):
raise six.reraise(type(error), error, _stacktrace) raise six.reraise(type(error), error, _stacktrace)
elif read is not None: elif read is not None:
read -= 1 read -= 1
_observed_errors += 1
elif response and response.get_redirect_location(): elif response and response.get_redirect_location():
# Redirect retry? # Redirect retry?
if redirect is not None: if redirect is not None:
redirect -= 1 redirect -= 1
cause = 'too many redirects' cause = 'too many redirects'
redirect_location = response.get_redirect_location()
status = response.status
else: else:
# Incrementing because of a server error like a 500 in # Incrementing because of a server error like a 500 in
# status_forcelist and a the given method is in the whitelist # status_forcelist and a the given method is in the whitelist
_observed_errors += 1
cause = ResponseError.GENERIC_ERROR cause = ResponseError.GENERIC_ERROR
if response and response.status: if response and response.status:
cause = ResponseError.SPECIFIC_ERROR.format( cause = ResponseError.SPECIFIC_ERROR.format(
status_code=response.status) status_code=response.status)
status = response.status
history = self.history + (RequestHistory(method, url, error, status, redirect_location),)
new_retry = self.new( new_retry = self.new(
total=total, total=total,
connect=connect, read=read, redirect=redirect, connect=connect, read=read, redirect=redirect,
_observed_errors=_observed_errors) history=history)
if new_retry.is_exhausted(): if new_retry.is_exhausted():
raise MaxRetryError(_pool, url, error or ResponseError(cause)) raise MaxRetryError(_pool, url, error or ResponseError(cause))

View File

@ -287,6 +287,9 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
""" """
context = ssl_context context = ssl_context
if context is None: if context is None:
# Note: This branch of code and all the variables in it are no longer
# used by urllib3 itself. We should consider deprecating and removing
# this code.
context = create_urllib3_context(ssl_version, cert_reqs, context = create_urllib3_context(ssl_version, cert_reqs,
ciphers=ciphers) ciphers=ciphers)

View File

@ -10,7 +10,8 @@ url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment']
class Url(namedtuple('Url', url_attrs)): class Url(namedtuple('Url', url_attrs)):
""" """
Datastructure for representing an HTTP URL. Used as a return value for Datastructure for representing an HTTP URL. Used as a return value for
:func:`parse_url`. :func:`parse_url`. Both the scheme and host are normalized as they are
both case-insensitive according to RFC 3986.
""" """
slots = () slots = ()
@ -18,6 +19,10 @@ class Url(namedtuple('Url', url_attrs)):
query=None, fragment=None): query=None, fragment=None):
if path and not path.startswith('/'): if path and not path.startswith('/'):
path = '/' + path path = '/' + path
if scheme:
scheme = scheme.lower()
if host:
host = host.lower()
return super(Url, cls).__new__(cls, scheme, auth, host, port, path, return super(Url, cls).__new__(cls, scheme, auth, host, port, path,
query, fragment) query, fragment)
@ -211,7 +216,7 @@ def parse_url(url):
def get_host(url): def get_host(url):
""" """
Deprecated. Use :func:`.parse_url` instead. Deprecated. Use :func:`parse_url` instead.
""" """
p = parse_url(url) p = parse_url(url)
return p.scheme or 'http', p.hostname, p.port return p.scheme or 'http', p.hostname, p.port