From d46c3e96d5e489d45c2910898212f58c646518c9 Mon Sep 17 00:00:00 2001 From: Alan Hamlett Date: Sat, 21 May 2016 14:27:41 +0200 Subject: [PATCH] upgrade wakatime-cli to v6.0.3 --- plugin/packages/wakatime/__about__.py | 2 +- .../wakatime/dependencies/templates.py | 22 - plugin/packages/wakatime/main.py | 6 +- .../wakatime/packages/requests/__init__.py | 14 +- .../wakatime/packages/requests/adapters.py | 50 +- .../wakatime/packages/requests/api.py | 6 +- .../wakatime/packages/requests/auth.py | 1 + .../wakatime/packages/requests/models.py | 12 +- .../requests/packages/urllib3/__init__.py | 11 +- .../requests/packages/urllib3/_collections.py | 2 +- .../requests/packages/urllib3/connection.py | 64 +- .../packages/urllib3/connectionpool.py | 93 ++- .../packages/urllib3/contrib/appengine.py | 12 +- .../packages/urllib3/contrib/ntlmpool.py | 20 +- .../packages/urllib3/contrib/pyopenssl.py | 64 +- .../packages/urllib3/contrib/socks.py | 172 ++++ .../requests/packages/urllib3/exceptions.py | 8 + .../requests/packages/urllib3/fields.py | 4 +- .../urllib3/packages/backports/__init__.py | 0 .../urllib3/packages/backports/makefile.py | 53 ++ .../packages/ssl_match_hostname/.gitignore | 1 - .../requests/packages/urllib3/poolmanager.py | 17 +- .../requests/packages/urllib3/response.py | 38 +- .../packages/urllib3/util/__init__.py | 2 + .../packages/urllib3/util/response.py | 2 +- .../requests/packages/urllib3/util/retry.py | 14 +- .../requests/packages/urllib3/util/ssl_.py | 11 +- .../wakatime/packages/requests/sessions.py | 47 +- .../packages/requests/status_codes.py | 1 + .../wakatime/packages/requests/structures.py | 4 +- .../wakatime/packages/requests/utils.py | 56 +- plugin/packages/wakatime/packages/socks.py | 765 ++++++++++++++++++ .../wakatime/packages/sockshandler.py | 79 ++ plugin/packages/wakatime/projects/git.py | 4 +- 34 files changed, 1474 insertions(+), 183 deletions(-) create mode 100644 plugin/packages/wakatime/packages/requests/packages/urllib3/contrib/socks.py create mode 100644 plugin/packages/wakatime/packages/requests/packages/urllib3/packages/backports/__init__.py create mode 100644 plugin/packages/wakatime/packages/requests/packages/urllib3/packages/backports/makefile.py delete mode 100644 plugin/packages/wakatime/packages/requests/packages/urllib3/packages/ssl_match_hostname/.gitignore create mode 100644 plugin/packages/wakatime/packages/socks.py create mode 100644 plugin/packages/wakatime/packages/sockshandler.py diff --git a/plugin/packages/wakatime/__about__.py b/plugin/packages/wakatime/__about__.py index 9075cf3..d8eb88c 100644 --- a/plugin/packages/wakatime/__about__.py +++ b/plugin/packages/wakatime/__about__.py @@ -1,7 +1,7 @@ __title__ = 'wakatime' __description__ = 'Common interface to the WakaTime api.' __url__ = 'https://github.com/wakatime/wakatime' -__version_info__ = ('6', '0', '2') +__version_info__ = ('6', '0', '3') __version__ = '.'.join(__version_info__) __author__ = 'Alan Hamlett' __author_email__ = 'alan@wakatime.com' diff --git a/plugin/packages/wakatime/dependencies/templates.py b/plugin/packages/wakatime/dependencies/templates.py index 1430881..2ba605e 100644 --- a/plugin/packages/wakatime/dependencies/templates.py +++ b/plugin/packages/wakatime/dependencies/templates.py @@ -69,28 +69,6 @@ KEYWORDS = [ ] -class LassoJavascriptParser(TokenParser): - - def parse(self): - for index, token, content in self.tokens: - self._process_token(token, content) - return self.dependencies - - def _process_token(self, token, content): - if u(token) == 'Token.Name.Other': - self._process_name(token, content) - elif u(token) == 'Token.Literal.String.Single' or u(token) == 'Token.Literal.String.Double': - self._process_literal_string(token, content) - - def _process_name(self, token, content): - if content.lower() in KEYWORDS: - self.append(content.lower()) - - def _process_literal_string(self, token, content): - if 'famous/core/' in content.strip('"').strip("'"): - self.append('famous') - - class HtmlDjangoParser(TokenParser): tags = [] getting_attrs = False diff --git a/plugin/packages/wakatime/main.py b/plugin/packages/wakatime/main.py index e35a3cc..a32c040 100644 --- a/plugin/packages/wakatime/main.py +++ b/plugin/packages/wakatime/main.py @@ -127,8 +127,10 @@ def parseArguments(): help='entity type for this heartbeat. can be one of "file", '+ '"domain", or "app"; defaults to file.') parser.add_argument('--proxy', dest='proxy', - help='optional https proxy url; for example: '+ - 'https://user:pass@localhost:8080') + help='optional proxy configuration. Supports HTTPS '+ + 'and SOCKS proxies. For example: '+ + 'https://user:pass@host:port or '+ + 'socks5://user:pass@host:port') parser.add_argument('--project', dest='project', help='optional project name') parser.add_argument('--alternate-project', dest='alternate_project', diff --git a/plugin/packages/wakatime/packages/requests/__init__.py b/plugin/packages/wakatime/packages/requests/__init__.py index 1218d43..82c0f78 100644 --- a/plugin/packages/wakatime/packages/requests/__init__.py +++ b/plugin/packages/wakatime/packages/requests/__init__.py @@ -36,14 +36,14 @@ usage: The other HTTP methods are supported - see `requests.api`. Full documentation is at . -:copyright: (c) 2015 by Kenneth Reitz. +:copyright: (c) 2016 by Kenneth Reitz. :license: Apache 2.0, see LICENSE for more details. """ __title__ = 'requests' -__version__ = '2.9.1' -__build__ = 0x020901 +__version__ = '2.10.0' +__build__ = 0x021000 __author__ = 'Kenneth Reitz' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016 Kenneth Reitz' @@ -55,6 +55,12 @@ try: except ImportError: pass +import warnings + +# urllib3's DependencyWarnings should be silenced. +from .packages.urllib3.exceptions import DependencyWarning +warnings.simplefilter('ignore', DependencyWarning) + from . import utils from .models import Request, Response, PreparedRequest from .api import request, get, head, post, patch, put, delete, options @@ -63,7 +69,7 @@ from .status_codes import codes from .exceptions import ( RequestException, Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError, - FileModeWarning, + FileModeWarning, ConnectTimeout, ReadTimeout ) # Set default logging handler to avoid "No handler found" warnings. diff --git a/plugin/packages/wakatime/packages/requests/adapters.py b/plugin/packages/wakatime/packages/requests/adapters.py index 4f2b23c..23e448f 100644 --- a/plugin/packages/wakatime/packages/requests/adapters.py +++ b/plugin/packages/wakatime/packages/requests/adapters.py @@ -19,7 +19,7 @@ from .packages.urllib3.util.retry import Retry from .compat import urlparse, basestring from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers, prepend_scheme_if_needed, get_auth_from_url, urldefragauth, - select_proxy) + select_proxy, to_native_string) from .structures import CaseInsensitiveDict from .packages.urllib3.exceptions import ClosedPoolError from .packages.urllib3.exceptions import ConnectTimeoutError @@ -33,9 +33,15 @@ from .packages.urllib3.exceptions import SSLError as _SSLError from .packages.urllib3.exceptions import ResponseError from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError) + ProxyError, RetryError, InvalidSchema) from .auth import _basic_auth_str +try: + from .packages.urllib3.contrib.socks import SOCKSProxyManager +except ImportError: + def SOCKSProxyManager(*args, **kwargs): + raise InvalidSchema("Missing dependencies for SOCKS support.") + DEFAULT_POOLBLOCK = False DEFAULT_POOLSIZE = 10 DEFAULT_RETRIES = 0 @@ -149,9 +155,22 @@ class HTTPAdapter(BaseAdapter): :param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager. :returns: ProxyManager """ - if not proxy in self.proxy_manager: + if proxy in self.proxy_manager: + manager = self.proxy_manager[proxy] + elif proxy.lower().startswith('socks'): + username, password = get_auth_from_url(proxy) + manager = self.proxy_manager[proxy] = SOCKSProxyManager( + proxy, + username=username, + password=password, + num_pools=self._pool_connections, + maxsize=self._pool_maxsize, + block=self._pool_block, + **proxy_kwargs + ) + else: proxy_headers = self.proxy_headers(proxy) - self.proxy_manager[proxy] = proxy_from_url( + manager = self.proxy_manager[proxy] = proxy_from_url( proxy, proxy_headers=proxy_headers, num_pools=self._pool_connections, @@ -159,7 +178,7 @@ class HTTPAdapter(BaseAdapter): block=self._pool_block, **proxy_kwargs) - return self.proxy_manager[proxy] + return manager def cert_verify(self, conn, url, verify, cert): """Verify a SSL certificate. This method should not be called from user @@ -264,10 +283,12 @@ class HTTPAdapter(BaseAdapter): def close(self): """Disposes of any internal state. - Currently, this just closes the PoolManager, which closes pooled - connections. + Currently, this closes the PoolManager and any active ProxyManager, + which closes any pooled connections. """ self.poolmanager.clear() + for proxy in self.proxy_manager.values(): + proxy.clear() def request_url(self, request, proxies): """Obtain the url to use when making the final request. @@ -284,10 +305,16 @@ class HTTPAdapter(BaseAdapter): """ proxy = select_proxy(request.url, proxies) scheme = urlparse(request.url).scheme - if proxy and scheme != 'https': + + is_proxied_http_request = (proxy and scheme != 'https') + using_socks_proxy = False + if proxy: + proxy_scheme = urlparse(proxy).scheme.lower() + using_socks_proxy = proxy_scheme.startswith('socks') + + url = request.path_url + if is_proxied_http_request and not using_socks_proxy: url = urldefragauth(request.url) - else: - url = request.path_url return url @@ -434,6 +461,9 @@ class HTTPAdapter(BaseAdapter): if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) + if isinstance(e.reason, _ProxyError): + raise ProxyError(e, request=request) + raise ConnectionError(e, request=request) except ClosedPoolError as e: diff --git a/plugin/packages/wakatime/packages/requests/api.py b/plugin/packages/wakatime/packages/requests/api.py index b21a1a4..c2068d0 100644 --- a/plugin/packages/wakatime/packages/requests/api.py +++ b/plugin/packages/wakatime/packages/requests/api.py @@ -24,7 +24,11 @@ def request(method, url, **kwargs): :param json: (optional) json data to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. - :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': ('filename', fileobj)}``) for multipart encoding upload. + :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload. + ``file-tuple`` can be a 2-tuple ``('filename', fileobj)``, 3-tuple ``('filename', fileobj, 'content_type')`` + or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``, where ``'content-type'`` is a string + defining the content type of the given file and ``custom_headers`` a dict-like object containing additional headers + to add for the file. :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read diff --git a/plugin/packages/wakatime/packages/requests/auth.py b/plugin/packages/wakatime/packages/requests/auth.py index edf4c8d..73f8e9d 100644 --- a/plugin/packages/wakatime/packages/requests/auth.py +++ b/plugin/packages/wakatime/packages/requests/auth.py @@ -93,6 +93,7 @@ class HTTPDigestAuth(AuthBase): qop = self._thread_local.chal.get('qop') algorithm = self._thread_local.chal.get('algorithm') opaque = self._thread_local.chal.get('opaque') + hash_utf8 = None if algorithm is None: _algorithm = 'MD5' diff --git a/plugin/packages/wakatime/packages/requests/models.py b/plugin/packages/wakatime/packages/requests/models.py index 4bcbc54..0c10eef 100644 --- a/plugin/packages/wakatime/packages/requests/models.py +++ b/plugin/packages/wakatime/packages/requests/models.py @@ -103,8 +103,10 @@ class RequestEncodingMixin(object): """Build the body for a multipart/form-data request. Will successfully encode files when passed as a dict or a list of - 2-tuples. Order is retained if data is a list of 2-tuples but arbitrary + tuples. Order is retained if data is a list of tuples but arbitrary if parameters are supplied as a dict. + The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype) + or 4-tuples (filename, fileobj, contentype, custom_headers). """ if (not files): @@ -463,9 +465,11 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): def prepare_content_length(self, body): if hasattr(body, 'seek') and hasattr(body, 'tell'): + curr_pos = body.tell() body.seek(0, 2) - self.headers['Content-Length'] = builtin_str(body.tell()) - body.seek(0, 0) + end_pos = body.tell() + self.headers['Content-Length'] = builtin_str(max(0, end_pos - curr_pos)) + body.seek(curr_pos, 0) elif body is not None: l = super_len(body) if l: @@ -788,7 +792,7 @@ class Response(object): :param \*\*kwargs: Optional arguments that ``json.loads`` takes. """ - if not self.encoding and len(self.content) > 3: + if not self.encoding and self.content and len(self.content) > 3: # No encoding set. JSON RFC 4627 section 3 states we should expect # UTF-8, -16 or -32. Detect which one to use; If the detection or # decoding fails, fall back to `self.text` (using chardet to make diff --git a/plugin/packages/wakatime/packages/requests/packages/urllib3/__init__.py b/plugin/packages/wakatime/packages/requests/packages/urllib3/__init__.py index e43991a..7366899 100644 --- a/plugin/packages/wakatime/packages/requests/packages/urllib3/__init__.py +++ b/plugin/packages/wakatime/packages/requests/packages/urllib3/__init__.py @@ -32,7 +32,7 @@ except ImportError: __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.13.1' +__version__ = '1.15.1' __all__ = ( 'HTTPConnectionPool', @@ -68,22 +68,25 @@ def add_stderr_logger(level=logging.DEBUG): handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) logger.addHandler(handler) logger.setLevel(level) - logger.debug('Added a stderr logging handler to logger: %s' % __name__) + logger.debug('Added a stderr logging handler to logger: %s', __name__) return handler # ... Clean up. del NullHandler +# All warning filters *must* be appended unless you're really certain that they +# shouldn't be: otherwise, it's very hard for users to use most Python +# mechanisms to silence them. # SecurityWarning's always go off by default. warnings.simplefilter('always', exceptions.SecurityWarning, append=True) # SubjectAltNameWarning's should go off once per host -warnings.simplefilter('default', exceptions.SubjectAltNameWarning) +warnings.simplefilter('default', exceptions.SubjectAltNameWarning, append=True) # InsecurePlatformWarning's don't vary between requests, so we keep it default. warnings.simplefilter('default', exceptions.InsecurePlatformWarning, append=True) # SNIMissingWarnings should go off only once. -warnings.simplefilter('default', exceptions.SNIMissingWarning) +warnings.simplefilter('default', exceptions.SNIMissingWarning, append=True) def disable_warnings(category=exceptions.HTTPWarning): diff --git a/plugin/packages/wakatime/packages/requests/packages/urllib3/_collections.py b/plugin/packages/wakatime/packages/requests/packages/urllib3/_collections.py index 67f3ce9..77cee01 100644 --- a/plugin/packages/wakatime/packages/requests/packages/urllib3/_collections.py +++ b/plugin/packages/wakatime/packages/requests/packages/urllib3/_collections.py @@ -134,7 +134,7 @@ class HTTPHeaderDict(MutableMapping): def __init__(self, headers=None, **kwargs): super(HTTPHeaderDict, self).__init__() - self._container = {} + self._container = OrderedDict() if headers is not None: if isinstance(headers, HTTPHeaderDict): self._copy_from(headers) diff --git a/plugin/packages/wakatime/packages/requests/packages/urllib3/connection.py b/plugin/packages/wakatime/packages/requests/packages/urllib3/connection.py index 1e4cd41..5ce0080 100644 --- a/plugin/packages/wakatime/packages/requests/packages/urllib3/connection.py +++ b/plugin/packages/wakatime/packages/requests/packages/urllib3/connection.py @@ -1,5 +1,6 @@ from __future__ import absolute_import import datetime +import logging import os import sys import socket @@ -38,7 +39,7 @@ from .exceptions import ( SubjectAltNameWarning, SystemTimeWarning, ) -from .packages.ssl_match_hostname import match_hostname +from .packages.ssl_match_hostname import match_hostname, CertificateError from .util.ssl_ import ( resolve_cert_reqs, @@ -50,6 +51,10 @@ from .util.ssl_ import ( from .util import connection +from ._collections import HTTPHeaderDict + +log = logging.getLogger(__name__) + port_by_scheme = { 'http': 80, 'https': 443, @@ -162,6 +167,38 @@ class HTTPConnection(_HTTPConnection, object): conn = self._new_conn() self._prepare_conn(conn) + def request_chunked(self, method, url, body=None, headers=None): + """ + Alternative to the common request method, which sends the + body with chunked encoding and not as one block + """ + headers = HTTPHeaderDict(headers if headers is not None else {}) + skip_accept_encoding = 'accept-encoding' in headers + self.putrequest(method, url, skip_accept_encoding=skip_accept_encoding) + for header, value in headers.items(): + self.putheader(header, value) + if 'transfer-encoding' not in headers: + self.putheader('Transfer-Encoding', 'chunked') + self.endheaders() + + if body is not None: + stringish_types = six.string_types + (six.binary_type,) + if isinstance(body, stringish_types): + body = (body,) + for chunk in body: + if not chunk: + continue + if not isinstance(chunk, six.binary_type): + chunk = chunk.encode('utf8') + len_str = hex(len(chunk))[2:] + self.send(len_str.encode('utf-8')) + self.send(b'\r\n') + self.send(chunk) + self.send(b'\r\n') + + # After the if clause, to always have a closed body + self.send(b'0\r\n\r\n') + class HTTPSConnection(HTTPConnection): default_port = port_by_scheme['https'] @@ -265,21 +302,26 @@ class VerifiedHTTPSConnection(HTTPSConnection): 'for details.)'.format(hostname)), SubjectAltNameWarning ) - - # In case the hostname is an IPv6 address, strip the square - # brackets from it before using it to validate. This is because - # a certificate with an IPv6 address in it won't have square - # brackets around that address. Sadly, match_hostname won't do this - # for us: it expects the plain host part without any extra work - # that might have been done to make it palatable to httplib. - asserted_hostname = self.assert_hostname or hostname - asserted_hostname = asserted_hostname.strip('[]') - match_hostname(cert, asserted_hostname) + _match_hostname(cert, self.assert_hostname or hostname) self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED or self.assert_fingerprint is not None) +def _match_hostname(cert, asserted_hostname): + try: + match_hostname(cert, asserted_hostname) + except CertificateError as e: + log.error( + 'Certificate did not match expected hostname: %s. ' + 'Certificate: %s', asserted_hostname, cert + ) + # Add cert to exception and reraise so client code can inspect + # the cert when catching the exception, if they want to + e._peer_cert = cert + raise + + if ssl: # Make a copy for testing. UnverifiedHTTPSConnection = HTTPSConnection diff --git a/plugin/packages/wakatime/packages/requests/packages/urllib3/connectionpool.py b/plugin/packages/wakatime/packages/requests/packages/urllib3/connectionpool.py index 995b416..3fcfb12 100644 --- a/plugin/packages/wakatime/packages/requests/packages/urllib3/connectionpool.py +++ b/plugin/packages/wakatime/packages/requests/packages/urllib3/connectionpool.py @@ -69,7 +69,13 @@ class ConnectionPool(object): if not host: raise LocationValueError("No host specified.") - self.host = host + # httplib doesn't like it when we include brackets in ipv6 addresses + # Specifically, if we include brackets but also pass the port then + # httplib crazily doubles up the square brackets on the Host header. + # Instead, we need to make sure we never pass ``None`` as the port. + # However, for backward compatibility reasons we can't actually + # *assert* that. + self.host = host.strip('[]') self.port = port def __str__(self): @@ -203,8 +209,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): Return a fresh :class:`HTTPConnection`. """ self.num_connections += 1 - log.info("Starting new HTTP connection (%d): %s" % - (self.num_connections, self.host)) + log.info("Starting new HTTP connection (%d): %s", + self.num_connections, self.host) conn = self.ConnectionCls(host=self.host, port=self.port, timeout=self.timeout.connect_timeout, @@ -239,7 +245,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # If this is a persistent connection, check if it got disconnected if conn and is_connection_dropped(conn): - log.info("Resetting dropped connection: %s" % self.host) + log.info("Resetting dropped connection: %s", self.host) conn.close() if getattr(conn, 'auto_open', 1) == 0: # This is a proxied connection that has been mutated by @@ -272,7 +278,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): except Full: # This should never happen if self.block == True log.warning( - "Connection pool is full, discarding connection: %s" % + "Connection pool is full, discarding connection: %s", self.host) # Connection never got put back into the pool, close it. @@ -318,7 +324,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python 2.6 raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) - def _make_request(self, conn, method, url, timeout=_Default, + def _make_request(self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw): """ Perform a request on a given urllib connection object taken from our @@ -350,7 +356,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # conn.request() calls httplib.*.request, not the method in # urllib3.request. It also calls makefile (recv) on the socket. - conn.request(method, url, **httplib_request_kw) + if chunked: + conn.request_chunked(method, url, **httplib_request_kw) + else: + conn.request(method, url, **httplib_request_kw) # Reset the timeout for the recv() on the socket read_timeout = timeout_obj.read_timeout @@ -382,9 +391,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # AppEngine doesn't have a version attr. http_version = getattr(conn, '_http_vsn_str', 'HTTP/?') - log.debug("\"%s %s %s\" %s %s" % (method, url, http_version, - httplib_response.status, - httplib_response.length)) + log.debug("\"%s %s %s\" %s %s", method, url, http_version, + httplib_response.status, httplib_response.length) try: assert_header_parsing(httplib_response.msg) @@ -435,7 +443,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): def urlopen(self, method, url, body=None, headers=None, retries=None, redirect=True, assert_same_host=True, timeout=_Default, - pool_timeout=None, release_conn=None, **response_kw): + pool_timeout=None, release_conn=None, chunked=False, + **response_kw): """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all @@ -512,6 +521,11 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): back into the pool. If None, it takes the value of ``response_kw.get('preload_content', True)``. + :param chunked: + If True, urllib3 will send the body using chunked transfer + encoding. Otherwise, urllib3 will send the body using the standard + content-length form. Defaults to False. + :param \**response_kw: Additional parameters are passed to :meth:`urllib3.response.HTTPResponse.from_httplib` @@ -542,6 +556,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # complains about UnboundLocalError. err = None + # Keep track of whether we cleanly exited the except block. This + # ensures we do proper cleanup in finally. + clean_exit = False + try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) @@ -556,13 +574,14 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Make the request on the httplib connection object. httplib_response = self._make_request(conn, method, url, timeout=timeout_obj, - body=body, headers=headers) + body=body, headers=headers, + chunked=chunked) # If we're going to release the connection in ``finally:``, then - # the request doesn't need to know about the connection. Otherwise + # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. - response_conn = not release_conn and conn + response_conn = conn if not release_conn else None # Import httplib's response into our own wrapper object response = HTTPResponse.from_httplib(httplib_response, @@ -570,10 +589,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): connection=response_conn, **response_kw) - # else: - # The connection will be put back into the pool when - # ``response.release_conn()`` is called (implicitly by - # ``response.read()``) + # Everything went great! + clean_exit = True except Empty: # Timed out by queue. @@ -583,22 +600,19 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Close the connection. If a connection is reused on which there # was a Certificate error, the next request will certainly raise # another Certificate error. - conn = conn and conn.close() - release_conn = True + clean_exit = False raise SSLError(e) except SSLError: # Treat SSLError separately from BaseSSLError to preserve # traceback. - conn = conn and conn.close() - release_conn = True + clean_exit = False raise except (TimeoutError, HTTPException, SocketError, ProtocolError) as e: # Discard the connection for these exceptions. It will be # be replaced during the next _get_conn() call. - conn = conn and conn.close() - release_conn = True + clean_exit = False if isinstance(e, (SocketError, NewConnectionError)) and self.proxy: e = ProxyError('Cannot connect to proxy.', e) @@ -613,6 +627,14 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): err = e finally: + if not clean_exit: + # We hit some kind of exception, handled or otherwise. We need + # to throw the connection away unless explicitly told not to. + # Close the connection, set the variable to None, and make sure + # we put the None back in the pool to avoid leaking it. + conn = conn and conn.close() + release_conn = True + if release_conn: # Put the connection back to be reused. If the connection is # expired then it will be None, which will get replaced with a @@ -622,7 +644,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if not conn: # Try again log.warning("Retrying (%r) after connection " - "broken by '%r': %s" % (retries, err, url)) + "broken by '%r': %s", retries, err, url) return self.urlopen(method, url, body, headers, retries, redirect, assert_same_host, timeout=timeout, pool_timeout=pool_timeout, @@ -644,7 +666,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): raise return response - log.info("Redirecting %s -> %s" % (url, redirect_location)) + log.info("Redirecting %s -> %s", url, redirect_location) return self.urlopen( method, redirect_location, body, headers, retries=retries, redirect=redirect, @@ -654,9 +676,17 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Check if we should retry the HTTP response. if retries.is_forced_retry(method, status_code=response.status): - retries = retries.increment(method, url, response=response, _pool=self) + try: + retries = retries.increment(method, url, response=response, _pool=self) + except MaxRetryError: + if retries.raise_on_status: + # Release the connection for this response, since we're not + # returning it to be released manually. + response.release_conn() + raise + return response retries.sleep() - log.info("Forced retry: %s" % url) + log.info("Forced retry: %s", url) return self.urlopen( method, url, body, headers, retries=retries, redirect=redirect, @@ -742,7 +772,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): except AttributeError: # Platform-specific: Python 2.6 set_tunnel = conn._set_tunnel - if sys.version_info <= (2, 6, 4) and not self.proxy_headers: # Python 2.6.4 and older + if sys.version_info <= (2, 6, 4) and not self.proxy_headers: # Python 2.6.4 and older set_tunnel(self.host, self.port) else: set_tunnel(self.host, self.port, self.proxy_headers) @@ -754,8 +784,8 @@ class HTTPSConnectionPool(HTTPConnectionPool): Return a fresh :class:`httplib.HTTPSConnection`. """ self.num_connections += 1 - log.info("Starting new HTTPS connection (%d): %s" - % (self.num_connections, self.host)) + log.info("Starting new HTTPS connection (%d): %s", + self.num_connections, self.host) if not self.ConnectionCls or self.ConnectionCls is DummyConnection: raise SSLError("Can't connect to HTTPS URL because the SSL " @@ -812,6 +842,7 @@ def connection_from_url(url, **kw): >>> r = conn.request('GET', '/') """ scheme, host, port = get_host(url) + port = port or port_by_scheme.get(scheme, 80) if scheme == 'https': return HTTPSConnectionPool(host, port=port, **kw) else: diff --git a/plugin/packages/wakatime/packages/requests/packages/urllib3/contrib/appengine.py b/plugin/packages/wakatime/packages/requests/packages/urllib3/contrib/appengine.py index 884cdb2..f4289c0 100644 --- a/plugin/packages/wakatime/packages/requests/packages/urllib3/contrib/appengine.py +++ b/plugin/packages/wakatime/packages/requests/packages/urllib3/contrib/appengine.py @@ -144,7 +144,7 @@ class AppEngineManager(RequestMethods): if retries.is_forced_retry(method, status_code=http_response.status): retries = retries.increment( method, url, response=http_response, _pool=self) - log.info("Forced retry: %s" % url) + log.info("Forced retry: %s", url) retries.sleep() return self.urlopen( method, url, @@ -164,6 +164,14 @@ class AppEngineManager(RequestMethods): if content_encoding == 'deflate': del urlfetch_resp.headers['content-encoding'] + transfer_encoding = urlfetch_resp.headers.get('transfer-encoding') + # We have a full response's content, + # so let's make sure we don't report ourselves as chunked data. + if transfer_encoding == 'chunked': + encodings = transfer_encoding.split(",") + encodings.remove('chunked') + urlfetch_resp.headers['transfer-encoding'] = ','.join(encodings) + return HTTPResponse( # In order for decoding to work, we must present the content as # a file-like object. @@ -177,7 +185,7 @@ class AppEngineManager(RequestMethods): if timeout is Timeout.DEFAULT_TIMEOUT: return 5 # 5s is the default timeout for URLFetch. if isinstance(timeout, Timeout): - if timeout.read is not timeout.connect: + if timeout._read is not timeout._connect: warnings.warn( "URLFetch does not support granular timeout settings, " "reverting to total timeout.", AppEnginePlatformWarning) diff --git a/plugin/packages/wakatime/packages/requests/packages/urllib3/contrib/ntlmpool.py b/plugin/packages/wakatime/packages/requests/packages/urllib3/contrib/ntlmpool.py index c136a23..11d0b5c 100644 --- a/plugin/packages/wakatime/packages/requests/packages/urllib3/contrib/ntlmpool.py +++ b/plugin/packages/wakatime/packages/requests/packages/urllib3/contrib/ntlmpool.py @@ -43,8 +43,8 @@ class NTLMConnectionPool(HTTPSConnectionPool): # Performs the NTLM handshake that secures the connection. The socket # must be kept open while requests are performed. self.num_connections += 1 - log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s' % - (self.num_connections, self.host, self.authurl)) + log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s', + self.num_connections, self.host, self.authurl) headers = {} headers['Connection'] = 'Keep-Alive' @@ -56,13 +56,13 @@ class NTLMConnectionPool(HTTPSConnectionPool): # Send negotiation message headers[req_header] = ( 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser)) - log.debug('Request headers: %s' % headers) + log.debug('Request headers: %s', headers) conn.request('GET', self.authurl, None, headers) res = conn.getresponse() reshdr = dict(res.getheaders()) - log.debug('Response status: %s %s' % (res.status, res.reason)) - log.debug('Response headers: %s' % reshdr) - log.debug('Response data: %s [...]' % res.read(100)) + log.debug('Response status: %s %s', res.status, res.reason) + log.debug('Response headers: %s', reshdr) + log.debug('Response data: %s [...]', res.read(100)) # Remove the reference to the socket, so that it can not be closed by # the response object (we want to keep the socket open) @@ -87,12 +87,12 @@ class NTLMConnectionPool(HTTPSConnectionPool): self.pw, NegotiateFlags) headers[req_header] = 'NTLM %s' % auth_msg - log.debug('Request headers: %s' % headers) + log.debug('Request headers: %s', headers) conn.request('GET', self.authurl, None, headers) res = conn.getresponse() - log.debug('Response status: %s %s' % (res.status, res.reason)) - log.debug('Response headers: %s' % dict(res.getheaders())) - log.debug('Response data: %s [...]' % res.read()[:100]) + log.debug('Response status: %s %s', res.status, res.reason) + log.debug('Response headers: %s', dict(res.getheaders())) + log.debug('Response data: %s [...]', res.read()[:100]) if res.status != 200: if res.status == 401: raise Exception('Server rejected request: wrong ' diff --git a/plugin/packages/wakatime/packages/requests/packages/urllib3/contrib/pyopenssl.py b/plugin/packages/wakatime/packages/requests/packages/urllib3/contrib/pyopenssl.py index 5996153..ed3b9cc 100644 --- a/plugin/packages/wakatime/packages/requests/packages/urllib3/contrib/pyopenssl.py +++ b/plugin/packages/wakatime/packages/requests/packages/urllib3/contrib/pyopenssl.py @@ -54,9 +54,17 @@ except SyntaxError as e: import OpenSSL.SSL from pyasn1.codec.der import decoder as der_decoder from pyasn1.type import univ, constraint -from socket import _fileobject, timeout, error as SocketError +from socket import timeout, error as SocketError + +try: # Platform-specific: Python 2 + from socket import _fileobject +except ImportError: # Platform-specific: Python 3 + _fileobject = None + from urllib3.packages.backports.makefile import backport_makefile + import ssl import select +import six from .. import connection from .. import util @@ -90,7 +98,7 @@ _openssl_verify = { OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, } -DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS +DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS.encode('ascii') # OpenSSL will only write 16K at a time SSL_WRITE_BLOCKSIZE = 16384 @@ -104,6 +112,7 @@ def inject_into_urllib3(): connection.ssl_wrap_socket = ssl_wrap_socket util.HAS_SNI = HAS_SNI + util.IS_PYOPENSSL = True def extract_from_urllib3(): @@ -111,6 +120,7 @@ def extract_from_urllib3(): connection.ssl_wrap_socket = orig_connection_ssl_wrap_socket util.HAS_SNI = orig_util_HAS_SNI + util.IS_PYOPENSSL = False # Note: This is a slightly bug-fixed version of same from ndg-httpsclient. @@ -135,7 +145,7 @@ def get_subj_alt_name(peer_cert): for i in range(peer_cert.get_extension_count()): ext = peer_cert.get_extension(i) ext_name = ext.get_short_name() - if ext_name != 'subjectAltName': + if ext_name != b'subjectAltName': continue # PyOpenSSL returns extension data in ASN.1 encoded form @@ -167,13 +177,17 @@ class WrappedSocket(object): self.socket = socket self.suppress_ragged_eofs = suppress_ragged_eofs self._makefile_refs = 0 + self._closed = False def fileno(self): return self.socket.fileno() - def makefile(self, mode, bufsize=-1): - self._makefile_refs += 1 - return _fileobject(self, mode, bufsize, close=True) + # Copy-pasted from Python 3.5 source code + def _decref_socketios(self): + if self._makefile_refs > 0: + self._makefile_refs -= 1 + if self._closed: + self.close() def recv(self, *args, **kwargs): try: @@ -182,7 +196,7 @@ class WrappedSocket(object): if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'): return b'' else: - raise SocketError(e) + raise SocketError(str(e)) except OpenSSL.SSL.ZeroReturnError as e: if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: return b'' @@ -198,6 +212,27 @@ class WrappedSocket(object): else: return data + def recv_into(self, *args, **kwargs): + try: + return self.connection.recv_into(*args, **kwargs) + except OpenSSL.SSL.SysCallError as e: + if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'): + return 0 + else: + raise SocketError(str(e)) + except OpenSSL.SSL.ZeroReturnError as e: + if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: + return 0 + else: + raise + except OpenSSL.SSL.WantReadError: + rd, wd, ed = select.select( + [self.socket], [], [], self.socket.gettimeout()) + if not rd: + raise timeout('The read operation timed out') + else: + return self.recv_into(*args, **kwargs) + def settimeout(self, timeout): return self.socket.settimeout(timeout) @@ -225,6 +260,7 @@ class WrappedSocket(object): def close(self): if self._makefile_refs < 1: try: + self._closed = True return self.connection.close() except OpenSSL.SSL.Error: return @@ -262,6 +298,16 @@ class WrappedSocket(object): self._makefile_refs -= 1 +if _fileobject: # Platform-specific: Python 2 + def makefile(self, mode, bufsize=-1): + self._makefile_refs += 1 + return _fileobject(self, mode, bufsize, close=True) +else: # Platform-specific: Python 3 + makefile = backport_makefile + +WrappedSocket.makefile = makefile + + def _verify_callback(cnx, x509, err_no, err_depth, return_code): return err_no == 0 @@ -285,7 +331,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, else: ctx.set_default_verify_paths() - # Disable TLS compression to migitate CRIME attack (issue #309) + # Disable TLS compression to mitigate CRIME attack (issue #309) OP_NO_COMPRESSION = 0x20000 ctx.set_options(OP_NO_COMPRESSION) @@ -293,6 +339,8 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, ctx.set_cipher_list(DEFAULT_SSL_CIPHER_LIST) cnx = OpenSSL.SSL.Connection(ctx, sock) + if isinstance(server_hostname, six.text_type): # Platform-specific: Python 3 + server_hostname = server_hostname.encode('utf-8') cnx.set_tlsext_host_name(server_hostname) cnx.set_connect_state() while True: diff --git a/plugin/packages/wakatime/packages/requests/packages/urllib3/contrib/socks.py b/plugin/packages/wakatime/packages/requests/packages/urllib3/contrib/socks.py new file mode 100644 index 0000000..3748fee --- /dev/null +++ b/plugin/packages/wakatime/packages/requests/packages/urllib3/contrib/socks.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- +""" +SOCKS support for urllib3 +~~~~~~~~~~~~~~~~~~~~~~~~~ + +This contrib module contains provisional support for SOCKS proxies from within +urllib3. This module supports SOCKS4 (specifically the SOCKS4A variant) and +SOCKS5. To enable its functionality, either install PySocks or install this +module with the ``socks`` extra. + +Known Limitations: + +- Currently PySocks does not support contacting remote websites via literal + IPv6 addresses. Any such connection attempt will fail. +- Currently PySocks does not support IPv6 connections to the SOCKS proxy. Any + such connection attempt will fail. +""" +from __future__ import absolute_import + +try: + import socks +except ImportError: + import warnings + from ..exceptions import DependencyWarning + + warnings.warn(( + 'SOCKS support in urllib3 requires the installation of optional ' + 'dependencies: specifically, PySocks. For more information, see ' + 'https://urllib3.readthedocs.org/en/latest/contrib.html#socks-proxies' + ), + DependencyWarning + ) + raise + +from socket import error as SocketError, timeout as SocketTimeout + +from ..connection import ( + HTTPConnection, HTTPSConnection +) +from ..connectionpool import ( + HTTPConnectionPool, HTTPSConnectionPool +) +from ..exceptions import ConnectTimeoutError, NewConnectionError +from ..poolmanager import PoolManager +from ..util.url import parse_url + +try: + import ssl +except ImportError: + ssl = None + + +class SOCKSConnection(HTTPConnection): + """ + A plain-text HTTP connection that connects via a SOCKS proxy. + """ + def __init__(self, *args, **kwargs): + self._socks_options = kwargs.pop('_socks_options') + super(SOCKSConnection, self).__init__(*args, **kwargs) + + def _new_conn(self): + """ + Establish a new connection via the SOCKS proxy. + """ + extra_kw = {} + if self.source_address: + extra_kw['source_address'] = self.source_address + + if self.socket_options: + extra_kw['socket_options'] = self.socket_options + + try: + conn = socks.create_connection( + (self.host, self.port), + proxy_type=self._socks_options['socks_version'], + proxy_addr=self._socks_options['proxy_host'], + proxy_port=self._socks_options['proxy_port'], + proxy_username=self._socks_options['username'], + proxy_password=self._socks_options['password'], + timeout=self.timeout, + **extra_kw + ) + + except SocketTimeout as e: + raise ConnectTimeoutError( + self, "Connection to %s timed out. (connect timeout=%s)" % + (self.host, self.timeout)) + + except socks.ProxyError as e: + # This is fragile as hell, but it seems to be the only way to raise + # useful errors here. + if e.socket_err: + error = e.socket_err + if isinstance(error, SocketTimeout): + raise ConnectTimeoutError( + self, + "Connection to %s timed out. (connect timeout=%s)" % + (self.host, self.timeout) + ) + else: + raise NewConnectionError( + self, + "Failed to establish a new connection: %s" % error + ) + else: + raise NewConnectionError( + self, + "Failed to establish a new connection: %s" % e + ) + + except SocketError as e: # Defensive: PySocks should catch all these. + raise NewConnectionError( + self, "Failed to establish a new connection: %s" % e) + + return conn + + +# We don't need to duplicate the Verified/Unverified distinction from +# urllib3/connection.py here because the HTTPSConnection will already have been +# correctly set to either the Verified or Unverified form by that module. This +# means the SOCKSHTTPSConnection will automatically be the correct type. +class SOCKSHTTPSConnection(SOCKSConnection, HTTPSConnection): + pass + + +class SOCKSHTTPConnectionPool(HTTPConnectionPool): + ConnectionCls = SOCKSConnection + + +class SOCKSHTTPSConnectionPool(HTTPSConnectionPool): + ConnectionCls = SOCKSHTTPSConnection + + +class SOCKSProxyManager(PoolManager): + """ + A version of the urllib3 ProxyManager that routes connections via the + defined SOCKS proxy. + """ + pool_classes_by_scheme = { + 'http': SOCKSHTTPConnectionPool, + 'https': SOCKSHTTPSConnectionPool, + } + + def __init__(self, proxy_url, username=None, password=None, + num_pools=10, headers=None, **connection_pool_kw): + parsed = parse_url(proxy_url) + + if parsed.scheme == 'socks5': + socks_version = socks.PROXY_TYPE_SOCKS5 + elif parsed.scheme == 'socks4': + socks_version = socks.PROXY_TYPE_SOCKS4 + else: + raise ValueError( + "Unable to determine SOCKS version from %s" % proxy_url + ) + + self.proxy_url = proxy_url + + socks_options = { + 'socks_version': socks_version, + 'proxy_host': parsed.host, + 'proxy_port': parsed.port, + 'username': username, + 'password': password, + } + connection_pool_kw['_socks_options'] = socks_options + + super(SOCKSProxyManager, self).__init__( + num_pools, headers, **connection_pool_kw + ) + + self.pool_classes_by_scheme = SOCKSProxyManager.pool_classes_by_scheme diff --git a/plugin/packages/wakatime/packages/requests/packages/urllib3/exceptions.py b/plugin/packages/wakatime/packages/requests/packages/urllib3/exceptions.py index 8e07eb6..f2e6591 100644 --- a/plugin/packages/wakatime/packages/requests/packages/urllib3/exceptions.py +++ b/plugin/packages/wakatime/packages/requests/packages/urllib3/exceptions.py @@ -180,6 +180,14 @@ class SNIMissingWarning(HTTPWarning): pass +class DependencyWarning(HTTPWarning): + """ + Warned when an attempt is made to import a module with missing optional + dependencies. + """ + pass + + class ResponseNotChunked(ProtocolError, ValueError): "Response needs to be chunked in order to read it as chunks." pass diff --git a/plugin/packages/wakatime/packages/requests/packages/urllib3/fields.py b/plugin/packages/wakatime/packages/requests/packages/urllib3/fields.py index c7d4811..8fa2a12 100644 --- a/plugin/packages/wakatime/packages/requests/packages/urllib3/fields.py +++ b/plugin/packages/wakatime/packages/requests/packages/urllib3/fields.py @@ -36,11 +36,11 @@ def format_header_param(name, value): result = '%s="%s"' % (name, value) try: result.encode('ascii') - except UnicodeEncodeError: + except (UnicodeEncodeError, UnicodeDecodeError): pass else: return result - if not six.PY3: # Python 2: + if not six.PY3 and isinstance(value, six.text_type): # Python 2: value = value.encode('utf-8') value = email.utils.encode_rfc2231(value, 'utf-8') value = '%s*=%s' % (name, value) diff --git a/plugin/packages/wakatime/packages/requests/packages/urllib3/packages/backports/__init__.py b/plugin/packages/wakatime/packages/requests/packages/urllib3/packages/backports/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugin/packages/wakatime/packages/requests/packages/urllib3/packages/backports/makefile.py b/plugin/packages/wakatime/packages/requests/packages/urllib3/packages/backports/makefile.py new file mode 100644 index 0000000..75b80dc --- /dev/null +++ b/plugin/packages/wakatime/packages/requests/packages/urllib3/packages/backports/makefile.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +""" +backports.makefile +~~~~~~~~~~~~~~~~~~ + +Backports the Python 3 ``socket.makefile`` method for use with anything that +wants to create a "fake" socket object. +""" +import io + +from socket import SocketIO + + +def backport_makefile(self, mode="r", buffering=None, encoding=None, + errors=None, newline=None): + """ + Backport of ``socket.makefile`` from Python 3.5. + """ + if not set(mode) <= set(["r", "w", "b"]): + raise ValueError( + "invalid mode %r (only r, w, b allowed)" % (mode,) + ) + writing = "w" in mode + reading = "r" in mode or not writing + assert reading or writing + binary = "b" in mode + rawmode = "" + if reading: + rawmode += "r" + if writing: + rawmode += "w" + raw = SocketIO(self, rawmode) + self._makefile_refs += 1 + if buffering is None: + buffering = -1 + if buffering < 0: + buffering = io.DEFAULT_BUFFER_SIZE + if buffering == 0: + if not binary: + raise ValueError("unbuffered streams must be binary") + return raw + if reading and writing: + buffer = io.BufferedRWPair(raw, raw, buffering) + elif reading: + buffer = io.BufferedReader(raw, buffering) + else: + assert writing + buffer = io.BufferedWriter(raw, buffering) + if binary: + return buffer + text = io.TextIOWrapper(buffer, encoding, errors, newline) + text.mode = mode + return text diff --git a/plugin/packages/wakatime/packages/requests/packages/urllib3/packages/ssl_match_hostname/.gitignore b/plugin/packages/wakatime/packages/requests/packages/urllib3/packages/ssl_match_hostname/.gitignore deleted file mode 100644 index 0a764a4..0000000 --- a/plugin/packages/wakatime/packages/requests/packages/urllib3/packages/ssl_match_hostname/.gitignore +++ /dev/null @@ -1 +0,0 @@ -env diff --git a/plugin/packages/wakatime/packages/requests/packages/urllib3/poolmanager.py b/plugin/packages/wakatime/packages/requests/packages/urllib3/poolmanager.py index f13e673..1023dcb 100644 --- a/plugin/packages/wakatime/packages/requests/packages/urllib3/poolmanager.py +++ b/plugin/packages/wakatime/packages/requests/packages/urllib3/poolmanager.py @@ -18,16 +18,16 @@ from .util.retry import Retry __all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] -pool_classes_by_scheme = { - 'http': HTTPConnectionPool, - 'https': HTTPSConnectionPool, -} - log = logging.getLogger(__name__) SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs', 'ssl_version', 'ca_cert_dir') +pool_classes_by_scheme = { + 'http': HTTPConnectionPool, + 'https': HTTPSConnectionPool, +} + class PoolManager(RequestMethods): """ @@ -65,6 +65,9 @@ class PoolManager(RequestMethods): self.pools = RecentlyUsedContainer(num_pools, dispose_func=lambda p: p.close()) + # Locally set the pool classes so other PoolManagers can override them. + self.pool_classes_by_scheme = pool_classes_by_scheme + def __enter__(self): return self @@ -81,7 +84,7 @@ class PoolManager(RequestMethods): by :meth:`connection_from_url` and companion methods. It is intended to be overridden for customization. """ - pool_cls = pool_classes_by_scheme[scheme] + pool_cls = self.pool_classes_by_scheme[scheme] kwargs = self.connection_pool_kw if scheme == 'http': kwargs = self.connection_pool_kw.copy() @@ -186,7 +189,7 @@ class PoolManager(RequestMethods): kw['retries'] = retries kw['redirect'] = redirect - log.info("Redirecting %s -> %s" % (url, redirect_location)) + log.info("Redirecting %s -> %s", url, redirect_location) return self.urlopen(method, redirect_location, **kw) diff --git a/plugin/packages/wakatime/packages/requests/packages/urllib3/response.py b/plugin/packages/wakatime/packages/requests/packages/urllib3/response.py index 8f2a1b5..ac1b2f1 100644 --- a/plugin/packages/wakatime/packages/requests/packages/urllib3/response.py +++ b/plugin/packages/wakatime/packages/requests/packages/urllib3/response.py @@ -221,6 +221,8 @@ class HTTPResponse(io.IOBase): On exit, release the connection back to the pool. """ + clean_exit = False + try: try: yield @@ -243,20 +245,27 @@ class HTTPResponse(io.IOBase): # This includes IncompleteRead. raise ProtocolError('Connection broken: %r' % e, e) - except Exception: - # The response may not be closed but we're not going to use it anymore - # so close it now to ensure that the connection is released back to the pool. - if self._original_response and not self._original_response.isclosed(): - self._original_response.close() - - # Closing the response may not actually be sufficient to close - # everything, so if we have a hold of the connection close that - # too. - if self._connection is not None: - self._connection.close() - - raise + # If no exception is thrown, we should avoid cleaning up + # unnecessarily. + clean_exit = True finally: + # If we didn't terminate cleanly, we need to throw away our + # connection. + if not clean_exit: + # The response may not be closed but we're not going to use it + # anymore so close it now to ensure that the connection is + # released back to the pool. + if self._original_response: + self._original_response.close() + + # Closing the response may not actually be sufficient to close + # everything, so if we have a hold of the connection close that + # too. + if self._connection: + self._connection.close() + + # If we hold the original response but it's closed now, we should + # return the connection back to the pool. if self._original_response and self._original_response.isclosed(): self.release_conn() @@ -387,6 +396,9 @@ class HTTPResponse(io.IOBase): if not self.closed: self._fp.close() + if self._connection: + self._connection.close() + @property def closed(self): if self._fp is None: diff --git a/plugin/packages/wakatime/packages/requests/packages/urllib3/util/__init__.py b/plugin/packages/wakatime/packages/requests/packages/urllib3/util/__init__.py index c6c6243..4778cf9 100644 --- a/plugin/packages/wakatime/packages/requests/packages/urllib3/util/__init__.py +++ b/plugin/packages/wakatime/packages/requests/packages/urllib3/util/__init__.py @@ -6,6 +6,7 @@ from .response import is_fp_closed from .ssl_ import ( SSLContext, HAS_SNI, + IS_PYOPENSSL, assert_fingerprint, resolve_cert_reqs, resolve_ssl_version, @@ -26,6 +27,7 @@ from .url import ( __all__ = ( 'HAS_SNI', + 'IS_PYOPENSSL', 'SSLContext', 'Retry', 'Timeout', diff --git a/plugin/packages/wakatime/packages/requests/packages/urllib3/util/response.py b/plugin/packages/wakatime/packages/requests/packages/urllib3/util/response.py index bc72327..0b5c75c 100644 --- a/plugin/packages/wakatime/packages/requests/packages/urllib3/util/response.py +++ b/plugin/packages/wakatime/packages/requests/packages/urllib3/util/response.py @@ -61,7 +61,7 @@ def assert_header_parsing(headers): def is_response_to_head(response): """ - Checks, wether a the request of a response has been a HEAD-request. + Checks whether the request of a response has been a HEAD-request. Handles the quirks of AppEngine. :param conn: diff --git a/plugin/packages/wakatime/packages/requests/packages/urllib3/util/retry.py b/plugin/packages/wakatime/packages/requests/packages/urllib3/util/retry.py index 03a0124..2d3aa20 100644 --- a/plugin/packages/wakatime/packages/requests/packages/urllib3/util/retry.py +++ b/plugin/packages/wakatime/packages/requests/packages/urllib3/util/retry.py @@ -102,6 +102,11 @@ class Retry(object): :param bool raise_on_redirect: Whether, if the number of redirects is exhausted, to raise a MaxRetryError, or to return a response with a response code in the 3xx range. + + :param bool raise_on_status: Similar meaning to ``raise_on_redirect``: + whether we should raise an exception, or return a response, + if status falls in ``status_forcelist`` range and retries have + been exhausted. """ DEFAULT_METHOD_WHITELIST = frozenset([ @@ -112,7 +117,8 @@ class Retry(object): def __init__(self, total=10, connect=None, read=None, redirect=None, method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None, - backoff_factor=0, raise_on_redirect=True, _observed_errors=0): + backoff_factor=0, raise_on_redirect=True, raise_on_status=True, + _observed_errors=0): self.total = total self.connect = connect @@ -127,6 +133,7 @@ class Retry(object): self.method_whitelist = method_whitelist self.backoff_factor = backoff_factor self.raise_on_redirect = raise_on_redirect + self.raise_on_status = raise_on_status self._observed_errors = _observed_errors # TODO: use .history instead? def new(self, **kw): @@ -137,6 +144,7 @@ class Retry(object): status_forcelist=self.status_forcelist, backoff_factor=self.backoff_factor, raise_on_redirect=self.raise_on_redirect, + raise_on_status=self.raise_on_status, _observed_errors=self._observed_errors, ) params.update(kw) @@ -153,7 +161,7 @@ class Retry(object): redirect = bool(redirect) and None new_retries = cls(retries, redirect=redirect) - log.debug("Converted retries value: %r -> %r" % (retries, new_retries)) + log.debug("Converted retries value: %r -> %r", retries, new_retries) return new_retries def get_backoff_time(self): @@ -272,7 +280,7 @@ class Retry(object): if new_retry.is_exhausted(): raise MaxRetryError(_pool, url, error or ResponseError(cause)) - log.debug("Incremented Retry for (url='%s'): %r" % (url, new_retry)) + log.debug("Incremented Retry for (url='%s'): %r", url, new_retry) return new_retry diff --git a/plugin/packages/wakatime/packages/requests/packages/urllib3/util/ssl_.py b/plugin/packages/wakatime/packages/requests/packages/urllib3/util/ssl_.py index 67f8344..e8d9e7d 100644 --- a/plugin/packages/wakatime/packages/requests/packages/urllib3/util/ssl_.py +++ b/plugin/packages/wakatime/packages/requests/packages/urllib3/util/ssl_.py @@ -12,6 +12,7 @@ from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning SSLContext = None HAS_SNI = False create_default_context = None +IS_PYOPENSSL = False # Maps the length of a digest to a possible hash function producing this digest HASHFUNC_MAP = { @@ -110,11 +111,12 @@ except ImportError: ) self.ciphers = cipher_suite - def wrap_socket(self, socket, server_hostname=None): + def wrap_socket(self, socket, server_hostname=None, server_side=False): warnings.warn( 'A true SSLContext object is not available. This prevents ' 'urllib3 from configuring SSL appropriately and may cause ' - 'certain SSL connections to fail. For more information, see ' + 'certain SSL connections to fail. You can upgrade to a newer ' + 'version of Python to solve this. For more information, see ' 'https://urllib3.readthedocs.org/en/latest/security.html' '#insecureplatformwarning.', InsecurePlatformWarning @@ -125,6 +127,7 @@ except ImportError: 'ca_certs': self.ca_certs, 'cert_reqs': self.verify_mode, 'ssl_version': self.protocol, + 'server_side': server_side, } if self.supports_set_ciphers: # Platform-specific: Python 2.7+ return wrap_socket(socket, ciphers=self.ciphers, **kwargs) @@ -308,8 +311,8 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, 'An HTTPS request has been made, but the SNI (Subject Name ' 'Indication) extension to TLS is not available on this platform. ' 'This may cause the server to present an incorrect TLS ' - 'certificate, which can cause validation failures. For more ' - 'information, see ' + 'certificate, which can cause validation failures. You can upgrade to ' + 'a newer version of Python to solve this. For more information, see ' 'https://urllib3.readthedocs.org/en/latest/security.html' '#snimissingwarning.', SNIMissingWarning diff --git a/plugin/packages/wakatime/packages/requests/sessions.py b/plugin/packages/wakatime/packages/requests/sessions.py index 639668f..d124ea0 100644 --- a/plugin/packages/wakatime/packages/requests/sessions.py +++ b/plugin/packages/wakatime/packages/requests/sessions.py @@ -116,7 +116,6 @@ class SessionRedirectMixin(object): resp.close() url = resp.headers['location'] - method = req.method # Handle redirection without scheme (see: RFC 1808 Section 4) if url.startswith('//'): @@ -140,22 +139,7 @@ class SessionRedirectMixin(object): if resp.is_permanent_redirect and req.url != prepared_request.url: self.redirect_cache[req.url] = prepared_request.url - # http://tools.ietf.org/html/rfc7231#section-6.4.4 - if (resp.status_code == codes.see_other and - method != 'HEAD'): - method = 'GET' - - # Do what the browsers do, despite standards... - # First, turn 302s into GETs. - if resp.status_code == codes.found and method != 'HEAD': - method = 'GET' - - # Second, if a POST is responded to with a 301, turn it into a GET. - # This bizarre behaviour is explained in Issue 1704. - if resp.status_code == codes.moved and method == 'POST': - method = 'GET' - - prepared_request.method = method + self.rebuild_method(prepared_request, resp) # https://github.com/kennethreitz/requests/issues/1084 if resp.status_code not in (codes.temporary_redirect, codes.permanent_redirect): @@ -244,10 +228,10 @@ class SessionRedirectMixin(object): if self.trust_env and not should_bypass_proxies(url): environ_proxies = get_environ_proxies(url) - proxy = environ_proxies.get(scheme) + proxy = environ_proxies.get('all', environ_proxies.get(scheme)) if proxy: - new_proxies.setdefault(scheme, environ_proxies[scheme]) + new_proxies.setdefault(scheme, proxy) if 'Proxy-Authorization' in headers: del headers['Proxy-Authorization'] @@ -262,6 +246,28 @@ class SessionRedirectMixin(object): return new_proxies + def rebuild_method(self, prepared_request, response): + """When being redirected we may want to change the method of the request + based on certain specs or browser behavior. + """ + method = prepared_request.method + + # http://tools.ietf.org/html/rfc7231#section-6.4.4 + if response.status_code == codes.see_other and method != 'HEAD': + method = 'GET' + + # Do what the browsers do, despite standards... + # First, turn 302s into GETs. + if response.status_code == codes.found and method != 'HEAD': + method = 'GET' + + # Second, if a POST is responded to with a 301, turn it into a GET. + # This bizarre behaviour is explained in Issue 1704. + if response.status_code == codes.moved and method == 'POST': + method = 'GET' + + prepared_request.method = method + class Session(SessionRedirectMixin): """A Requests session. @@ -437,6 +443,7 @@ class Session(SessionRedirectMixin): A CA_BUNDLE path can also be provided. Defaults to ``True``. :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. + :rtype: requests.Response """ # Create the Request. req = Request( @@ -550,7 +557,7 @@ class Session(SessionRedirectMixin): # It's possible that users might accidentally send a Request object. # Guard against that specific failure case. - if not isinstance(request, PreparedRequest): + if isinstance(request, Request): raise ValueError('You can only send PreparedRequests.') # Set up variables needed for resolve_redirects and dispatching of hooks diff --git a/plugin/packages/wakatime/packages/requests/status_codes.py b/plugin/packages/wakatime/packages/requests/status_codes.py index a852574..0137c91 100644 --- a/plugin/packages/wakatime/packages/requests/status_codes.py +++ b/plugin/packages/wakatime/packages/requests/status_codes.py @@ -53,6 +53,7 @@ _codes = { 416: ('requested_range_not_satisfiable', 'requested_range', 'range_not_satisfiable'), 417: ('expectation_failed',), 418: ('im_a_teapot', 'teapot', 'i_am_a_teapot'), + 421: ('misdirected_request',), 422: ('unprocessable_entity', 'unprocessable'), 423: ('locked',), 424: ('failed_dependency', 'dependency'), diff --git a/plugin/packages/wakatime/packages/requests/structures.py b/plugin/packages/wakatime/packages/requests/structures.py index 3e5f2fa..991056e 100644 --- a/plugin/packages/wakatime/packages/requests/structures.py +++ b/plugin/packages/wakatime/packages/requests/structures.py @@ -10,6 +10,8 @@ Data structures that power Requests. import collections +from .compat import OrderedDict + class CaseInsensitiveDict(collections.MutableMapping): """ @@ -40,7 +42,7 @@ class CaseInsensitiveDict(collections.MutableMapping): """ def __init__(self, data=None, **kwargs): - self._store = dict() + self._store = OrderedDict() if data is None: data = {} self.update(data, **kwargs) diff --git a/plugin/packages/wakatime/packages/requests/utils.py b/plugin/packages/wakatime/packages/requests/utils.py index c5c3fd0..8d17b6b 100644 --- a/plugin/packages/wakatime/packages/requests/utils.py +++ b/plugin/packages/wakatime/packages/requests/utils.py @@ -14,9 +14,7 @@ import codecs import collections import io import os -import platform import re -import sys import socket import struct import warnings @@ -83,7 +81,14 @@ def super_len(o): ) if hasattr(o, 'tell'): - current_position = o.tell() + try: + current_position = o.tell() + except (OSError, IOError): + # This can happen in some weird situations, such as when the file + # is actually a special file descriptor like stdin. In this + # instance, we don't know what the length is, so set it to zero and + # let requests chunk it instead. + current_position = total_length return max(0, total_length - current_position) @@ -534,6 +539,10 @@ def should_bypass_proxies(url): if is_valid_cidr(proxy_ip): if address_in_network(ip, proxy_ip): return True + elif ip == proxy_ip: + # If no_proxy ip was defined in plain IP notation instead of cidr notation & + # matches the IP of the index + return True else: for host in no_proxy: if netloc.endswith(host) or netloc.split(':')[0].endswith(host): @@ -557,6 +566,7 @@ def should_bypass_proxies(url): return False + def get_environ_proxies(url): """Return a dict of environment proxies.""" if should_bypass_proxies(url): @@ -564,6 +574,7 @@ def get_environ_proxies(url): else: return getproxies() + def select_proxy(url, proxies): """Select a proxy for the url, if applicable. @@ -572,11 +583,24 @@ def select_proxy(url, proxies): """ proxies = proxies or {} urlparts = urlparse(url) - proxy = proxies.get(urlparts.scheme+'://'+urlparts.hostname) - if proxy is None: - proxy = proxies.get(urlparts.scheme) + if urlparts.hostname is None: + return proxies.get('all', proxies.get(urlparts.scheme)) + + proxy_keys = [ + 'all://' + urlparts.hostname, + 'all', + urlparts.scheme + '://' + urlparts.hostname, + urlparts.scheme, + ] + proxy = None + for proxy_key in proxy_keys: + if proxy_key in proxies: + proxy = proxies[proxy_key] + break + return proxy + def default_user_agent(name="python-requests"): """Return a string representing the default user agent.""" return '%s/%s' % (name, __version__) @@ -600,21 +624,19 @@ def parse_header_links(value): links = [] - replace_chars = " '\"" + replace_chars = ' \'"' - for val in re.split(", *<", value): + for val in re.split(', *<', value): try: - url, params = val.split(";", 1) + url, params = val.split(';', 1) except ValueError: url, params = val, '' - link = {} + link = {'url': url.strip('<> \'"')} - link["url"] = url.strip("<> '\"") - - for param in params.split(";"): + for param in params.split(';'): try: - key, value = param.split("=") + key, value = param.split('=') except ValueError: break @@ -661,8 +683,8 @@ def guess_json_utf(data): def prepend_scheme_if_needed(url, new_scheme): - '''Given a URL that may or may not have a scheme, prepend the given scheme. - Does not replace a present scheme with the one provided as an argument.''' + """Given a URL that may or may not have a scheme, prepend the given scheme. + Does not replace a present scheme with the one provided as an argument.""" scheme, netloc, path, params, query, fragment = urlparse(url, new_scheme) # urlparse is a finicky beast, and sometimes decides that there isn't a @@ -693,8 +715,6 @@ def to_native_string(string, encoding='ascii'): string in the native string type, encoding and decoding where necessary. This assumes ASCII unless told otherwise. """ - out = None - if isinstance(string, builtin_str): out = string else: diff --git a/plugin/packages/wakatime/packages/socks.py b/plugin/packages/wakatime/packages/socks.py new file mode 100644 index 0000000..6ffbbad --- /dev/null +++ b/plugin/packages/wakatime/packages/socks.py @@ -0,0 +1,765 @@ +""" +SocksiPy - Python SOCKS module. +Version 1.5.6 + +Copyright 2006 Dan-Haim. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of Dan Haim nor the names of his contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE. + + +This module provides a standard socket-like interface for Python +for tunneling connections through SOCKS proxies. + +=============================================================================== + +Minor modifications made by Christopher Gilbert (http://motomastyle.com/) +for use in PyLoris (http://pyloris.sourceforge.net/) + +Minor modifications made by Mario Vilas (http://breakingcode.wordpress.com/) +mainly to merge bug fixes found in Sourceforge + +Modifications made by Anorov (https://github.com/Anorov) +-Forked and renamed to PySocks +-Fixed issue with HTTP proxy failure checking (same bug that was in the old ___recvall() method) +-Included SocksiPyHandler (sockshandler.py), to be used as a urllib2 handler, + courtesy of e000 (https://github.com/e000): https://gist.github.com/869791#file_socksipyhandler.py +-Re-styled code to make it readable + -Aliased PROXY_TYPE_SOCKS5 -> SOCKS5 etc. + -Improved exception handling and output + -Removed irritating use of sequence indexes, replaced with tuple unpacked variables + -Fixed up Python 3 bytestring handling - chr(0x03).encode() -> b"\x03" + -Other general fixes +-Added clarification that the HTTP proxy connection method only supports CONNECT-style tunneling HTTP proxies +-Various small bug fixes +""" + +__version__ = "1.5.6" + +import socket +import struct +from errno import EOPNOTSUPP, EINVAL, EAGAIN +from io import BytesIO +from os import SEEK_CUR +from collections import Callable +from base64 import b64encode + +PROXY_TYPE_SOCKS4 = SOCKS4 = 1 +PROXY_TYPE_SOCKS5 = SOCKS5 = 2 +PROXY_TYPE_HTTP = HTTP = 3 + +PROXY_TYPES = {"SOCKS4": SOCKS4, "SOCKS5": SOCKS5, "HTTP": HTTP} +PRINTABLE_PROXY_TYPES = dict(zip(PROXY_TYPES.values(), PROXY_TYPES.keys())) + +_orgsocket = _orig_socket = socket.socket + +class ProxyError(IOError): + """ + socket_err contains original socket.error exception. + """ + def __init__(self, msg, socket_err=None): + self.msg = msg + self.socket_err = socket_err + + if socket_err: + self.msg += ": {0}".format(socket_err) + + def __str__(self): + return self.msg + +class GeneralProxyError(ProxyError): pass +class ProxyConnectionError(ProxyError): pass +class SOCKS5AuthError(ProxyError): pass +class SOCKS5Error(ProxyError): pass +class SOCKS4Error(ProxyError): pass +class HTTPError(ProxyError): pass + +SOCKS4_ERRORS = { 0x5B: "Request rejected or failed", + 0x5C: "Request rejected because SOCKS server cannot connect to identd on the client", + 0x5D: "Request rejected because the client program and identd report different user-ids" + } + +SOCKS5_ERRORS = { 0x01: "General SOCKS server failure", + 0x02: "Connection not allowed by ruleset", + 0x03: "Network unreachable", + 0x04: "Host unreachable", + 0x05: "Connection refused", + 0x06: "TTL expired", + 0x07: "Command not supported, or protocol error", + 0x08: "Address type not supported" + } + +DEFAULT_PORTS = { SOCKS4: 1080, + SOCKS5: 1080, + HTTP: 8080 + } + +def set_default_proxy(proxy_type=None, addr=None, port=None, rdns=True, username=None, password=None): + """ + set_default_proxy(proxy_type, addr[, port[, rdns[, username, password]]]) + + Sets a default proxy which all further socksocket objects will use, + unless explicitly changed. All parameters are as for socket.set_proxy(). + """ + socksocket.default_proxy = (proxy_type, addr, port, rdns, + username.encode() if username else None, + password.encode() if password else None) + +setdefaultproxy = set_default_proxy + +def get_default_proxy(): + """ + Returns the default proxy, set by set_default_proxy. + """ + return socksocket.default_proxy + +getdefaultproxy = get_default_proxy + +def wrap_module(module): + """ + Attempts to replace a module's socket library with a SOCKS socket. Must set + a default proxy using set_default_proxy(...) first. + This will only work on modules that import socket directly into the namespace; + most of the Python Standard Library falls into this category. + """ + if socksocket.default_proxy: + module.socket.socket = socksocket + else: + raise GeneralProxyError("No default proxy specified") + +wrapmodule = wrap_module + +def create_connection(dest_pair, proxy_type=None, proxy_addr=None, + proxy_port=None, proxy_rdns=True, + proxy_username=None, proxy_password=None, + timeout=None, source_address=None, + socket_options=None): + """create_connection(dest_pair, *[, timeout], **proxy_args) -> socket object + + Like socket.create_connection(), but connects to proxy + before returning the socket object. + + dest_pair - 2-tuple of (IP/hostname, port). + **proxy_args - Same args passed to socksocket.set_proxy() if present. + timeout - Optional socket timeout value, in seconds. + source_address - tuple (host, port) for the socket to bind to as its source + address before connecting (only for compatibility) + """ + # Remove IPv6 brackets on the remote address and proxy address. + remote_host, remote_port = dest_pair + if remote_host.startswith('['): + remote_host = remote_host.strip('[]') + if proxy_addr and proxy_addr.startswith('['): + proxy_addr = proxy_addr.strip('[]') + + err = None + + # Allow the SOCKS proxy to be on IPv4 or IPv6 addresses. + for r in socket.getaddrinfo(proxy_addr, proxy_port, 0, socket.SOCK_STREAM): + family, socket_type, proto, canonname, sa = r + sock = None + try: + sock = socksocket(family, socket_type, proto) + + if socket_options is not None: + for opt in socket_options: + sock.setsockopt(*opt) + + if isinstance(timeout, (int, float)): + sock.settimeout(timeout) + + if proxy_type is not None: + sock.set_proxy(proxy_type, proxy_addr, proxy_port, proxy_rdns, + proxy_username, proxy_password) + if source_address is not None: + sock.bind(source_address) + + sock.connect((remote_host, remote_port)) + return sock + + except socket.error as e: + err = e + if sock is not None: + sock.close() + sock = None + + if err is not None: + raise err + + raise socket.error("gai returned empty list.") + +class _BaseSocket(socket.socket): + """Allows Python 2's "delegated" methods such as send() to be overridden + """ + def __init__(self, *pos, **kw): + _orig_socket.__init__(self, *pos, **kw) + + self._savedmethods = dict() + for name in self._savenames: + self._savedmethods[name] = getattr(self, name) + delattr(self, name) # Allows normal overriding mechanism to work + + _savenames = list() + +def _makemethod(name): + return lambda self, *pos, **kw: self._savedmethods[name](*pos, **kw) +for name in ("sendto", "send", "recvfrom", "recv"): + method = getattr(_BaseSocket, name, None) + + # Determine if the method is not defined the usual way + # as a function in the class. + # Python 2 uses __slots__, so there are descriptors for each method, + # but they are not functions. + if not isinstance(method, Callable): + _BaseSocket._savenames.append(name) + setattr(_BaseSocket, name, _makemethod(name)) + +class socksocket(_BaseSocket): + """socksocket([family[, type[, proto]]]) -> socket object + + Open a SOCKS enabled socket. The parameters are the same as + those of the standard socket init. In order for SOCKS to work, + you must specify family=AF_INET and proto=0. + The "type" argument must be either SOCK_STREAM or SOCK_DGRAM. + """ + + default_proxy = None + + def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, *args, **kwargs): + if type not in (socket.SOCK_STREAM, socket.SOCK_DGRAM): + msg = "Socket type must be stream or datagram, not {!r}" + raise ValueError(msg.format(type)) + + _BaseSocket.__init__(self, family, type, proto, *args, **kwargs) + self._proxyconn = None # TCP connection to keep UDP relay alive + + if self.default_proxy: + self.proxy = self.default_proxy + else: + self.proxy = (None, None, None, None, None, None) + self.proxy_sockname = None + self.proxy_peername = None + + def _readall(self, file, count): + """ + Receive EXACTLY the number of bytes requested from the file object. + Blocks until the required number of bytes have been received. + """ + data = b"" + while len(data) < count: + d = file.read(count - len(data)) + if not d: + raise GeneralProxyError("Connection closed unexpectedly") + data += d + return data + + def set_proxy(self, proxy_type=None, addr=None, port=None, rdns=True, username=None, password=None): + """set_proxy(proxy_type, addr[, port[, rdns[, username[, password]]]]) + Sets the proxy to be used. + + proxy_type - The type of the proxy to be used. Three types + are supported: PROXY_TYPE_SOCKS4 (including socks4a), + PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP + addr - The address of the server (IP or DNS). + port - The port of the server. Defaults to 1080 for SOCKS + servers and 8080 for HTTP proxy servers. + rdns - Should DNS queries be performed on the remote side + (rather than the local side). The default is True. + Note: This has no effect with SOCKS4 servers. + username - Username to authenticate with to the server. + The default is no authentication. + password - Password to authenticate with to the server. + Only relevant when username is also provided. + """ + self.proxy = (proxy_type, addr, port, rdns, + username.encode() if username else None, + password.encode() if password else None) + + setproxy = set_proxy + + def bind(self, *pos, **kw): + """ + Implements proxy connection for UDP sockets, + which happens during the bind() phase. + """ + proxy_type, proxy_addr, proxy_port, rdns, username, password = self.proxy + if not proxy_type or self.type != socket.SOCK_DGRAM: + return _orig_socket.bind(self, *pos, **kw) + + if self._proxyconn: + raise socket.error(EINVAL, "Socket already bound to an address") + if proxy_type != SOCKS5: + msg = "UDP only supported by SOCKS5 proxy type" + raise socket.error(EOPNOTSUPP, msg) + _BaseSocket.bind(self, *pos, **kw) + + # Need to specify actual local port because + # some relays drop packets if a port of zero is specified. + # Avoid specifying host address in case of NAT though. + _, port = self.getsockname() + dst = ("0", port) + + self._proxyconn = _orig_socket() + proxy = self._proxy_addr() + self._proxyconn.connect(proxy) + + UDP_ASSOCIATE = b"\x03" + _, relay = self._SOCKS5_request(self._proxyconn, UDP_ASSOCIATE, dst) + + # The relay is most likely on the same host as the SOCKS proxy, + # but some proxies return a private IP address (10.x.y.z) + host, _ = proxy + _, port = relay + _BaseSocket.connect(self, (host, port)) + self.proxy_sockname = ("0.0.0.0", 0) # Unknown + + def sendto(self, bytes, *args, **kwargs): + if self.type != socket.SOCK_DGRAM: + return _BaseSocket.sendto(self, bytes, *args, **kwargs) + if not self._proxyconn: + self.bind(("", 0)) + + address = args[-1] + flags = args[:-1] + + header = BytesIO() + RSV = b"\x00\x00" + header.write(RSV) + STANDALONE = b"\x00" + header.write(STANDALONE) + self._write_SOCKS5_address(address, header) + + sent = _BaseSocket.send(self, header.getvalue() + bytes, *flags, **kwargs) + return sent - header.tell() + + def send(self, bytes, flags=0, **kwargs): + if self.type == socket.SOCK_DGRAM: + return self.sendto(bytes, flags, self.proxy_peername, **kwargs) + else: + return _BaseSocket.send(self, bytes, flags, **kwargs) + + def recvfrom(self, bufsize, flags=0): + if self.type != socket.SOCK_DGRAM: + return _BaseSocket.recvfrom(self, bufsize, flags) + if not self._proxyconn: + self.bind(("", 0)) + + buf = BytesIO(_BaseSocket.recv(self, bufsize, flags)) + buf.seek(+2, SEEK_CUR) + frag = buf.read(1) + if ord(frag): + raise NotImplementedError("Received UDP packet fragment") + fromhost, fromport = self._read_SOCKS5_address(buf) + + if self.proxy_peername: + peerhost, peerport = self.proxy_peername + if fromhost != peerhost or peerport not in (0, fromport): + raise socket.error(EAGAIN, "Packet filtered") + + return (buf.read(), (fromhost, fromport)) + + def recv(self, *pos, **kw): + bytes, _ = self.recvfrom(*pos, **kw) + return bytes + + def close(self): + if self._proxyconn: + self._proxyconn.close() + return _BaseSocket.close(self) + + def get_proxy_sockname(self): + """ + Returns the bound IP address and port number at the proxy. + """ + return self.proxy_sockname + + getproxysockname = get_proxy_sockname + + def get_proxy_peername(self): + """ + Returns the IP and port number of the proxy. + """ + return _BaseSocket.getpeername(self) + + getproxypeername = get_proxy_peername + + def get_peername(self): + """ + Returns the IP address and port number of the destination + machine (note: get_proxy_peername returns the proxy) + """ + return self.proxy_peername + + getpeername = get_peername + + def _negotiate_SOCKS5(self, *dest_addr): + """ + Negotiates a stream connection through a SOCKS5 server. + """ + CONNECT = b"\x01" + self.proxy_peername, self.proxy_sockname = self._SOCKS5_request(self, + CONNECT, dest_addr) + + def _SOCKS5_request(self, conn, cmd, dst): + """ + Send SOCKS5 request with given command (CMD field) and + address (DST field). Returns resolved DST address that was used. + """ + proxy_type, addr, port, rdns, username, password = self.proxy + + writer = conn.makefile("wb") + reader = conn.makefile("rb", 0) # buffering=0 renamed in Python 3 + try: + # First we'll send the authentication packages we support. + if username and password: + # The username/password details were supplied to the + # set_proxy method so we support the USERNAME/PASSWORD + # authentication (in addition to the standard none). + writer.write(b"\x05\x02\x00\x02") + else: + # No username/password were entered, therefore we + # only support connections with no authentication. + writer.write(b"\x05\x01\x00") + + # We'll receive the server's response to determine which + # method was selected + writer.flush() + chosen_auth = self._readall(reader, 2) + + if chosen_auth[0:1] != b"\x05": + # Note: string[i:i+1] is used because indexing of a bytestring + # via bytestring[i] yields an integer in Python 3 + raise GeneralProxyError("SOCKS5 proxy server sent invalid data") + + # Check the chosen authentication method + + if chosen_auth[1:2] == b"\x02": + # Okay, we need to perform a basic username/password + # authentication. + writer.write(b"\x01" + chr(len(username)).encode() + + username + + chr(len(password)).encode() + + password) + writer.flush() + auth_status = self._readall(reader, 2) + if auth_status[0:1] != b"\x01": + # Bad response + raise GeneralProxyError("SOCKS5 proxy server sent invalid data") + if auth_status[1:2] != b"\x00": + # Authentication failed + raise SOCKS5AuthError("SOCKS5 authentication failed") + + # Otherwise, authentication succeeded + + # No authentication is required if 0x00 + elif chosen_auth[1:2] != b"\x00": + # Reaching here is always bad + if chosen_auth[1:2] == b"\xFF": + raise SOCKS5AuthError("All offered SOCKS5 authentication methods were rejected") + else: + raise GeneralProxyError("SOCKS5 proxy server sent invalid data") + + # Now we can request the actual connection + writer.write(b"\x05" + cmd + b"\x00") + resolved = self._write_SOCKS5_address(dst, writer) + writer.flush() + + # Get the response + resp = self._readall(reader, 3) + if resp[0:1] != b"\x05": + raise GeneralProxyError("SOCKS5 proxy server sent invalid data") + + status = ord(resp[1:2]) + if status != 0x00: + # Connection failed: server returned an error + error = SOCKS5_ERRORS.get(status, "Unknown error") + raise SOCKS5Error("{0:#04x}: {1}".format(status, error)) + + # Get the bound address/port + bnd = self._read_SOCKS5_address(reader) + return (resolved, bnd) + finally: + reader.close() + writer.close() + + def _write_SOCKS5_address(self, addr, file): + """ + Return the host and port packed for the SOCKS5 protocol, + and the resolved address as a tuple object. + """ + host, port = addr + proxy_type, _, _, rdns, username, password = self.proxy + family_to_byte = {socket.AF_INET: b"\x01", socket.AF_INET6: b"\x04"} + + # If the given destination address is an IP address, we'll + # use the IP address request even if remote resolving was specified. + # Detect whether the address is IPv4/6 directly. + for family in (socket.AF_INET, socket.AF_INET6): + try: + addr_bytes = socket.inet_pton(family, host) + file.write(family_to_byte[family] + addr_bytes) + host = socket.inet_ntop(family, addr_bytes) + file.write(struct.pack(">H", port)) + return host, port + except socket.error: + continue + + # Well it's not an IP number, so it's probably a DNS name. + if rdns: + # Resolve remotely + host_bytes = host.encode('idna') + file.write(b"\x03" + chr(len(host_bytes)).encode() + host_bytes) + else: + # Resolve locally + addresses = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM, socket.IPPROTO_TCP, socket.AI_ADDRCONFIG) + # We can't really work out what IP is reachable, so just pick the + # first. + target_addr = addresses[0] + family = target_addr[0] + host = target_addr[4][0] + + addr_bytes = socket.inet_pton(family, host) + file.write(family_to_byte[family] + addr_bytes) + host = socket.inet_ntop(family, addr_bytes) + file.write(struct.pack(">H", port)) + return host, port + + def _read_SOCKS5_address(self, file): + atyp = self._readall(file, 1) + if atyp == b"\x01": + addr = socket.inet_ntoa(self._readall(file, 4)) + elif atyp == b"\x03": + length = self._readall(file, 1) + addr = self._readall(file, ord(length)) + elif atyp == b"\x04": + addr = socket.inet_ntop(socket.AF_INET6, self._readall(file, 16)) + else: + raise GeneralProxyError("SOCKS5 proxy server sent invalid data") + + port = struct.unpack(">H", self._readall(file, 2))[0] + return addr, port + + def _negotiate_SOCKS4(self, dest_addr, dest_port): + """ + Negotiates a connection through a SOCKS4 server. + """ + proxy_type, addr, port, rdns, username, password = self.proxy + + writer = self.makefile("wb") + reader = self.makefile("rb", 0) # buffering=0 renamed in Python 3 + try: + # Check if the destination address provided is an IP address + remote_resolve = False + try: + addr_bytes = socket.inet_aton(dest_addr) + except socket.error: + # It's a DNS name. Check where it should be resolved. + if rdns: + addr_bytes = b"\x00\x00\x00\x01" + remote_resolve = True + else: + addr_bytes = socket.inet_aton(socket.gethostbyname(dest_addr)) + + # Construct the request packet + writer.write(struct.pack(">BBH", 0x04, 0x01, dest_port)) + writer.write(addr_bytes) + + # The username parameter is considered userid for SOCKS4 + if username: + writer.write(username) + writer.write(b"\x00") + + # DNS name if remote resolving is required + # NOTE: This is actually an extension to the SOCKS4 protocol + # called SOCKS4A and may not be supported in all cases. + if remote_resolve: + writer.write(dest_addr.encode('idna') + b"\x00") + writer.flush() + + # Get the response from the server + resp = self._readall(reader, 8) + if resp[0:1] != b"\x00": + # Bad data + raise GeneralProxyError("SOCKS4 proxy server sent invalid data") + + status = ord(resp[1:2]) + if status != 0x5A: + # Connection failed: server returned an error + error = SOCKS4_ERRORS.get(status, "Unknown error") + raise SOCKS4Error("{0:#04x}: {1}".format(status, error)) + + # Get the bound address/port + self.proxy_sockname = (socket.inet_ntoa(resp[4:]), struct.unpack(">H", resp[2:4])[0]) + if remote_resolve: + self.proxy_peername = socket.inet_ntoa(addr_bytes), dest_port + else: + self.proxy_peername = dest_addr, dest_port + finally: + reader.close() + writer.close() + + def _negotiate_HTTP(self, dest_addr, dest_port): + """ + Negotiates a connection through an HTTP server. + NOTE: This currently only supports HTTP CONNECT-style proxies. + """ + proxy_type, addr, port, rdns, username, password = self.proxy + + # If we need to resolve locally, we do this now + addr = dest_addr if rdns else socket.gethostbyname(dest_addr) + + http_headers = [ + b"CONNECT " + addr.encode('idna') + b":" + str(dest_port).encode() + b" HTTP/1.1", + b"Host: " + dest_addr.encode('idna') + ] + + if username and password: + http_headers.append(b"Proxy-Authorization: basic " + b64encode(username + b":" + password)) + + http_headers.append(b"\r\n") + + self.sendall(b"\r\n".join(http_headers)) + + # We just need the first line to check if the connection was successful + fobj = self.makefile() + status_line = fobj.readline() + fobj.close() + + if not status_line: + raise GeneralProxyError("Connection closed unexpectedly") + + try: + proto, status_code, status_msg = status_line.split(" ", 2) + except ValueError: + raise GeneralProxyError("HTTP proxy server sent invalid response") + + if not proto.startswith("HTTP/"): + raise GeneralProxyError("Proxy server does not appear to be an HTTP proxy") + + try: + status_code = int(status_code) + except ValueError: + raise HTTPError("HTTP proxy server did not return a valid HTTP status") + + if status_code != 200: + error = "{0}: {1}".format(status_code, status_msg) + if status_code in (400, 403, 405): + # It's likely that the HTTP proxy server does not support the CONNECT tunneling method + error += ("\n[*] Note: The HTTP proxy server may not be supported by PySocks" + " (must be a CONNECT tunnel proxy)") + raise HTTPError(error) + + self.proxy_sockname = (b"0.0.0.0", 0) + self.proxy_peername = addr, dest_port + + _proxy_negotiators = { + SOCKS4: _negotiate_SOCKS4, + SOCKS5: _negotiate_SOCKS5, + HTTP: _negotiate_HTTP + } + + + def connect(self, dest_pair): + """ + Connects to the specified destination through a proxy. + Uses the same API as socket's connect(). + To select the proxy server, use set_proxy(). + + dest_pair - 2-tuple of (IP/hostname, port). + """ + if len(dest_pair) != 2 or dest_pair[0].startswith("["): + # Probably IPv6, not supported -- raise an error, and hope + # Happy Eyeballs (RFC6555) makes sure at least the IPv4 + # connection works... + raise socket.error("PySocks doesn't support IPv6") + + dest_addr, dest_port = dest_pair + + if self.type == socket.SOCK_DGRAM: + if not self._proxyconn: + self.bind(("", 0)) + dest_addr = socket.gethostbyname(dest_addr) + + # If the host address is INADDR_ANY or similar, reset the peer + # address so that packets are received from any peer + if dest_addr == "0.0.0.0" and not dest_port: + self.proxy_peername = None + else: + self.proxy_peername = (dest_addr, dest_port) + return + + proxy_type, proxy_addr, proxy_port, rdns, username, password = self.proxy + + # Do a minimal input check first + if (not isinstance(dest_pair, (list, tuple)) + or len(dest_pair) != 2 + or not dest_addr + or not isinstance(dest_port, int)): + raise GeneralProxyError("Invalid destination-connection (host, port) pair") + + + if proxy_type is None: + # Treat like regular socket object + self.proxy_peername = dest_pair + _BaseSocket.connect(self, (dest_addr, dest_port)) + return + + proxy_addr = self._proxy_addr() + + try: + # Initial connection to proxy server + _BaseSocket.connect(self, proxy_addr) + + except socket.error as error: + # Error while connecting to proxy + self.close() + proxy_addr, proxy_port = proxy_addr + proxy_server = "{0}:{1}".format(proxy_addr, proxy_port) + printable_type = PRINTABLE_PROXY_TYPES[proxy_type] + + msg = "Error connecting to {0} proxy {1}".format(printable_type, + proxy_server) + raise ProxyConnectionError(msg, error) + + else: + # Connected to proxy server, now negotiate + try: + # Calls negotiate_{SOCKS4, SOCKS5, HTTP} + negotiate = self._proxy_negotiators[proxy_type] + negotiate(self, dest_addr, dest_port) + except socket.error as error: + # Wrap socket errors + self.close() + raise GeneralProxyError("Socket error", error) + except ProxyError: + # Protocol error while negotiating with proxy + self.close() + raise + + def _proxy_addr(self): + """ + Return proxy address to connect to as tuple object + """ + proxy_type, proxy_addr, proxy_port, rdns, username, password = self.proxy + proxy_port = proxy_port or DEFAULT_PORTS.get(proxy_type) + if not proxy_port: + raise GeneralProxyError("Invalid proxy type") + return proxy_addr, proxy_port diff --git a/plugin/packages/wakatime/packages/sockshandler.py b/plugin/packages/wakatime/packages/sockshandler.py new file mode 100644 index 0000000..26c8343 --- /dev/null +++ b/plugin/packages/wakatime/packages/sockshandler.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +""" +SocksiPy + urllib2 handler + +version: 0.3 +author: e + +This module provides a Handler which you can use with urllib2 to allow it to tunnel your connection through a socks.sockssocket socket, with out monkey patching the original socket... +""" +import ssl + +try: + import urllib2 + import httplib +except ImportError: # Python 3 + import urllib.request as urllib2 + import http.client as httplib + +import socks # $ pip install PySocks + +def merge_dict(a, b): + d = a.copy() + d.update(b) + return d + +class SocksiPyConnection(httplib.HTTPConnection): + def __init__(self, proxytype, proxyaddr, proxyport=None, rdns=True, username=None, password=None, *args, **kwargs): + self.proxyargs = (proxytype, proxyaddr, proxyport, rdns, username, password) + httplib.HTTPConnection.__init__(self, *args, **kwargs) + + def connect(self): + self.sock = socks.socksocket() + self.sock.setproxy(*self.proxyargs) + if type(self.timeout) in (int, float): + self.sock.settimeout(self.timeout) + self.sock.connect((self.host, self.port)) + +class SocksiPyConnectionS(httplib.HTTPSConnection): + def __init__(self, proxytype, proxyaddr, proxyport=None, rdns=True, username=None, password=None, *args, **kwargs): + self.proxyargs = (proxytype, proxyaddr, proxyport, rdns, username, password) + httplib.HTTPSConnection.__init__(self, *args, **kwargs) + + def connect(self): + sock = socks.socksocket() + sock.setproxy(*self.proxyargs) + if type(self.timeout) in (int, float): + sock.settimeout(self.timeout) + sock.connect((self.host, self.port)) + self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file) + +class SocksiPyHandler(urllib2.HTTPHandler, urllib2.HTTPSHandler): + def __init__(self, *args, **kwargs): + self.args = args + self.kw = kwargs + urllib2.HTTPHandler.__init__(self) + + def http_open(self, req): + def build(host, port=None, timeout=0, **kwargs): + kw = merge_dict(self.kw, kwargs) + conn = SocksiPyConnection(*self.args, host=host, port=port, timeout=timeout, **kw) + return conn + return self.do_open(build, req) + + def https_open(self, req): + def build(host, port=None, timeout=0, **kwargs): + kw = merge_dict(self.kw, kwargs) + conn = SocksiPyConnectionS(*self.args, host=host, port=port, timeout=timeout, **kw) + return conn + return self.do_open(build, req) + +if __name__ == "__main__": + import sys + try: + port = int(sys.argv[1]) + except (ValueError, IndexError): + port = 9050 + opener = urllib2.build_opener(SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, "localhost", port)) + print("HTTP: " + opener.open("http://httpbin.org/ip").read().decode()) + print("HTTPS: " + opener.open("https://httpbin.org/ip").read().decode()) diff --git a/plugin/packages/wakatime/projects/git.py b/plugin/packages/wakatime/projects/git.py index fb4f2ac..7f98a4d 100644 --- a/plugin/packages/wakatime/projects/git.py +++ b/plugin/packages/wakatime/projects/git.py @@ -47,12 +47,12 @@ class Git(BaseProject): log.traceback('warn') except IOError: # pragma: nocover log.traceback('warn') - return None + return u('master') def _project_base(self): if self.configFile: return os.path.dirname(os.path.dirname(self.configFile)) - return None + return None # pragma: nocover def _find_git_config_file(self, path): path = os.path.realpath(path)