diff --git a/HISTORY.rst b/HISTORY.rst index 6bf01706d..97f4af82e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,6 +4,10 @@ Shotgun Python API Changelog Here you can see the full list of changes between each Python API release. +v3.2.4 (2020 May 25) +===================== +- Updates httplib2 to v0.18.0. + v3.2.3 (2020 Apr 21) ===================== - Fixes an import bug in httplib2 by using the `forked repository `_. diff --git a/README.md b/README.md index 22d25ef4e..747528220 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ Integration and unit tests are provided. - Add bullet points for any changes that have happened since the previous release. This may include changes you did not make so look at the commit history and make sure we don't miss anything. If you notice something was done that wasn't added to the changelog, hunt down that engineer and make them feel guilty for not doing so. This is a required step in making changes to the API. - Try and match the language of previous change log messages. We want to keep a consistent voice. - Make sure the date of the release matches today. We try and keep this TBD until we're ready to do a release so it's easy to catch that it needs to be updated. - - Make sure the version number is filled out and correct. We follow semantic versioning. Or more correctly, we should be following it. + - Make sure the version number is filled out and correct. We follow semantic versioning. 2) Ensure any changes or additions to public methods are documented - Ensure that doc strings are updated in the code itself to work with Sphinx and are correctly formatted. - Examples are always good especially if this a new feature or method. diff --git a/setup.py b/setup.py index dc099f485..f8808a060 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ setup( name='shotgun_api3', - version='3.2.3', + version='3.2.4', description='Shotgun Python API ', long_description=readme, author='Shotgun Software', diff --git a/shotgun_api3/lib/httplib2/README.md b/shotgun_api3/lib/httplib2/README.md deleted file mode 100644 index 0a93f3d71..000000000 --- a/shotgun_api3/lib/httplib2/README.md +++ /dev/null @@ -1,5 +0,0 @@ -Currently this library is pulled from a fork of httplib2 (https://github.com/shotgunsoftware/httplib2) - -The reason for the fork is to make a minor fix to the imports that was causing IronPython2.7 ImportErrors. - -The fix has been submitted to httplib2 repo and already merged so this will likely not be necessary as soon as httplib2 is next released. (>v0.17.2) \ No newline at end of file diff --git a/shotgun_api3/lib/httplib2/python2/__init__.py b/shotgun_api3/lib/httplib2/python2/__init__.py index 6f023417b..26b2c4a82 100644 --- a/shotgun_api3/lib/httplib2/python2/__init__.py +++ b/shotgun_api3/lib/httplib2/python2/__init__.py @@ -19,7 +19,7 @@ "Alex Yu", ] __license__ = "MIT" -__version__ = '0.12.0' +__version__ = "0.18.0" import base64 import calendar @@ -76,7 +76,7 @@ def _ssl_wrap_socket( - sock, key_file, cert_file, disable_validation, ca_certs, ssl_version, hostname + sock, key_file, cert_file, disable_validation, ca_certs, ssl_version, hostname, key_password ): if disable_validation: cert_reqs = ssl.CERT_NONE @@ -90,11 +90,16 @@ def _ssl_wrap_socket( context.verify_mode = cert_reqs context.check_hostname = cert_reqs != ssl.CERT_NONE if cert_file: - context.load_cert_chain(cert_file, key_file) + if key_password: + context.load_cert_chain(cert_file, key_file, key_password) + else: + context.load_cert_chain(cert_file, key_file) if ca_certs: context.load_verify_locations(ca_certs) return context.wrap_socket(sock, server_hostname=hostname) else: + if key_password: + raise NotSupportedOnThisPlatform("Certificate with password is not supported.") return ssl.wrap_socket( sock, keyfile=key_file, @@ -106,7 +111,7 @@ def _ssl_wrap_socket( def _ssl_wrap_socket_unsupported( - sock, key_file, cert_file, disable_validation, ca_certs, ssl_version, hostname + sock, key_file, cert_file, disable_validation, ca_certs, ssl_version, hostname, key_password ): if not disable_validation: raise CertificateValidationUnsupported( @@ -114,6 +119,8 @@ def _ssl_wrap_socket_unsupported( "the ssl module installed. To avoid this error, install " "the ssl module, or explicity disable validation." ) + if key_password: + raise NotSupportedOnThisPlatform("Certificate with password is not supported.") ssl_sock = socket.ssl(sock, key_file, cert_file) return httplib.FakeSocket(sock, ssl_sock) @@ -284,6 +291,12 @@ class NotRunningAppEngineEnvironment(HttpLib2Error): "upgrade", ] +# https://tools.ietf.org/html/rfc7231#section-8.1.3 +SAFE_METHODS = ("GET", "HEAD") # TODO add "OPTIONS", "TRACE" + +# To change, assign to `Http().redirect_codes` +REDIRECT_CODES = frozenset((300, 301, 302, 303, 307, 308)) + def _get_end2end_headers(response): hopbyhop = list(HOP_BY_HOP) @@ -978,8 +991,13 @@ def iter(self, domain): class KeyCerts(Credentials): """Identical to Credentials except that name/password are mapped to key/cert.""" + def add(self, key, cert, domain, password): + self.credentials.append((domain.lower(), key, cert, password)) - pass + def iter(self, domain): + for (cdomain, key, cert, password) in self.credentials: + if cdomain == "" or domain == cdomain: + yield (key, cert, password) class AllHosts(object): @@ -1150,7 +1168,6 @@ def connect(self): raise ProxiesUnavailableError( "Proxy support missing but proxy use was requested!" ) - msg = "getaddrinfo returns an empty list" if self.proxy_info and self.proxy_info.isgood(): use_proxy = True proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers = ( @@ -1165,6 +1182,8 @@ def connect(self): host = self.host port = self.port + socket_err = None + for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM): af, socktype, proto, canonname, sa = res try: @@ -1206,7 +1225,8 @@ def connect(self): self.sock.connect((self.host, self.port) + sa[2:]) else: self.sock.connect(sa) - except socket.error as msg: + except socket.error as e: + socket_err = e if self.debuglevel > 0: print("connect fail: (%s, %s)" % (self.host, self.port)) if use_proxy: @@ -1229,7 +1249,7 @@ def connect(self): continue break if not self.sock: - raise socket.error(msg) + raise socket_err or socket.error("getaddrinfo returns an empty list") class HTTPSConnectionWithTimeout(httplib.HTTPSConnection): @@ -1253,10 +1273,19 @@ def __init__( ca_certs=None, disable_ssl_certificate_validation=False, ssl_version=None, + key_password=None, ): - httplib.HTTPSConnection.__init__( - self, host, port=port, key_file=key_file, cert_file=cert_file, strict=strict - ) + if key_password: + httplib.HTTPSConnection.__init__(self, host, port=port, strict=strict) + self._context.load_cert_chain(cert_file, key_file, key_password) + self.key_file = key_file + self.cert_file = cert_file + self.key_password = key_password + else: + httplib.HTTPSConnection.__init__( + self, host, port=port, key_file=key_file, cert_file=cert_file, strict=strict + ) + self.key_password = None self.timeout = timeout self.proxy_info = proxy_info if ca_certs is None: @@ -1317,7 +1346,6 @@ def _ValidateCertificateHostname(self, cert, hostname): def connect(self): "Connect to a host on a given (SSL) port." - msg = "getaddrinfo returns an empty list" if self.proxy_info and self.proxy_info.isgood(): use_proxy = True proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers = ( @@ -1332,6 +1360,8 @@ def connect(self): host = self.host port = self.port + socket_err = None + address_info = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM) for family, socktype, proto, canonname, sockaddr in address_info: try: @@ -1366,6 +1396,7 @@ def connect(self): self.ca_certs, self.ssl_version, self.host, + self.key_password, ) if self.debuglevel > 0: print("connect: (%s, %s)" % (self.host, self.port)) @@ -1413,7 +1444,8 @@ def connect(self): raise except (socket.timeout, socket.gaierror): raise - except socket.error as msg: + except socket.error as e: + socket_err = e if self.debuglevel > 0: print("connect fail: (%s, %s)" % (self.host, self.port)) if use_proxy: @@ -1436,7 +1468,7 @@ def connect(self): continue break if not self.sock: - raise socket.error(msg) + raise socket_err or socket.error("getaddrinfo returns an empty list") SCHEME_TO_CONNECTION = { @@ -1515,7 +1547,10 @@ def __init__( ca_certs=None, disable_ssl_certificate_validation=False, ssl_version=None, + key_password=None, ): + if key_password: + raise NotSupportedOnThisPlatform("Certificate with password is not supported.") httplib.HTTPSConnection.__init__( self, host, @@ -1632,10 +1667,14 @@ def __init__( # If set to False then no redirects are followed, even safe ones. self.follow_redirects = True + self.redirect_codes = REDIRECT_CODES + # Which HTTP methods do we apply optimistic concurrency to, i.e. # which methods get an "if-match:" etag header added to them. self.optimistic_concurrency_methods = ["PUT", "PATCH"] + self.safe_methods = list(SAFE_METHODS) + # If 'follow_redirects' is True, and this is set to True then # all redirecs are followed, including unsafe ones. self.follow_all_redirects = False @@ -1649,6 +1688,16 @@ def __init__( # Keep Authorization: headers on a redirect. self.forward_authorization_headers = False + def close(self): + """Close persistent connections, clear sensitive data. + Not thread-safe, requires external synchronization against concurrent requests. + """ + existing, self.connections = self.connections, {} + for _, c in existing.iteritems(): + c.close() + self.certificates.clear() + self.clear_credentials() + def __getstate__(self): state_dict = copy.copy(self.__dict__) # In case request is augmented by some foreign object such as @@ -1680,10 +1729,10 @@ def add_credentials(self, name, password, domain=""): any time a request requires authentication.""" self.credentials.add(name, password, domain) - def add_certificate(self, key, cert, domain): + def add_certificate(self, key, cert, domain, password=None): """Add a key and cert that will be used any time a request requires authentication.""" - self.certificates.add(key, cert, domain) + self.certificates.add(key, cert, domain, password) def clear_credentials(self): """Remove all the names and passwords @@ -1819,10 +1868,10 @@ def _request( if ( self.follow_all_redirects - or (method in ["GET", "HEAD"]) - or response.status == 303 + or method in self.safe_methods + or response.status in (303, 308) ): - if self.follow_redirects and response.status in [300, 301, 302, 303, 307]: + if self.follow_redirects and response.status in self.redirect_codes: # Pick out the location header and basically start from the beginning # remembering first to strip the ETag header and decrement our 'depth' if redirections: @@ -1842,7 +1891,7 @@ def _request( response["location"] = urlparse.urljoin( absolute_uri, location ) - if response.status == 301 and method in ["GET", "HEAD"]: + if response.status == 308 or (response.status == 301 and method in self.safe_methods): response["-x-permanent-redirect-url"] = response["location"] if "content-location" not in response: response["content-location"] = absolute_uri @@ -1879,7 +1928,7 @@ def _request( response, content, ) - elif response.status in [200, 203] and method in ["GET", "HEAD"]: + elif response.status in [200, 203] and method in self.safe_methods: # Don't cache 206's since we aren't going to handle byte range requests if "content-location" not in response: response["content-location"] = absolute_uri @@ -1924,6 +1973,8 @@ def request( being and instance of the 'Response' class, the second being a string that contains the response entity body. """ + conn_key = '' + try: if headers is None: headers = {} @@ -1934,6 +1985,9 @@ def request( headers["user-agent"] = "Python-httplib2/%s (gzip)" % __version__ uri = iri2uri(uri) + # Prevent CWE-75 space injection to manipulate request via part of uri. + # Prevent CWE-93 CRLF injection to modify headers via part of uri. + uri = uri.replace(" ", "%20").replace("\r", "%0D").replace("\n", "%0A") (scheme, authority, request_uri, defrag_uri) = urlnorm(uri) @@ -1956,6 +2010,7 @@ def request( ca_certs=self.ca_certs, disable_ssl_certificate_validation=self.disable_ssl_certificate_validation, ssl_version=self.ssl_version, + key_password=certs[0][2], ) else: conn = self.connections[conn_key] = connection_type( @@ -1976,6 +2031,7 @@ def request( headers["accept-encoding"] = "gzip, deflate" info = email.Message.Message() + cachekey = None cached_value = None if self.cache: cachekey = defrag_uri.encode("utf-8") @@ -1996,8 +2052,6 @@ def request( self.cache.delete(cachekey) cachekey = None cached_value = None - else: - cachekey = None if ( method in self.optimistic_concurrency_methods @@ -2009,13 +2063,15 @@ def request( # http://www.w3.org/1999/04/Editing/ headers["if-match"] = info["etag"] - if method not in ["GET", "HEAD"] and self.cache and cachekey: - # RFC 2616 Section 13.10 + # https://tools.ietf.org/html/rfc7234 + # A cache MUST invalidate the effective Request URI as well as [...] Location and Content-Location + # when a non-error status code is received in response to an unsafe request method. + if self.cache and cachekey and method not in self.safe_methods: self.cache.delete(cachekey) # Check the vary header in the cache to see if this request # matches what varies in the cache. - if method in ["GET", "HEAD"] and "vary" in info: + if method in self.safe_methods and "vary" in info: vary = info["vary"] vary_headers = vary.lower().replace(" ", "").split(",") for header in vary_headers: @@ -2026,11 +2082,14 @@ def request( break if ( - cached_value - and method in ["GET", "HEAD"] - and self.cache + self.cache + and cached_value + and (method in self.safe_methods or info["status"] == "308") and "range" not in headers ): + redirect_method = method + if info["status"] not in ("307", "308"): + redirect_method = "GET" if "-x-permanent-redirect-url" in info: # Should cached permanent redirects be counted in our redirection count? For now, yes. if redirections <= 0: @@ -2041,7 +2100,7 @@ def request( ) (response, new_content) = self.request( info["-x-permanent-redirect-url"], - method="GET", + method=redirect_method, headers=headers, redirections=redirections - 1, ) @@ -2133,13 +2192,19 @@ def request( cachekey, ) except Exception as e: + is_timeout = isinstance(e, socket.timeout) + if is_timeout: + conn = self.connections.pop(conn_key, None) + if conn: + conn.close() + if self.force_exception_to_status_code: if isinstance(e, HttpLib2ErrorWithResponse): response = e.response content = e.content response.status = 500 response.reason = str(e) - elif isinstance(e, socket.timeout): + elif is_timeout: content = "Request Timeout" response = Response( { diff --git a/shotgun_api3/lib/httplib2/python2/socks.py b/shotgun_api3/lib/httplib2/python2/socks.py index 5cef77606..71eb4ebf9 100644 --- a/shotgun_api3/lib/httplib2/python2/socks.py +++ b/shotgun_api3/lib/httplib2/python2/socks.py @@ -238,7 +238,15 @@ def setproxy( headers - Additional or modified headers for the proxy connect request. """ - self.__proxy = (proxytype, addr, port, rdns, username, password, headers) + self.__proxy = ( + proxytype, + addr, + port, + rdns, + username.encode() if username else None, + password.encode() if password else None, + headers, + ) def __negotiatesocks5(self, destaddr, destport): """__negotiatesocks5(self,destaddr,destport) diff --git a/shotgun_api3/lib/httplib2/python3/__init__.py b/shotgun_api3/lib/httplib2/python3/__init__.py index e0242df74..3c61aac8a 100644 --- a/shotgun_api3/lib/httplib2/python3/__init__.py +++ b/shotgun_api3/lib/httplib2/python3/__init__.py @@ -15,7 +15,7 @@ "Alex Yu", ] __license__ = "MIT" -__version__ = '0.12.0' +__version__ = "0.18.0" import base64 import calendar @@ -161,6 +161,13 @@ class ProxiesUnavailableError(HttpLib2Error): "upgrade", ] +# https://tools.ietf.org/html/rfc7231#section-8.1.3 +SAFE_METHODS = ("GET", "HEAD", "OPTIONS", "TRACE") + +# To change, assign to `Http().redirect_codes` +REDIRECT_CODES = frozenset((300, 301, 302, 303, 307, 308)) + + from . import certs CA_CERTS = certs.where() @@ -173,9 +180,9 @@ class ProxiesUnavailableError(HttpLib2Error): ssl, "PROTOCOL_SSLv23" ) - def _build_ssl_context( - disable_ssl_certificate_validation, ca_certs, cert_file=None, key_file=None + disable_ssl_certificate_validation, ca_certs, cert_file=None, key_file=None, + maximum_version=None, minimum_version=None, key_password=None, ): if not hasattr(ssl, "SSLContext"): raise RuntimeError("httplib2 requires Python 3.2+ for ssl.SSLContext") @@ -185,6 +192,19 @@ def _build_ssl_context( ssl.CERT_NONE if disable_ssl_certificate_validation else ssl.CERT_REQUIRED ) + # SSLContext.maximum_version and SSLContext.minimum_version are python 3.7+. + # source: https://docs.python.org/3/library/ssl.html#ssl.SSLContext.maximum_version + if maximum_version is not None: + if hasattr(context, "maximum_version"): + context.maximum_version = getattr(ssl.TLSVersion, maximum_version) + else: + raise RuntimeError("setting tls_maximum_version requires Python 3.7 and OpenSSL 1.1 or newer") + if minimum_version is not None: + if hasattr(context, "minimum_version"): + context.minimum_version = getattr(ssl.TLSVersion, minimum_version) + else: + raise RuntimeError("setting tls_minimum_version requires Python 3.7 and OpenSSL 1.1 or newer") + # check_hostname requires python 3.4+ # we will perform the equivalent in HTTPSConnectionWithTimeout.connect() by calling ssl.match_hostname # if check_hostname is not supported. @@ -194,7 +214,7 @@ def _build_ssl_context( context.load_verify_locations(ca_certs) if cert_file: - context.load_cert_chain(cert_file, key_file) + context.load_cert_chain(cert_file, key_file, key_password) return context @@ -302,7 +322,7 @@ def _parse_cache_control(headers): # Whether to use a strict mode to parse WWW-Authenticate headers # Might lead to bad results in case of ill-formed header value, # so disabled by default, falling back to relaxed parsing. -# Set to true to turn on, usefull for testing servers. +# Set to true to turn on, useful for testing servers. USE_WWW_AUTH_STRICT_PARSING = 0 # In regex below: @@ -946,8 +966,13 @@ def iter(self, domain): class KeyCerts(Credentials): """Identical to Credentials except that name/password are mapped to key/cert.""" + def add(self, key, cert, domain, password): + self.credentials.append((domain.lower(), key, cert, password)) - pass + def iter(self, domain): + for (cdomain, key, cert, password) in self.credentials: + if cdomain == "" or domain == cdomain: + yield (key, cert, password) class AllHosts(object): @@ -986,6 +1011,10 @@ def __init__( proxy_headers: Additional or modified headers for the proxy connect request. """ + if isinstance(proxy_user, bytes): + proxy_user = proxy_user.decode() + if isinstance(proxy_pass, bytes): + proxy_pass = proxy_pass.decode() self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, self.proxy_user, self.proxy_pass, self.proxy_headers = ( proxy_type, proxy_host, @@ -1123,7 +1152,7 @@ def connect(self): raise ProxiesUnavailableError( "Proxy support missing but proxy use was requested!" ) - if self.proxy_info and self.proxy_info.isgood(): + if self.proxy_info and self.proxy_info.isgood() and self.proxy_info.applies_to(self.host): use_proxy = True proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers = ( self.proxy_info.astuple() @@ -1226,6 +1255,9 @@ def __init__( proxy_info=None, ca_certs=None, disable_ssl_certificate_validation=False, + tls_maximum_version=None, + tls_minimum_version=None, + key_password=None, ): self.disable_ssl_certificate_validation = disable_ssl_certificate_validation @@ -1236,20 +1268,23 @@ def __init__( self.proxy_info = proxy_info("https") context = _build_ssl_context( - self.disable_ssl_certificate_validation, self.ca_certs, cert_file, key_file + self.disable_ssl_certificate_validation, self.ca_certs, cert_file, key_file, + maximum_version=tls_maximum_version, minimum_version=tls_minimum_version, + key_password=key_password, ) super(HTTPSConnectionWithTimeout, self).__init__( host, port=port, - key_file=key_file, - cert_file=cert_file, timeout=timeout, context=context, ) + self.key_file = key_file + self.cert_file = cert_file + self.key_password = key_password def connect(self): """Connect to a host on a given (SSL) port.""" - if self.proxy_info and self.proxy_info.isgood(): + if self.proxy_info and self.proxy_info.isgood() and self.proxy_info.applies_to(self.host): use_proxy = True proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass, proxy_headers = ( self.proxy_info.astuple() @@ -1331,7 +1366,7 @@ def connect(self): except socket.error as e: socket_err = e if self.debuglevel > 0: - print("connect fail: ({0}, {1})".format((self.host, self.port))) + print("connect fail: ({0}, {1})".format(self.host, self.port)) if use_proxy: print( "proxy: {0}".format( @@ -1384,6 +1419,8 @@ def __init__( proxy_info=proxy_info_from_environment, ca_certs=None, disable_ssl_certificate_validation=False, + tls_maximum_version=None, + tls_minimum_version=None, ): """If 'cache' is a string then it is used as a directory name for a disk cache. Otherwise it must be an object that supports the @@ -1407,10 +1444,15 @@ def __init__( If disable_ssl_certificate_validation is true, SSL cert validation will not be performed. + + tls_maximum_version / tls_minimum_version require Python 3.7+ / + OpenSSL 1.1.0g+. A value of "TLSv1_3" requires OpenSSL 1.1.1+. """ self.proxy_info = proxy_info self.ca_certs = ca_certs self.disable_ssl_certificate_validation = disable_ssl_certificate_validation + self.tls_maximum_version = tls_maximum_version + self.tls_minimum_version = tls_minimum_version # Map domain name to an httplib connection self.connections = {} # The location of the cache, for now a directory @@ -1432,10 +1474,14 @@ def __init__( # If set to False then no redirects are followed, even safe ones. self.follow_redirects = True + self.redirect_codes = REDIRECT_CODES + # Which HTTP methods do we apply optimistic concurrency to, i.e. # which methods get an "if-match:" etag header added to them. self.optimistic_concurrency_methods = ["PUT", "PATCH"] + self.safe_methods = list(SAFE_METHODS) + # If 'follow_redirects' is True, and this is set to True then # all redirecs are followed, including unsafe ones. self.follow_all_redirects = False @@ -1449,6 +1495,16 @@ def __init__( # Keep Authorization: headers on a redirect. self.forward_authorization_headers = False + def close(self): + """Close persistent connections, clear sensitive data. + Not thread-safe, requires external synchronization against concurrent requests. + """ + existing, self.connections = self.connections, {} + for _, c in existing.items(): + c.close() + self.certificates.clear() + self.clear_credentials() + def __getstate__(self): state_dict = copy.copy(self.__dict__) # In case request is augmented by some foreign object such as @@ -1480,10 +1536,10 @@ def add_credentials(self, name, password, domain=""): any time a request requires authentication.""" self.credentials.add(name, password, domain) - def add_certificate(self, key, cert, domain): + def add_certificate(self, key, cert, domain, password=None): """Add a key and cert that will be used any time a request requires authentication.""" - self.certificates.add(key, cert, domain) + self.certificates.add(key, cert, domain, password) def clear_credentials(self): """Remove all the names and passwords @@ -1618,10 +1674,10 @@ def _request( if ( self.follow_all_redirects - or (method in ["GET", "HEAD"]) - or response.status == 303 + or method in self.safe_methods + or response.status in (303, 308) ): - if self.follow_redirects and response.status in [300, 301, 302, 303, 307]: + if self.follow_redirects and response.status in self.redirect_codes: # Pick out the location header and basically start from the beginning # remembering first to strip the ETag header and decrement our 'depth' if redirections: @@ -1641,7 +1697,7 @@ def _request( response["location"] = urllib.parse.urljoin( absolute_uri, location ) - if response.status == 301 and method in ["GET", "HEAD"]: + if response.status == 308 or (response.status == 301 and (method in self.safe_methods)): response["-x-permanent-redirect-url"] = response["location"] if "content-location" not in response: response["content-location"] = absolute_uri @@ -1678,7 +1734,7 @@ def _request( response, content, ) - elif response.status in [200, 203] and method in ["GET", "HEAD"]: + elif response.status in [200, 203] and method in self.safe_methods: # Don't cache 206's since we aren't going to handle byte range requests if "content-location" not in response: response["content-location"] = absolute_uri @@ -1734,6 +1790,9 @@ def request( headers["user-agent"] = "Python-httplib2/%s (gzip)" % __version__ uri = iri2uri(uri) + # Prevent CWE-75 space injection to manipulate request via part of uri. + # Prevent CWE-93 CRLF injection to modify headers via part of uri. + uri = uri.replace(" ", "%20").replace("\r", "%0D").replace("\n", "%0A") (scheme, authority, request_uri, defrag_uri) = urlnorm(uri) @@ -1753,6 +1812,9 @@ def request( proxy_info=self.proxy_info, ca_certs=self.ca_certs, disable_ssl_certificate_validation=self.disable_ssl_certificate_validation, + tls_maximum_version=self.tls_maximum_version, + tls_minimum_version=self.tls_minimum_version, + key_password=certs[0][2], ) else: conn = self.connections[conn_key] = connection_type( @@ -1761,6 +1823,8 @@ def request( proxy_info=self.proxy_info, ca_certs=self.ca_certs, disable_ssl_certificate_validation=self.disable_ssl_certificate_validation, + tls_maximum_version=self.tls_maximum_version, + tls_minimum_version=self.tls_minimum_version, ) else: conn = self.connections[conn_key] = connection_type( @@ -1772,6 +1836,7 @@ def request( headers["accept-encoding"] = "gzip, deflate" info = email.message.Message() + cachekey = None cached_value = None if self.cache: cachekey = defrag_uri @@ -1789,8 +1854,6 @@ def request( self.cache.delete(cachekey) cachekey = None cached_value = None - else: - cachekey = None if ( method in self.optimistic_concurrency_methods @@ -1802,13 +1865,15 @@ def request( # http://www.w3.org/1999/04/Editing/ headers["if-match"] = info["etag"] - if method not in ["GET", "HEAD"] and self.cache and cachekey: - # RFC 2616 Section 13.10 + # https://tools.ietf.org/html/rfc7234 + # A cache MUST invalidate the effective Request URI as well as [...] Location and Content-Location + # when a non-error status code is received in response to an unsafe request method. + if self.cache and cachekey and method not in self.safe_methods: self.cache.delete(cachekey) # Check the vary header in the cache to see if this request # matches what varies in the cache. - if method in ["GET", "HEAD"] and "vary" in info: + if method in self.safe_methods and "vary" in info: vary = info["vary"] vary_headers = vary.lower().replace(" ", "").split(",") for header in vary_headers: @@ -1819,11 +1884,14 @@ def request( break if ( - cached_value - and method in ["GET", "HEAD"] - and self.cache + self.cache + and cached_value + and (method in self.safe_methods or info["status"] == "308") and "range" not in headers ): + redirect_method = method + if info["status"] not in ("307", "308"): + redirect_method = "GET" if "-x-permanent-redirect-url" in info: # Should cached permanent redirects be counted in our redirection count? For now, yes. if redirections <= 0: @@ -1834,7 +1902,7 @@ def request( ) (response, new_content) = self.request( info["-x-permanent-redirect-url"], - method="GET", + method=redirect_method, headers=headers, redirections=redirections - 1, ) @@ -2009,4 +2077,4 @@ def __getattr__(self, name): if name == "dict": return self else: - raise AttributeError(name) \ No newline at end of file + raise AttributeError(name) diff --git a/shotgun_api3/lib/httplib2/python3/cacerts.txt b/shotgun_api3/lib/httplib2/python3/cacerts.txt index a2a9833de..8020c1b4d 100644 --- a/shotgun_api3/lib/httplib2/python3/cacerts.txt +++ b/shotgun_api3/lib/httplib2/python3/cacerts.txt @@ -2194,3 +2194,4 @@ Fdtom/DzMNU+MeKNhJ7jitralj41E6Vf8PlwUHBHQRFXGU7Aj64GxJUTFy8bJZ91 pLiaWN0bfVKfjllDiIGknibVb63dDcY3fe0Dkhvld1927jyNxF1WW6LZZm6zNTfl MrY= -----END CERTIFICATE----- + diff --git a/shotgun_api3/lib/httplib2/python3/certs.py b/shotgun_api3/lib/httplib2/python3/certs.py index f8fcdd8eb..59d1ffc70 100644 --- a/shotgun_api3/lib/httplib2/python3/certs.py +++ b/shotgun_api3/lib/httplib2/python3/certs.py @@ -39,4 +39,4 @@ def where(): if __name__ == "__main__": - print(where()) \ No newline at end of file + print(where()) diff --git a/shotgun_api3/lib/httplib2/python3/iri2uri.py b/shotgun_api3/lib/httplib2/python3/iri2uri.py index 439de7342..86e361e62 100644 --- a/shotgun_api3/lib/httplib2/python3/iri2uri.py +++ b/shotgun_api3/lib/httplib2/python3/iri2uri.py @@ -121,4 +121,4 @@ def test_iri(self): ), ) - unittest.main() \ No newline at end of file + unittest.main() diff --git a/shotgun_api3/lib/httplib2/python3/socks.py b/shotgun_api3/lib/httplib2/python3/socks.py index d5e5433a8..cc68e634c 100644 --- a/shotgun_api3/lib/httplib2/python3/socks.py +++ b/shotgun_api3/lib/httplib2/python3/socks.py @@ -206,13 +206,7 @@ def __rewriteproxy(self, header): return "\r\n".join(hdrs) def __getauthheader(self): - username = self.__proxy[4] - password = self.__proxy[5] - if isinstance(username, str): - username = username.encode() - if isinstance(password, str): - password = password.encode() - auth = username + b":" + password + auth = self.__proxy[4] + b":" + self.__proxy[5] return "Proxy-Authorization: Basic " + base64.b64encode(auth).decode() def setproxy( @@ -244,7 +238,15 @@ def setproxy( headers - Additional or modified headers for the proxy connect request. """ - self.__proxy = (proxytype, addr, port, rdns, username, password, headers) + self.__proxy = ( + proxytype, + addr, + port, + rdns, + username.encode() if username else None, + password.encode() if password else None, + headers, + ) def __negotiatesocks5(self, destaddr, destport): """__negotiatesocks5(self,destaddr,destport) @@ -273,13 +275,13 @@ def __negotiatesocks5(self, destaddr, destport): elif chosenauth[1:2] == chr(0x02).encode(): # Okay, we need to perform a basic username/password # authentication. - self.sendall( - chr(0x01).encode() - + chr(len(self.__proxy[4])) - + self.__proxy[4] - + chr(len(self.__proxy[5])) - + self.__proxy[5] - ) + packet = bytearray() + packet.append(0x01) + packet.append(len(self.__proxy[4])) + packet.extend(self.__proxy[4]) + packet.append(len(self.__proxy[5])) + packet.extend(self.__proxy[5]) + self.sendall(packet) authstat = self.__recvall(2) if authstat[0:1] != chr(0x01).encode(): # Bad response @@ -513,4 +515,4 @@ def connect(self, destpair): elif self.__proxy[0] == None: _orgsocket.connect(self, (destpair[0], destpair[1])) else: - raise GeneralProxyError((4, _generalerrors[4])) \ No newline at end of file + raise GeneralProxyError((4, _generalerrors[4])) diff --git a/shotgun_api3/lib/requirements.txt b/shotgun_api3/lib/requirements.txt index 15cf82712..e0eef25cf 100644 --- a/shotgun_api3/lib/requirements.txt +++ b/shotgun_api3/lib/requirements.txt @@ -28,5 +28,5 @@ # This file is unused. It is left there so Github can warn us is a CVE is # released for our dependencies. -httplib2==0.12.0 +httplib2==0.18.0 six==1.12.0 \ No newline at end of file diff --git a/shotgun_api3/shotgun.py b/shotgun_api3/shotgun.py index 3f259fd26..29341e988 100644 --- a/shotgun_api3/shotgun.py +++ b/shotgun_api3/shotgun.py @@ -117,7 +117,7 @@ def _is_mimetypes_broken(): # ---------------------------------------------------------------------------- # Version -__version__ = "3.2.3" +__version__ = "3.2.4" # ---------------------------------------------------------------------------- # Errors diff --git a/update_httplib2.py b/update_httplib2.py new file mode 100755 index 000000000..b8d526f06 --- /dev/null +++ b/update_httplib2.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +""" +Updates the httplib2 module. + +Run as "./upgrade_httplib2.py vX.Y.Z" to get a specific release from github. +""" + +import pathlib +import tempfile +import shutil +import subprocess +import sys + +def main(temp_path, repo_root, version): + # Output folders for the python2 and python3 copies of httplib2 + httplib2_dir = repo_root / "shotgun_api3" / "lib" / "httplib2" + python2_dir = str(httplib2_dir / "python2") + python3_dir = str(httplib2_dir / "python3") + + file_name = f"{version}.zip" + # Downloads the archive from github. + print(f"Downloading {file_name}") + file_path = temp_path / file_name + subprocess.check_output(["curl", "-L", f"https://github.com/httplib2/httplib2/archive/{file_name}", "-o", file_path]) + + # Unzips in a temp dir. + print(f"Unzipping {file_name}") + unzipped_folder = temp_path / "unzipped" + unzipped_folder.mkdir() + subprocess.check_output(["unzip", str(file_path), "-d", str(unzipped_folder)]) + shutil.rmtree(python2_dir) + shutil.rmtree(python3_dir) + + # Removes the previous version of httplib2 + print("Removing previous version of httplib2") + subprocess.check_output(["git", "rm", "-rf", str(python2_dir), str(python3_dir)]) + + # Copies a new version into place. + print("Copying new version of httplib2") + root_folder = unzipped_folder / f"httplib2-{version[1:]}" + shutil.copytree(str(root_folder / "python2" / "httplib2"), python2_dir) + shutil.copytree(str(root_folder / "python3" / "httplib2"), python3_dir) + shutil.rmtree(f"{python2_dir}/test") + shutil.rmtree(f"{python3_dir}/test") + + # Patches the httplib2 imports so they are relative instead of absolute. + print("Patching imports") + for python_file in httplib2_dir.rglob("*.py"): + subprocess.check_output( + ["sed", "-i", "", "-e" "s/from httplib2/from ./g", python_file] + ) + + # Adding files to the git repo. + print("Adding to git") + subprocess.check_output(["git", "add", str(python2_dir), str(python3_dir)]) + + +try: + temp_path = pathlib.Path(tempfile.mkdtemp()) + main(temp_path, pathlib.Path(__file__).parent, sys.argv[1]) +finally: + shutil.rmtree(temp_path)