From a526cc7c98059bf9f5edf07f06c4c68a026150fb Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 18 May 2018 22:19:58 +0200 Subject: [PATCH] bpo-33618: Enable TLS 1.3 in tests TLS 1.3 behaves slightly different than TLS 1.2. Session tickets and TLS client cert auth are now handled after the initialy handshake. Tests now either send/recv data to trigger session and client certs. Or tests ignore ConnectionResetError / BrokenPipeError on the server side to handle clients that force-close the socket fd. To test TLS 1.3, OpenSSL 1.1.1-pre7-dev (git master + OpenSSL PR https://github.com/openssl/openssl/pull/6340) is required. Signed-off-by: Christian Heimes --- Doc/library/ssl.rst | 28 +++++- Doc/whatsnew/3.7.rst | 12 ++- Lib/test/test_asyncio/test_sslproto.py | 2 + Lib/test/test_asyncio/utils.py | 2 - Lib/test/test_ftplib.py | 10 +-- Lib/test/test_poplib.py | 9 +- Lib/test/test_ssl.py | 90 +++++++++++++++---- .../2018-05-23-20-14-34.bpo-33618.xU39lr.rst | 2 + Tools/ssl/multissltests.py | 33 +++++-- 9 files changed, 142 insertions(+), 46 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2018-05-23-20-14-34.bpo-33618.xU39lr.rst diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 2ccea13b614210c..14eac2c589479d9 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -2587,7 +2587,33 @@ successful call of :func:`~ssl.RAND_add`, :func:`~ssl.RAND_bytes` or :func:`~ssl.RAND_pseudo_bytes` is sufficient. -.. ssl-libressl: +.. _ssl-tlsv1_3: + +TLS 1.3 +------- + +.. versionadded:: 3.7 + +Python has provisional and experimental support for TLS 1.3 with OpenSSL +1.1.1. The new protocol behaves slightly differently than previous version +of TLS/SSL. Some new TLS 1.3 features are not yet available. + +- TLS 1.3 uses a disjunct set of cipher suites. All AES-GCM and + ChaCha20 cipher suites are enabled by default. The method + :meth:`SSLContext.set_ciphers` cannot enable or disable any TLS 1.3 + ciphers yet, but :meth:`SSLContext.get_cipers` returns them. +- Session tickets are no longer sent as part of the initial handshake and + are handled differently. :attr:`SSLSocket.session` and :class:`SSLSession` + are not compatible with TLS 1.3. +- Client-side certificates are also no longer verified during the initial + handshake. A server can request a certificate at any time. Clients + process certificate requests while they send or receive application data + from the server. +- TLS 1.3 features like early data, deferred TLS client cert request, + signature algorithm configuration, and rekeying are not supported yet. + + +.. _ssl-libressl: LibreSSL support ---------------- diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index af2aad9e81e7df7..46015af3e73952e 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -1244,8 +1244,8 @@ Host name validation can be customized with .. note:: The improved host name check requires a *libssl* implementation compatible with OpenSSL 1.0.2 or 1.1. Consequently, OpenSSL 0.9.8 and 1.0.1 are no - longer supported and LibreSSL is temporarily not supported until it gains - the necessary OpenSSL 1.0.2 APIs. + longer supported. The ssl module is mostly compatible with LibreSSL 2.7.2 + and newer. The ``ssl`` module no longer sends IP addresses in SNI TLS extension. (Contributed by Christian Heimes in :issue:`32185`.) @@ -1270,8 +1270,12 @@ rather than the U-label form (``"pythön.org"``). (Contributed by Nathaniel J. Smith and Christian Heimes in :issue:`28414`.) The ``ssl`` module has preliminary and experimental support for TLS 1.3 and -OpenSSL 1.1.1. (Contributed by Christian Heimes in :issue:`32947`, -:issue:`20995`, :issue:`29136`, and :issue:`30622`) +OpenSSL 1.1.1. At the time of Python 3.7.0 release, OpenSSL 1.1.1 is still +under development and TLS 1.3 hasn't been finalized yet. The TLS 1.3 +handshake and protocol behaves slightly differently than TLS 1.2 and earlier, +see :ref:`ssl-tlsv1_3`. +(Contributed by Christian Heimes in :issue:`32947`, :issue:`20995`, +:issue:`29136`, :issue:`30622` and :issue:`33618`) :class:`~ssl.SSLSocket` and :class:`~ssl.SSLObject` no longer have a public constructor. Direct instantiation was never a documented and supported diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index 7b27f4cfa3223a8..c534a341352b00f 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -251,6 +251,8 @@ def test_start_tls_server_1(self): server_context = test_utils.simple_server_sslcontext() client_context = test_utils.simple_client_sslcontext() + # TODO: fix TLSv1.3 support + client_context.options |= ssl.OP_NO_TLSv1_3 def client(sock, addr): time.sleep(0.5) diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index 711085fde5c5b1c..96dfe2f85b4de11 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -74,8 +74,6 @@ def simple_server_sslcontext(): server_context.load_cert_chain(ONLYCERT, ONLYKEY) server_context.check_hostname = False server_context.verify_mode = ssl.CERT_NONE - # TODO: fix TLSv1.3 support - server_context.options |= ssl.OP_NO_TLSv1_3 return server_context diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index 1a8e2f91d386dfb..f9488a96cfcaaaf 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -257,6 +257,7 @@ class DummyFTPServer(asyncore.dispatcher, threading.Thread): def __init__(self, address, af=socket.AF_INET): threading.Thread.__init__(self) asyncore.dispatcher.__init__(self) + self.daemon = True self.create_socket(af, socket.SOCK_STREAM) self.bind(address) self.listen(5) @@ -312,8 +313,6 @@ class SSLConnection(asyncore.dispatcher): def secure_connection(self): context = ssl.SSLContext() - # TODO: fix TLSv1.3 support - context.options |= ssl.OP_NO_TLSv1_3 context.load_cert_chain(CERTFILE) socket = context.wrap_socket(self.socket, suppress_ragged_eofs=False, @@ -405,7 +404,7 @@ def handle_error(self): def close(self): if (isinstance(self.socket, ssl.SSLSocket) and - self.socket._sslobj is not None): + self.socket._sslobj is not None): self._do_ssl_shutdown() else: super(SSLConnection, self).close() @@ -910,8 +909,6 @@ def test_auth_issued_twice(self): def test_context(self): self.client.quit() ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - # TODO: fix TLSv1.3 support - ctx.options |= ssl.OP_NO_TLSv1_3 ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE self.assertRaises(ValueError, ftplib.FTP_TLS, keyfile=CERTFILE, @@ -944,8 +941,6 @@ def test_ccc(self): def test_check_hostname(self): self.client.quit() ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - # TODO: fix TLSv1.3 support - ctx.options |= ssl.OP_NO_TLSv1_3 self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) self.assertEqual(ctx.check_hostname, True) ctx.load_verify_locations(CAFILE) @@ -982,6 +977,7 @@ def setUp(self): self.sock.settimeout(20) self.port = support.bind_port(self.sock) self.server_thread = threading.Thread(target=self.server) + self.server_thread.daemon = True self.server_thread.start() # Wait for the server to be ready. self.evt.wait() diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index bbedbbdb10a1048..20d4eeac12d4b2f 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -153,8 +153,6 @@ def cmd_stls(self, arg): if self.tls_active is False: self.push('+OK Begin TLS negotiation') context = ssl.SSLContext() - # TODO: fix TLSv1.3 support - context.options |= ssl.OP_NO_TLSv1_3 context.load_cert_chain(CERTFILE) tls_sock = context.wrap_socket(self.socket, server_side=True, @@ -206,6 +204,7 @@ class DummyPOP3Server(asyncore.dispatcher, threading.Thread): def __init__(self, address, af=socket.AF_INET): threading.Thread.__init__(self) asyncore.dispatcher.__init__(self) + self.daemon = True self.create_socket(af, socket.SOCK_STREAM) self.bind(address) self.listen(5) @@ -370,8 +369,6 @@ def test_stls(self): def test_stls_context(self): expected = b'+OK Begin TLS negotiation' ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - # TODO: fix TLSv1.3 support - ctx.options |= ssl.OP_NO_TLSv1_3 ctx.load_verify_locations(CAFILE) self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) self.assertEqual(ctx.check_hostname, True) @@ -412,8 +409,6 @@ def test__all__(self): def test_context(self): ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - # TODO: fix TLSv1.3 support - ctx.options |= ssl.OP_NO_TLSv1_3 ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE self.assertRaises(ValueError, poplib.POP3_SSL, self.server.host, @@ -482,7 +477,7 @@ def setUp(self): self.sock.settimeout(60) # Safety net. Look issue 11812 self.port = test_support.bind_port(self.sock) self.thread = threading.Thread(target=self.server, args=(self.evt,self.sock)) - self.thread.setDaemon(True) + self.thread.daemon = True self.thread.start() self.evt.wait() diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 03952b10a8e7919..73d3e3bbcdaeb80 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1826,6 +1826,7 @@ def test_connect_capath(self): s.connect(self.server_addr) cert = s.getpeercert() self.assertTrue(cert) + # Same with a bytes `capath` argument ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) ctx.verify_mode = ssl.CERT_REQUIRED @@ -1841,8 +1842,6 @@ def test_connect_cadata(self): der = ssl.PEM_cert_to_DER_cert(pem) ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) ctx.verify_mode = ssl.CERT_REQUIRED - # TODO: fix TLSv1.3 support - ctx.options |= ssl.OP_NO_TLSv1_3 ctx.load_verify_locations(cadata=pem) with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: s.connect(self.server_addr) @@ -1852,8 +1851,6 @@ def test_connect_cadata(self): # same with DER ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) ctx.verify_mode = ssl.CERT_REQUIRED - # TODO: fix TLSv1.3 support - ctx.options |= ssl.OP_NO_TLSv1_3 ctx.load_verify_locations(cadata=der) with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: s.connect(self.server_addr) @@ -2109,11 +2106,21 @@ def wrap_conn(self): self.sock, server_side=True) self.server.selected_npn_protocols.append(self.sslconn.selected_npn_protocol()) self.server.selected_alpn_protocols.append(self.sslconn.selected_alpn_protocol()) - except (ssl.SSLError, ConnectionResetError, OSError) as e: + except (ConnectionResetError, BrokenPipeError) as e: # We treat ConnectionResetError as though it were an # SSLError - OpenSSL on Ubuntu abruptly closes the # connection when asked to use an unsupported protocol. # + # BrokenPipeError is raised in TLS 1.3 mode, when OpenSSL + # tries to send session tickets after handshake. + # https://github.com/openssl/openssl/issues/6342 + self.server.conn_errors.append(str(e)) + if self.server.chatty: + handle_error("\n server: bad connection attempt from " + repr(self.addr) + ":\n") + self.running = False + self.close() + return False + except (ssl.SSLError, OSError) as e: # OSError may occur with wrong protocols, e.g. both # sides use PROTOCOL_TLS_SERVER. # @@ -2220,11 +2227,22 @@ def run(self): sys.stdout.write(" server: read %r (%s), sending back %r (%s)...\n" % (msg, ctype, msg.lower(), ctype)) self.write(msg.lower()) + except ConnectionResetError: + # XXX: OpenSSL 1.1.1 sometimes raises ConnectionResetError + # when connection is not shut down gracefully. + if self.server.chatty and support.verbose: + sys.stdout.write( + " Connection reset by peer: {}\n".format( + self.addr) + ) + self.close() + self.running = False except OSError: if self.server.chatty: handle_error("Test server failure:\n") self.close() self.running = False + # normally, we'd just stop here, but for the test # harness, we want to stop the server self.server.stop() @@ -2299,6 +2317,11 @@ def run(self): pass except KeyboardInterrupt: self.stop() + except BaseException as e: + if support.verbose and self.chatty: + sys.stdout.write( + ' connection handling failed: ' + repr(e) + '\n') + self.sock.close() def stop(self): @@ -2745,8 +2768,6 @@ def test_check_hostname_idn(self): server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) server_context.load_cert_chain(IDNSANSFILE) - # TODO: fix TLSv1.3 support - server_context.options |= ssl.OP_NO_TLSv1_3 context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) context.verify_mode = ssl.CERT_REQUIRED @@ -2797,7 +2818,7 @@ def test_check_hostname_idn(self): with self.assertRaises(ssl.CertificateError): s.connect((HOST, server.port)) - def test_wrong_cert(self): + def test_wrong_cert_tls12(self): """Connecting when the server rejects the client's certificate Launch a server with CERT_REQUIRED, and check that trying to @@ -2808,9 +2829,8 @@ def test_wrong_cert(self): client_context.load_cert_chain(WRONG_CERT) # require TLS client authentication server_context.verify_mode = ssl.CERT_REQUIRED - # TODO: fix TLSv1.3 support - # With TLS 1.3, test fails with exception in server thread - server_context.options |= ssl.OP_NO_TLSv1_3 + # TLS 1.3 has different handshake + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 server = ThreadedEchoServer( context=server_context, chatty=True, connectionchatty=True, @@ -2835,6 +2855,36 @@ def test_wrong_cert(self): else: self.fail("Use of invalid cert should have failed!") + @unittest.skipUnless(ssl.HAS_TLSv1_3, "Test needs TLS 1.3") + def test_wrong_cert_tls13(self): + client_context, server_context, hostname = testing_context() + client_context.load_cert_chain(WRONG_CERT) + server_context.verify_mode = ssl.CERT_REQUIRED + server_context.minimum_version = ssl.TLSVersion.TLSv1_3 + client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + + server = ThreadedEchoServer( + context=server_context, chatty=True, connectionchatty=True, + ) + with server, \ + client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + # TLS 1.3 perform client cert exchange after handshake + s.connect((HOST, server.port)) + try: + s.write(b'data') + s.read(4) + except ssl.SSLError as e: + if support.verbose: + sys.stdout.write("\nSSLError is %r\n" % e) + except OSError as e: + if e.errno != errno.ECONNRESET: + raise + if support.verbose: + sys.stdout.write("\nsocket.error is %r\n" % e) + else: + self.fail("Use of invalid cert should have failed!") + def test_rude_shutdown(self): """A brutal shutdown of an SSL server should raise an OSError in the client when attempting handshake. @@ -3405,7 +3455,7 @@ def serve(): # Block on the accept and wait on the connection to close. evt.set() remote, peer = server.accept() - remote.recv(1) + remote.send(remote.recv(4)) t = threading.Thread(target=serve) t.start() @@ -3413,6 +3463,8 @@ def serve(): evt.wait() client = context.wrap_socket(socket.socket()) client.connect((host, port)) + client.send(b'data') + client.recv() client_addr = client.getsockname() client.close() t.join() @@ -3465,7 +3517,7 @@ def test_version_basic(self): self.assertIs(s.version(), None) self.assertIs(s._sslobj, None) s.connect((HOST, server.port)) - if ssl.OPENSSL_VERSION_INFO >= (1, 1, 1): + if IS_OPENSSL_1_1_1 and ssl.HAS_TLSv1_3: self.assertEqual(s.version(), 'TLSv1.3') elif ssl.OPENSSL_VERSION_INFO >= (1, 0, 2): self.assertEqual(s.version(), 'TLSv1.2') @@ -3574,8 +3626,6 @@ def test_tls_unique_channel_binding(self): sys.stdout.write("\n") client_context, server_context, hostname = testing_context() - # TODO: fix TLSv1.3 support - client_context.options |= ssl.OP_NO_TLSv1_3 server = ThreadedEchoServer(context=server_context, chatty=True, @@ -3594,7 +3644,10 @@ def test_tls_unique_channel_binding(self): # check if it is sane self.assertIsNotNone(cb_data) - self.assertEqual(len(cb_data), 12) # True for TLSv1 + if s.version() == 'TLSv1.3': + self.assertEqual(len(cb_data), 48) + else: + self.assertEqual(len(cb_data), 12) # True for TLSv1 # and compare with the peers version s.write(b"CB tls-unique\n") @@ -3616,7 +3669,10 @@ def test_tls_unique_channel_binding(self): # is it really unique self.assertNotEqual(cb_data, new_cb_data) self.assertIsNotNone(cb_data) - self.assertEqual(len(cb_data), 12) # True for TLSv1 + if s.version() == 'TLSv1.3': + self.assertEqual(len(cb_data), 48) + else: + self.assertEqual(len(cb_data), 12) # True for TLSv1 s.write(b"CB tls-unique\n") peer_data_repr = s.read().strip() self.assertEqual(peer_data_repr, diff --git a/Misc/NEWS.d/next/Library/2018-05-23-20-14-34.bpo-33618.xU39lr.rst b/Misc/NEWS.d/next/Library/2018-05-23-20-14-34.bpo-33618.xU39lr.rst new file mode 100644 index 000000000000000..6cc2452b145c5ba --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-05-23-20-14-34.bpo-33618.xU39lr.rst @@ -0,0 +1,2 @@ +Finalize and document preliminary and experimental TLS 1.3 support with +OpenSSL 1.1.1 diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index bbc5c66520989c6..c4ebe317797d2b4 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -47,7 +47,7 @@ OPENSSL_RECENT_VERSIONS = [ "1.0.2o", "1.1.0h", - "1.1.1-pre6", + # "1.1.1-pre7", ] LIBRESSL_OLD_VERSIONS = [ @@ -73,7 +73,7 @@ parser.add_argument( '--debug', action='store_true', - help="Enable debug mode", + help="Enable debug logging", ) parser.add_argument( '--disable-ancient', @@ -130,6 +130,18 @@ default='', help="Override the automatic system type detection." ) +parser.add_argument( + '--force', + action='store_true', + dest='force', + help="Force build and installation." +) +parser.add_argument( + '--keep-sources', + action='store_true', + dest='keep_sources', + help="Keep original sources for debugging." +) class AbstractBuilder(object): @@ -260,26 +272,31 @@ def _build_src(self): """Now build openssl""" log.info("Running build in {}".format(self.build_dir)) cwd = self.build_dir - cmd = ["./config", "shared", "--prefix={}".format(self.install_dir)] - env = None + cmd = [ + "./config", + "shared", "--debug", + "--prefix={}".format(self.install_dir) + ] + env = os.environ.copy() + # set rpath + env["LD_RUN_PATH"] = self.lib_dir if self.system: - env = os.environ.copy() env['SYSTEM'] = self.system self._subprocess_call(cmd, cwd=cwd, env=env) # Old OpenSSL versions do not support parallel builds. self._subprocess_call(["make", "-j1"], cwd=cwd, env=env) - def _make_install(self, remove=True): + def _make_install(self): self._subprocess_call( ["make", "-j1", self.install_target], cwd=self.build_dir ) - if remove: + if not self.args.keep_sources: shutil.rmtree(self.build_dir) def install(self): log.info(self.openssl_cli) - if not self.has_openssl: + if not self.has_openssl or self.args.force: if not self.has_src: self._download_src() else: