Skip to content

Fix SSL memory leak#237

Merged
1st1 merged 1 commit into
MagicStack:masterfrom
fantix:fix-bpo34745
Mar 17, 2019
Merged

Fix SSL memory leak#237
1st1 merged 1 commit into
MagicStack:masterfrom
fantix:fix-bpo34745

Conversation

@fantix

@fantix fantix commented Mar 17, 2019

Copy link
Copy Markdown
Member

Refs bpo-34745, protocol and transport form circular reference, causing SSLContext stack up ugly.

Before the fix:

bpo2-uvloop-issue

After the fix:

bpo2-uvloop-fixed

The script to test:

import asyncio
import logging
import ssl
import socket
import sys
import uvloop

CREDENTIAL = ""

URLS = {
    "https://s3.us-west-2.amazonaws.com/archpi.dabase.com/style.css": {
        "method": "get",
        "headers": {
            "User-Agent": "Botocore/1.8.21 Python/3.6.4 Darwin/17.5.0",
            "X-Amz-Date": "20180518T025044Z",
            "X-Amz-Content-SHA256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
            "Authorization": f"AWS4-HMAC-SHA256 Credential={CREDENTIAL}/20180518/us-west-2/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=ae552641b9aa9a7a267fcb4e36960cd5863e55d91c9b45fd39b30fdcd2e81489",
            "Accept-Encoding": "identity",
        },
    },
    "https://s3.ap-southeast-1.amazonaws.com/archpi.dabase.com/doesnotexist": {
        "method": "GET" if sys.argv[1] == "get_object" else "HEAD",
        "headers": {
            "User-Agent": "Botocore/1.8.21 Python/3.6.4 Darwin/17.5.0",
            "X-Amz-Date": "20180518T025221Z",
            "X-Amz-Content-SHA256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
            "Authorization": f"AWS4-HMAC-SHA256 Credential={CREDENTIAL}/20180518/ap-southeast-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=7a7675ef6d70cb647ed59e02d532ffa80d437fb03976d8246ea9ef102d118794",
            "Accept-Encoding": "identity",
        },
    },
}

asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())


class HttpClient(asyncio.streams.FlowControlMixin):
    transport = None

    def __init__(self, *args, **kwargs):
        self.__url = kwargs.pop("url")
        self.__logger = logging.getLogger()
        super().__init__()

    def connection_made(self, transport):
        self.transport = transport

        entry = URLS[self.__url]

        body = "HEAD /archpi.dabase.com/doesnotexist HTTP/1.1\r\nAccept: */*\r\nHost: s3.ap-southeast-1.amazonaws.com\r\n"
        for name, value in entry["headers"].items():
            body += f"{name}: {value}\r\n"

        body += "\r\n"
        self.transport.write(body.encode("ascii"))
        self.__logger.info(f'data sent: {body}')

    def data_received(self, data):
        self.__logger.info(f'data received: {data}')

        self.transport.close()
        # asyncio.get_event_loop().call_later(1.0, )

    def eof_received(self):
        self.__logger.info('eof_received')

    def connection_lost(self, exc):
        self.__logger.info(f'connection lost: {exc}')
        super().connection_lost(exc)

    @classmethod
    def create_factory(cls, url: str):
        def factory(*args, **kwargs):
            return cls(*args, url=url, **kwargs)

        return factory


async def test_asyncio(ssl_context):
    loop = asyncio.get_event_loop()

    url = "https://s3.ap-southeast-1.amazonaws.com/archpi.dabase.com/doesnotexist"
    port = 443
    host = "s3.ap-southeast-1.amazonaws.com"
    infos = await loop.getaddrinfo(host, port, family=socket.AF_INET)
    family, type, proto, canonname, sockaddr = infos[0]
    await loop.create_connection(
        HttpClient.create_factory(url),
        sockaddr[0],
        port,
        ssl=ssl_context,
        family=family,
        proto=proto,
        flags=socket.AI_NUMERICHOST,
        server_hostname=host,
        local_addr=None,
    )


async def asyncio_test():
    ssl_context = ssl.create_default_context()

    while True:
        await test_asyncio(ssl_context)


def main():
    print("running")
    loop = asyncio.get_event_loop()
    for i in range(16):
        loop.create_task(asyncio_test())
    loop.run_forever()


main()

Refs bpo-34745, protocol and transport form circular reference,
causing SSLContext stack up ugly.
@1st1

1st1 commented Mar 17, 2019

Copy link
Copy Markdown
Member

This is great, thank you! BTW, do you want to start porting this implementation to CPython? We almost have no time left until the feature freeze.

@1st1 1st1 merged commit 6c70216 into MagicStack:master Mar 17, 2019
@fantix

fantix commented Mar 17, 2019

Copy link
Copy Markdown
Member Author

@1st1 thanks for the merge! I'll port this fix to CPython today, and yes I've been working on the port of this SSL implementation to CPython, the process is kind of slow due to my limited time, I'm not 100% sure that it could catch up with the 4/28 or 5/26 deadline, but I'll try!

@fantix fantix deleted the fix-bpo34745 branch March 17, 2019 22:48
@fantix

fantix commented Mar 17, 2019

Copy link
Copy Markdown
Member Author

The same fix comitted at python/cpython#12386

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

2 participants