From f431d86e8e26915426b33e98d09b1c52df053569 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sat, 22 May 2021 15:06:56 -0700 Subject: [PATCH] Change code to prevent creating client every conn --- main.py | 21 +++++++++++++++++---- quicclient.py | 43 ++++++++++++++++++++++++------------------- 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/main.py b/main.py index 6631bbe..e1a1ef4 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,6 @@ import pathlib import logging +import asyncio import pytomlpp import appdirs @@ -31,13 +32,18 @@ routes = web.RouteTableDef() async def post(request): arguments = await request.json() post_data = str(arguments.get("data", "")) + post_data = post_data if post_data else None method = arguments["method"] # Create heders intermediate_header_processing = [(k, v) for k, v in arguments.get("headers", {}).items()] processed_headers = CIMultiDict(intermediate_header_processing) - result = await quicclient.request(arguments["url"], method, processed_headers, post_data if post_data else None) + packaged_request = quicclient.InvidiousRequest(url=arguments["url"], method=method, headers=processed_headers, + content=post_data) + result = {} + await request_processor.requests_to_do.put([packaged_request, result]) + await packaged_request.completed.wait() if result["headers"][":status"] == "304": return web.Response(body=b"", headers=result["headers"], status=304) @@ -45,7 +51,14 @@ async def post(request): return web.Response(body=result["response"], headers=result["headers"]) -app = web.Application() -app.add_routes(routes) +async def main(): + asyncio.create_task(request_processor.request_worker()) + app = web.Application() + app.add_routes(routes) + return app + + +request_processor = quicclient.RequestProcessor() + if __name__ == '__main__': - web.run_app(app, **config) + web.run_app(main(), **config) \ No newline at end of file diff --git a/quicclient.py b/quicclient.py index ead2c3d..3a8e865 100644 --- a/quicclient.py +++ b/quicclient.py @@ -26,6 +26,16 @@ USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Ge HTTPRequest = collections.namedtuple("HTTPRequest", ["content", "headers", "method", "url"]) +class InvidiousRequest: + def __init__(self, url, method, headers, content): + self.url = urlparse(url) + self.method = method + self.headers = headers + self.content = content + + self.completed = asyncio.Event() + + class URL: def __init__(self, url): self.authority = url.netloc @@ -167,25 +177,20 @@ async def handle_response(http_events, store_at=None): return store_at -async def request(url, method, headers=None, data=None): - configuration = QuicConfiguration(is_client=True, alpn_protocols=H3_ALPN, idle_timeout=5) - headers = {} if not headers else headers +class RequestProcessor: + def __init__(self): + # {InvidiousRequest, storage_dict} + self.requests_to_do = asyncio.Queue(0) - parsed = urlparse(url) - if not parsed.scheme: - parsed = urlparse(f"https://{url}") + async def request_worker(self): + configuration = QuicConfiguration(is_client=True, alpn_protocols=H3_ALPN) + async with connect_quic_client("youtube.com", 443, configuration=configuration, + create_protocol=HttpClient) as client: + while True: + request, storage = await self.requests_to_do.get() - host = parsed.hostname - if parsed.port is not None: - port = parsed.port - else: - port = 443 + await perform_http_request(client=client, url=request.url, method=request.method, + headers=request.headers, data=request.content, + store_at=storage) - async with connect_quic_client(host, port, configuration=configuration, create_protocol=HttpClient) as client: - client = cast(HttpClient, client) - response_storage = {} - - await perform_http_request(client=client, url=parsed, method=method, headers=headers, data=data, - store_at=response_storage) - - return response_storage + request.completed.set()