Change code to prevent creating client every conn

This commit is contained in:
syeopite 2021-05-22 15:06:56 -07:00
parent 1cf2b276c5
commit f431d86e8e
No known key found for this signature in database
GPG key ID: 6FA616E5A5294A82
2 changed files with 41 additions and 23 deletions

17
main.py
View file

@ -1,5 +1,6 @@
import pathlib import pathlib
import logging import logging
import asyncio
import pytomlpp import pytomlpp
import appdirs import appdirs
@ -31,13 +32,18 @@ routes = web.RouteTableDef()
async def post(request): async def post(request):
arguments = await request.json() arguments = await request.json()
post_data = str(arguments.get("data", "")) post_data = str(arguments.get("data", ""))
post_data = post_data if post_data else None
method = arguments["method"] method = arguments["method"]
# Create heders # Create heders
intermediate_header_processing = [(k, v) for k, v in arguments.get("headers", {}).items()] intermediate_header_processing = [(k, v) for k, v in arguments.get("headers", {}).items()]
processed_headers = CIMultiDict(intermediate_header_processing) processed_headers = CIMultiDict(intermediate_header_processing)
result = await quicclient.request(arguments["url"], method, processed_headers, post_data if post_data else None) packaged_request = quicclient.InvidiousRequest(url=arguments["url"], method=method, headers=processed_headers,
content=post_data)
result = {}
await request_processor.requests_to_do.put([packaged_request, result])
await packaged_request.completed.wait()
if result["headers"][":status"] == "304": if result["headers"][":status"] == "304":
return web.Response(body=b"", headers=result["headers"], status=304) return web.Response(body=b"", headers=result["headers"], status=304)
@ -45,7 +51,14 @@ async def post(request):
return web.Response(body=result["response"], headers=result["headers"]) return web.Response(body=result["response"], headers=result["headers"])
async def main():
asyncio.create_task(request_processor.request_worker())
app = web.Application() app = web.Application()
app.add_routes(routes) app.add_routes(routes)
return app
request_processor = quicclient.RequestProcessor()
if __name__ == '__main__': if __name__ == '__main__':
web.run_app(app, **config) web.run_app(main(), **config)

View file

@ -26,6 +26,16 @@ USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Ge
HTTPRequest = collections.namedtuple("HTTPRequest", ["content", "headers", "method", "url"]) HTTPRequest = collections.namedtuple("HTTPRequest", ["content", "headers", "method", "url"])
class InvidiousRequest:
def __init__(self, url, method, headers, content):
self.url = urlparse(url)
self.method = method
self.headers = headers
self.content = content
self.completed = asyncio.Event()
class URL: class URL:
def __init__(self, url): def __init__(self, url):
self.authority = url.netloc self.authority = url.netloc
@ -167,25 +177,20 @@ async def handle_response(http_events, store_at=None):
return store_at return store_at
async def request(url, method, headers=None, data=None): class RequestProcessor:
configuration = QuicConfiguration(is_client=True, alpn_protocols=H3_ALPN, idle_timeout=5) def __init__(self):
headers = {} if not headers else headers # {InvidiousRequest, storage_dict}
self.requests_to_do = asyncio.Queue(0)
parsed = urlparse(url) async def request_worker(self):
if not parsed.scheme: configuration = QuicConfiguration(is_client=True, alpn_protocols=H3_ALPN)
parsed = urlparse(f"https://{url}") async with connect_quic_client("youtube.com", 443, configuration=configuration,
create_protocol=HttpClient) as client:
while True:
request, storage = await self.requests_to_do.get()
host = parsed.hostname await perform_http_request(client=client, url=request.url, method=request.method,
if parsed.port is not None: headers=request.headers, data=request.content,
port = parsed.port store_at=storage)
else:
port = 443
async with connect_quic_client(host, port, configuration=configuration, create_protocol=HttpClient) as client: request.completed.set()
client = cast(HttpClient, client)
response_storage = {}
await perform_http_request(client=client, url=parsed, method=method, headers=headers, data=data,
store_at=response_storage)
return response_storage