From 7e64dcd1330aa177ea089f696da282599d348278 Mon Sep 17 00:00:00 2001 From: dsc Date: Mon, 14 Mar 2022 19:00:16 +0200 Subject: [PATCH] YellWOWpages API - distribute contacts to WS clients --- requirements.txt | 2 +- settings.py_example | 15 +++---- wowlet_backend/factory.py | 24 +++++------ wowlet_backend/routes.py | 13 +++--- wowlet_backend/tasks/__init__.py | 1 + wowlet_backend/tasks/yellow.py | 28 +++++++++++++ wowlet_backend/utils.py | 70 ++++++++------------------------ 7 files changed, 70 insertions(+), 83 deletions(-) create mode 100644 wowlet_backend/tasks/yellow.py diff --git a/requirements.txt b/requirements.txt index f01da00..fb038a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ quart -aioredis +aioredis>=2.0.0 aiohttp aiofiles quart_session diff --git a/settings.py_example b/settings.py_example index 799084e..9e86a99 100644 --- a/settings.py_example +++ b/settings.py_example @@ -14,21 +14,18 @@ DEBUG = bool_env(os.environ.get("WOWLET_DEBUG", False)) HOST = os.environ.get("WOWLET_HOST", "127.0.0.1") PORT = int(os.environ.get("WOWLET_PORT", 1337)) -REDIS_ADDRESS = os.environ.get("WOWLET_REDIS_ADDRESS", "redis://localhost") +REDIS_HOST = os.environ.get("WOWLET_REDIS_HOST", "localhost") +REDIS_PORT = os.environ.get("WOWLET_REDIS_PORT", 6379) REDIS_PASSWORD = os.environ.get("WOWLET_REDIS_PASSWORD") -COIN_NAME = os.environ.get("WOWLET_COIN_NAME", "monero").lower() # as per coingecko -COIN_SYMBOL = os.environ.get("WOWLET_COIN_SYMBOL", "xmr").lower() # as per coingecko -COIN_GENESIS_DATE = os.environ.get("WOWLET_COIN_GENESIS_DATE", "20140418") +COIN_NAME = os.environ.get("WOWLET_COIN_NAME", "wownero").lower() # as per coingecko +COIN_SYMBOL = os.environ.get("WOWLET_COIN_SYMBOL", "wow").lower() # as per coingecko +COIN_GENESIS_DATE = os.environ.get("WOWLET_COIN_GENESIS_DATE", "20180401") COIN_MODE = os.environ.get("WOWLET_COIN_MODE", "mainnet").lower() TOR_SOCKS_PROXY = os.environ.get("WOWLET_TOR_SOCKS_PROXY", "socks5://127.0.0.1:9050") # while fetching USD price from coingecko, also include these extra coins: CRYPTO_RATES_COINS_EXTRA = { - "wownero": "wow", - "aeon": "aeon", - "turtlecoin": "trtl", - "haven": "xhv", - "loki": "loki" + "wownero": "wow" } diff --git a/wowlet_backend/factory.py b/wowlet_backend/factory.py index 182c11d..560ffa5 100644 --- a/wowlet_backend/factory.py +++ b/wowlet_backend/factory.py @@ -12,7 +12,7 @@ from quart import Quart from quart_session import Session import aioredis -from wowlet_backend.utils import current_worker_thread_is_primary, print_banner +from wowlet_backend.utils import print_banner import settings now = datetime.now() @@ -20,7 +20,6 @@ app: Quart = None cache = None user_agents: List[str] = None broadcast = MultisubscriberQueue() -_is_primary_worker_thread = False async def _setup_nodes(app: Quart): @@ -39,14 +38,16 @@ async def _setup_user_agents(app: Quart): async def _setup_cache(app: Quart): global cache # Each coin has it's own Redis DB index; `redis-cli -n $INDEX` - db = {"xmr": 0, "wow": 1, "aeon": 2, "trtl": 3, "msr": 4, "xhv": 5, "loki": 6}[settings.COIN_SYMBOL] + db = {"wow": 0, "xmr": 1, "aeon": 2, "trtl": 3, "msr": 4, "xhv": 5, "loki": 6}[settings.COIN_SYMBOL] + data = { - "address": settings.REDIS_ADDRESS, + "host": settings.REDIS_HOST, + "port": settings.REDIS_PORT, "db": db, "password": settings.REDIS_PASSWORD if settings.REDIS_PASSWORD else None } - cache = await aioredis.create_redis_pool(**data) + cache = await aioredis.Redis(**data) app.config['SESSION_TYPE'] = 'redis' app.config['SESSION_REDIS'] = cache Session(app) @@ -54,13 +55,11 @@ async def _setup_cache(app: Quart): async def _setup_tasks(app: Quart): """Schedules a series of tasks at an interval.""" - if not _is_primary_worker_thread: - return - from wowlet_backend.tasks import ( BlockheightTask, HistoricalPriceTask, FundingProposalsTask, CryptoRatesTask, FiatRatesTask, RedditTask, RPCNodeCheckTask, - XmrigTask, SuchWowTask, WowletReleasesTask, ForumThreadsTask) + XmrigTask, SuchWowTask, WowletReleasesTask, ForumThreadsTask, + YellWowTask) asyncio.create_task(BlockheightTask().start()) asyncio.create_task(HistoricalPriceTask().start()) @@ -72,6 +71,7 @@ async def _setup_tasks(app: Quart): asyncio.create_task(SuchWowTask().start()) asyncio.create_task(WowletReleasesTask().start()) asyncio.create_task(ForumThreadsTask().start()) + asyncio.create_task(YellWowTask().start()) if settings.COIN_SYMBOL in ["xmr", "wow"]: asyncio.create_task(FundingProposalsTask().start()) @@ -101,11 +101,7 @@ def create_app(): @app.before_serving async def startup(): - global _is_primary_worker_thread - _is_primary_worker_thread = current_worker_thread_is_primary() - - if _is_primary_worker_thread: - print_banner() + print_banner() await _setup_cache(app) await _setup_nodes(app) diff --git a/wowlet_backend/routes.py b/wowlet_backend/routes.py index 57a749d..0349bb6 100644 --- a/wowlet_backend/routes.py +++ b/wowlet_backend/routes.py @@ -9,14 +9,14 @@ import json from quart import websocket, jsonify, send_from_directory import settings -from wowlet_backend.factory import app +from wowlet_backend.factory import app, broadcast from wowlet_backend.wsparse import WebsocketParse -from wowlet_backend.utils import broadcast, feather_data +from wowlet_backend.utils import wowlet_data @app.route("/") async def root(): - data = await feather_data() + data = await wowlet_data() return jsonify(data) @@ -29,7 +29,7 @@ async def suchwow(name: str): @app.websocket('/ws') async def ws(): - data = await feather_data() + data = await wowlet_data() # blast available data on connect for task_key, task_value in data.items(): @@ -54,9 +54,8 @@ async def ws(): continue async def tx(): - while True: - data = await broadcast.get() - payload = json.dumps(data).encode() + async for _data in broadcast.subscribe(): + payload = json.dumps(_data).encode() await websocket.send(payload) # bidirectional async rx and tx loops diff --git a/wowlet_backend/tasks/__init__.py b/wowlet_backend/tasks/__init__.py index 1376332..290ab11 100644 --- a/wowlet_backend/tasks/__init__.py +++ b/wowlet_backend/tasks/__init__.py @@ -167,3 +167,4 @@ from wowlet_backend.tasks.xmrig import XmrigTask from wowlet_backend.tasks.suchwow import SuchWowTask from wowlet_backend.tasks.wowlet import WowletReleasesTask from wowlet_backend.tasks.forum import ForumThreadsTask +from wowlet_backend.tasks.yellow import YellWowTask diff --git a/wowlet_backend/tasks/yellow.py b/wowlet_backend/tasks/yellow.py new file mode 100644 index 0000000..7b47852 --- /dev/null +++ b/wowlet_backend/tasks/yellow.py @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2022, The Monero Project. +# Copyright (c) 2022, dsc@xmr.pm + +from dateutil.parser import parse + +import settings +from wowlet_backend.utils import httpget +from wowlet_backend.tasks import WowletTask + + +class YellWowTask(WowletTask): + """Fetches yellwowpages usernames/addresses""" + def __init__(self, interval: int = 3600): + super(YellWowTask, self).__init__(interval) + + self._cache_key = "yellwow" + self._cache_expiry = self.interval * 10 + + self._websocket_cmd = "yellwow" + + self._http_endpoint = "https://yellow.wownero.com/api/user/" + + async def task(self) -> list[dict]: + blob = await httpget(self._http_endpoint) + if not isinstance(blob, list) or not blob: + raise Exception(f"Invalid JSON response for {self._http_endpoint}") + return blob diff --git a/wowlet_backend/utils.py b/wowlet_backend/utils.py index 49b3555..36304e5 100644 --- a/wowlet_backend/utils.py +++ b/wowlet_backend/utils.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2020, The Monero Project. -# Copyright (c) 2020, dsc@xmr.pm +# Copyright (c) 2022, The Monero Project. +# Copyright (c) 2022, dsc@xmr.pm import re import json @@ -41,19 +41,6 @@ def print_banner(): """.strip()) -def collect_websocket(func): - @wraps(func) - async def wrapper(*args, **kwargs): - from wowlet_backend.factory import connected_websockets - queue = asyncio.Queue() - connected_websockets.add(queue) - try: - return await func(queue, *args, **kwargs) - finally: - connected_websockets.remove(queue) - return wrapper - - async def httpget(url: str, json=True, timeout: int = 5, socks5: str = None, raise_for_status=True, verify_tls=True): headers = {"User-Agent": random_agent()} opts = {"timeout": aiohttp.ClientTimeout(total=timeout)} @@ -76,23 +63,30 @@ def random_agent(): return random.choice(user_agents) -async def feather_data(): - """A collection of data collected by - `FeatherTask`, for Feather wallet clients.""" +async def wowlet_data(): + """A collection of data collected by the various wowlet tasks""" from wowlet_backend.factory import cache, now data = await cache.get("data") if data: data = json.loads(data) return data - keys = ["blockheights", "funding_proposals", "crypto_rates", "fiat_rates", "reddit", "rpc_nodes", "xmrig", "xmrto_rates", "suchwow", "forum", "wowlet_releases"] + keys = [ + "blockheights", + "funding_proposals", + "crypto_rates", + "fiat_rates", + "reddit", + "rpc_nodes", + "xmrig", + "xmrto_rates", + "suchwow", + "forum", + "wowlet_releases", + "yellwow" + ] data = {keys[i]: json.loads(val) if val else None for i, val in enumerate(await cache.mget(*keys))} - # @TODO: for backward-compat reasons we're including some legacy keys which can be removed after 1.0 release - data['nodes'] = data['rpc_nodes'] - data['ccs'] = data['funding_proposals'] - data['wfs'] = data['funding_proposals'] - # start caching when application lifetime is more than 20 seconds if (datetime.now() - now).total_seconds() > 20: await cache.setex("data", 30, json.dumps(data)) @@ -110,34 +104,6 @@ def popularity_contest(lst: List[int]) -> Union[int, None]: return Counter(lst).most_common(1)[0][0] -def current_worker_thread_is_primary() -> bool: - """ - ASGI server (Hypercorn) may start multiple - worker threads, but we only want one feather-ws - instance to schedule `FeatherTask` tasks at an - interval. Therefor this function determines if the - current instance is responsible for the - recurring Feather tasks. - """ - from wowlet_backend.factory import app - - current_pid = os.getpid() - parent_pid = os.getppid() - app.logger.debug(f"current_pid: {current_pid}, " - f"parent_pid: {parent_pid}") - - if parent_pid == 0: - return True - - parent = psutil.Process(parent_pid) - if parent.name() != "hypercorn": - return True - - lowest_pid = min(c.pid for c in parent.children(recursive=True) if c.name() == "hypercorn") - if current_pid == lowest_pid: - return True - - async def image_resize(buffer: bytes, max_bounding_box: int = 512, quality: int = 70) -> bytes: """ - Resize if the image is too large