Merge pull request 'YellWOWpages API - distribute contacts to WS clients' (#17) from yellwow into master

Reviewed-on: https://git.wownero.com/wowlet/wowlet-backend/pulls/17

requirements.txt

@@ -1,5 +1,5 @@
 quart
-aioredis
+aioredis>=2.0.0
 aiohttp
 aiofiles
 quart_session

settings.py_example

@@ -14,21 +14,18 @@ DEBUG = bool_env(os.environ.get("WOWLET_DEBUG", False))
 HOST = os.environ.get("WOWLET_HOST", "127.0.0.1")
 PORT = int(os.environ.get("WOWLET_PORT", 1337))
 
-REDIS_ADDRESS = os.environ.get("WOWLET_REDIS_ADDRESS", "redis://localhost")
+REDIS_HOST = os.environ.get("WOWLET_REDIS_HOST", "localhost")
+REDIS_PORT = os.environ.get("WOWLET_REDIS_PORT", 6379)
 REDIS_PASSWORD = os.environ.get("WOWLET_REDIS_PASSWORD")
 
-COIN_NAME = os.environ.get("WOWLET_COIN_NAME", "monero").lower()  # as per coingecko
-COIN_SYMBOL = os.environ.get("WOWLET_COIN_SYMBOL", "xmr").lower()  # as per coingecko
-COIN_GENESIS_DATE = os.environ.get("WOWLET_COIN_GENESIS_DATE", "20140418")
+COIN_NAME = os.environ.get("WOWLET_COIN_NAME", "wownero").lower()  # as per coingecko
+COIN_SYMBOL = os.environ.get("WOWLET_COIN_SYMBOL", "wow").lower()  # as per coingecko
+COIN_GENESIS_DATE = os.environ.get("WOWLET_COIN_GENESIS_DATE", "20180401")
 COIN_MODE = os.environ.get("WOWLET_COIN_MODE", "mainnet").lower()
 
 TOR_SOCKS_PROXY = os.environ.get("WOWLET_TOR_SOCKS_PROXY", "socks5://127.0.0.1:9050")
 
 # while fetching USD price from coingecko, also include these extra coins:
 CRYPTO_RATES_COINS_EXTRA = {
-    "wownero": "wow",
-    "aeon": "aeon",
-    "turtlecoin": "trtl",
-    "haven": "xhv",
-    "loki": "loki"
+    "wownero": "wow"
 }

wowlet_backend/factory.py

@@ -12,7 +12,7 @@ from quart import Quart
 from quart_session import Session
 import aioredis
 
-from wowlet_backend.utils import current_worker_thread_is_primary, print_banner
+from wowlet_backend.utils import print_banner
 import settings
 
 now = datetime.now()
@@ -20,7 +20,6 @@ app: Quart = None
 cache = None
 user_agents: List[str] = None
 broadcast = MultisubscriberQueue() 
-_is_primary_worker_thread = False
 
 
 async def _setup_nodes(app: Quart):
@@ -39,14 +38,16 @@ async def _setup_user_agents(app: Quart):
 async def _setup_cache(app: Quart):
     global cache
     # Each coin has it's own Redis DB index; `redis-cli -n $INDEX`
-    db = {"xmr": 0, "wow": 1, "aeon": 2, "trtl": 3, "msr": 4, "xhv": 5, "loki": 6}[settings.COIN_SYMBOL]
+    db = {"wow": 0, "xmr": 1, "aeon": 2, "trtl": 3, "msr": 4, "xhv": 5, "loki": 6}[settings.COIN_SYMBOL]
+
     data = {
-        "address": settings.REDIS_ADDRESS,
+        "host": settings.REDIS_HOST,
+        "port": settings.REDIS_PORT,
         "db": db,
         "password": settings.REDIS_PASSWORD if settings.REDIS_PASSWORD else None
     }
 
-    cache = await aioredis.create_redis_pool(**data)
+    cache = await aioredis.Redis(**data)
     app.config['SESSION_TYPE'] = 'redis'
     app.config['SESSION_REDIS'] = cache
     Session(app)
@@ -54,13 +55,11 @@ async def _setup_cache(app: Quart):
 
 async def _setup_tasks(app: Quart):
     """Schedules a series of tasks at an interval."""
-    if not _is_primary_worker_thread:
-        return
-
     from wowlet_backend.tasks import (
         BlockheightTask, HistoricalPriceTask, FundingProposalsTask,
         CryptoRatesTask, FiatRatesTask, RedditTask, RPCNodeCheckTask,
-        XmrigTask, SuchWowTask, WowletReleasesTask, ForumThreadsTask)
+        XmrigTask, SuchWowTask, WowletReleasesTask, ForumThreadsTask,
+        YellWowTask)
 
     asyncio.create_task(BlockheightTask().start())
     asyncio.create_task(HistoricalPriceTask().start())
@@ -72,6 +71,7 @@ async def _setup_tasks(app: Quart):
     asyncio.create_task(SuchWowTask().start())
     asyncio.create_task(WowletReleasesTask().start())
     asyncio.create_task(ForumThreadsTask().start())
+    asyncio.create_task(YellWowTask().start())
 
     if settings.COIN_SYMBOL in ["xmr", "wow"]:
         asyncio.create_task(FundingProposalsTask().start())
@@ -101,11 +101,7 @@ def create_app():
 
     @app.before_serving
     async def startup():
-        global _is_primary_worker_thread
-        _is_primary_worker_thread = current_worker_thread_is_primary()
-
-        if _is_primary_worker_thread:
-            print_banner()
+        print_banner()
 
         await _setup_cache(app)
         await _setup_nodes(app)

wowlet_backend/routes.py

@@ -9,14 +9,14 @@ import json
 from quart import websocket, jsonify, send_from_directory
 
 import settings
-from wowlet_backend.factory import app
+from wowlet_backend.factory import app, broadcast
 from wowlet_backend.wsparse import WebsocketParse
-from wowlet_backend.utils import broadcast, feather_data
+from wowlet_backend.utils import wowlet_data
 
 
 @app.route("/")
 async def root():
-    data = await feather_data()
+    data = await wowlet_data()
     return jsonify(data)
 
 
@@ -29,7 +29,7 @@ async def suchwow(name: str):
 
 @app.websocket('/ws')
 async def ws():
-    data = await feather_data()
+    data = await wowlet_data()
 
     # blast available data on connect
     for task_key, task_value in data.items():
@@ -54,9 +54,8 @@ async def ws():
                 continue
 
     async def tx():
-        while True:
-            data = await broadcast.get()
-            payload = json.dumps(data).encode()
+        async for _data in broadcast.subscribe():
+            payload = json.dumps(_data).encode()
             await websocket.send(payload)
 
     # bidirectional async rx and tx loops

wowlet_backend/tasks/__init__.py

@@ -167,3 +167,4 @@ from wowlet_backend.tasks.xmrig import XmrigTask
 from wowlet_backend.tasks.suchwow import SuchWowTask
 from wowlet_backend.tasks.wowlet import WowletReleasesTask
 from wowlet_backend.tasks.forum import ForumThreadsTask
+from wowlet_backend.tasks.yellow import YellWowTask

wowlet_backend/tasks/yellow.py

@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2022, The Monero Project.
+# Copyright (c) 2022, dsc@xmr.pm
+
+from dateutil.parser import parse
+
+import settings
+from wowlet_backend.utils import httpget
+from wowlet_backend.tasks import WowletTask
+
+
+class YellWowTask(WowletTask):
+    """Fetches yellwowpages usernames/addresses"""
+    def __init__(self, interval: int = 3600):
+        super(YellWowTask, self).__init__(interval)
+
+        self._cache_key = "yellwow"
+        self._cache_expiry = self.interval * 10
+
+        self._websocket_cmd = "yellwow"
+
+        self._http_endpoint = "https://yellow.wownero.com/api/user/"
+
+    async def task(self) -> list[dict]:
+        blob = await httpget(self._http_endpoint)
+        if not isinstance(blob, list) or not blob:
+            raise Exception(f"Invalid JSON response for {self._http_endpoint}")
+        return blob

wowlet_backend/utils.py

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright (c) 2020, The Monero Project.
-# Copyright (c) 2020, dsc@xmr.pm
+# Copyright (c) 2022, The Monero Project.
+# Copyright (c) 2022, dsc@xmr.pm
 
 import re
 import json
@@ -41,19 +41,6 @@ def print_banner():
     """.strip())
 
 
-def collect_websocket(func):
-    @wraps(func)
-    async def wrapper(*args, **kwargs):
-        from wowlet_backend.factory import connected_websockets
-        queue = asyncio.Queue()
-        connected_websockets.add(queue)
-        try:
-            return await func(queue, *args, **kwargs)
-        finally:
-            connected_websockets.remove(queue)
-    return wrapper
-
-
 async def httpget(url: str, json=True, timeout: int = 5, socks5: str = None, raise_for_status=True, verify_tls=True):
     headers = {"User-Agent": random_agent()}
     opts = {"timeout": aiohttp.ClientTimeout(total=timeout)}
@@ -76,23 +63,30 @@ def random_agent():
     return random.choice(user_agents)
 
 
-async def feather_data():
-    """A collection of data collected by
-    `FeatherTask`, for Feather wallet clients."""
+async def wowlet_data():
+    """A collection of data collected by the various wowlet tasks"""
     from wowlet_backend.factory import cache, now
     data = await cache.get("data")
     if data:
         data = json.loads(data)
         return data
 
-    keys = ["blockheights", "funding_proposals", "crypto_rates", "fiat_rates", "reddit", "rpc_nodes", "xmrig", "xmrto_rates", "suchwow", "forum", "wowlet_releases"]
+    keys = [
+        "blockheights",
+        "funding_proposals",
+        "crypto_rates",
+        "fiat_rates",
+        "reddit",
+        "rpc_nodes",
+        "xmrig",
+        "xmrto_rates",
+        "suchwow",
+        "forum",
+        "wowlet_releases",
+        "yellwow"
+    ]
     data = {keys[i]: json.loads(val) if val else None for i, val in enumerate(await cache.mget(*keys))}
 
-    # @TODO: for backward-compat reasons we're including some legacy keys which can be removed after 1.0 release
-    data['nodes'] = data['rpc_nodes']
-    data['ccs'] = data['funding_proposals']
-    data['wfs'] = data['funding_proposals']
-
     # start caching when application lifetime is more than 20 seconds
     if (datetime.now() - now).total_seconds() > 20:
         await cache.setex("data", 30, json.dumps(data))
@@ -110,34 +104,6 @@ def popularity_contest(lst: List[int]) -> Union[int, None]:
     return Counter(lst).most_common(1)[0][0]
 
 
-def current_worker_thread_is_primary() -> bool:
-    """
-    ASGI server (Hypercorn) may start multiple
-    worker threads, but we only want one feather-ws
-    instance to schedule `FeatherTask` tasks at an
-    interval. Therefor this function determines if the
-    current instance is responsible for the
-    recurring Feather tasks.
-    """
-    from wowlet_backend.factory import app
-
-    current_pid = os.getpid()
-    parent_pid = os.getppid()
-    app.logger.debug(f"current_pid: {current_pid}, "
-                     f"parent_pid: {parent_pid}")
-
-    if parent_pid == 0:
-        return True
-
-    parent = psutil.Process(parent_pid)
-    if parent.name() != "hypercorn":
-        return True
-
-    lowest_pid = min(c.pid for c in parent.children(recursive=True) if c.name() == "hypercorn")
-    if current_pid == lowest_pid:
-        return True
-
-
 async def image_resize(buffer: bytes, max_bounding_box: int = 512, quality: int = 70) -> bytes:
     """
     - Resize if the image is too large