feather-ws -> wowlet-backend

wowlet_backend/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2020, The Monero Project.
+# Copyright (c) 2020, dsc@xmr.pm

wowlet_backend/factory.py

@@ -0,0 +1,113 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2020, The Monero Project.
+# Copyright (c) 2020, dsc@xmr.pm
+
+import json
+import asyncio
+from typing import List, Set
+from datetime import datetime
+
+from quart import Quart
+from quart_session import Session
+import aioredis
+
+from wowlet_backend.utils import current_worker_thread_is_primary, print_banner
+import settings
+
+now = datetime.now()
+app: Quart = None
+cache = None
+user_agents: List[str] = None
+connected_websockets: Set[asyncio.Queue] = set()
+_is_primary_worker_thread = False
+
+
+async def _setup_nodes(app: Quart):
+    global cache
+    with open('data/nodes.json', 'r') as f:
+        nodes = json.loads(f.read()).get(settings.COIN_SYMBOL)
+        cache.execute('JSON.SET', 'nodes', '.', json.dumps(nodes))
+
+
+async def _setup_user_agents(app: Quart):
+    global user_agents
+    with open('data/user_agents.txt', 'r') as f:
+        user_agents = [l.strip() for l in f.readlines() if l.strip()]
+
+
+async def _setup_cache(app: Quart):
+    global cache
+    # Each coin has it's own Redis DB index; `redis-cli -n $INDEX`
+    db = {"xmr": 0, "wow": 1, "aeon": 2, "trtl": 3, "msr": 4, "xhv": 5, "loki": 6}[settings.COIN_SYMBOL]
+    data = {
+        "address": settings.REDIS_ADDRESS,
+        "db": db,
+        "password": settings.REDIS_PASSWORD if settings.REDIS_PASSWORD else None
+    }
+
+    cache = await aioredis.create_redis_pool(**data)
+    app.config['SESSION_TYPE'] = 'redis'
+    app.config['SESSION_REDIS'] = cache
+    Session(app)
+
+
+async def _setup_tasks(app: Quart):
+    """Schedules a series of tasks at an interval."""
+    if not _is_primary_worker_thread:
+        return
+
+    from wowlet_backend.tasks import (
+        BlockheightTask, HistoricalPriceTask, FundingProposalsTask,
+        CryptoRatesTask, FiatRatesTask, RedditTask, RPCNodeCheckTask,
+        XmrigTask, XmrToTask)
+
+    asyncio.create_task(BlockheightTask().start())
+    asyncio.create_task(HistoricalPriceTask().start())
+    asyncio.create_task(CryptoRatesTask().start())
+    asyncio.create_task(FiatRatesTask().start())
+    asyncio.create_task(RedditTask().start())
+    asyncio.create_task(RPCNodeCheckTask().start())
+    asyncio.create_task(XmrigTask().start())
+
+    if settings.COIN_SYMBOL in ["xmr", "wow"]:
+        asyncio.create_task(FundingProposalsTask().start())
+
+
+def _setup_logging():
+    from logging import Formatter
+    from logging.config import dictConfig
+    from quart.logging import default_handler
+    default_handler.setFormatter(Formatter('[%(asctime)s] %(levelname)s in %(funcName)s(): %(message)s (%(pathname)s)'))
+
+    dictConfig({
+        'version': 1,
+        'loggers': {
+            'quart.app': {
+                'level': 'DEBUG' if settings.DEBUG else 'INFO',
+            },
+        },
+    })
+
+
+def create_app():
+    global app
+
+    _setup_logging()
+    app = Quart(__name__)
+
+    @app.before_serving
+    async def startup():
+        global _is_primary_worker_thread
+        _is_primary_worker_thread = current_worker_thread_is_primary()
+
+        if _is_primary_worker_thread:
+            print_banner()
+
+        await _setup_cache(app)
+        await _setup_nodes(app)
+        await _setup_user_agents(app)
+        await _setup_tasks(app)
+
+        import wowlet_backend.routes
+
+    return app

wowlet_backend/routes.py

@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2020, The Monero Project.
+# Copyright (c) 2020, dsc@xmr.pm
+
+import asyncio
+import json
+
+from quart import websocket, jsonify
+
+from wowlet_backend.factory import app
+from wowlet_backend.wsparse import WebsocketParse
+from wowlet_backend.utils import collect_websocket, feather_data
+
+
+@app.route("/")
+async def root():
+    data = await feather_data()
+    return jsonify(data)
+
+
+@app.websocket('/ws')
+@collect_websocket
+async def ws(queue):
+    data = await feather_data()
+
+    # blast available data on connect
+    for task_key, task_value in data.items():
+        if not task_value:
+            continue
+        await websocket.send(json.dumps({"cmd": task_key, "data": task_value}).encode())
+
+    async def rx():
+        while True:
+            buffer = await websocket.receive()
+            try:
+                blob = json.loads(buffer)
+                if "cmd" not in blob:
+                    continue
+                cmd = blob.get('cmd')
+                _data = blob.get('data')
+                result = await WebsocketParse.parser(cmd, _data)
+                if result:
+                    rtn = json.dumps({"cmd": cmd, "data": result}).encode()
+                    await websocket.send(rtn)
+            except Exception as ex:
+                continue
+
+    async def tx():
+        while True:
+            data = await queue.get()
+            payload = json.dumps(data).encode()
+            await websocket.send(payload)
+
+    # bidirectional async rx and tx loops
+    consumer_task = asyncio.ensure_future(rx())
+    producer_task = asyncio.ensure_future(tx())
+    try:
+        await asyncio.gather(consumer_task, producer_task)
+    finally:
+        consumer_task.cancel()
+        producer_task.cancel()
+
+
+@app.errorhandler(403)
+@app.errorhandler(404)
+@app.errorhandler(405)
+@app.errorhandler(500)
+def page_not_found(e):
+    return ":)", 500

wowlet_backend/tasks/__init__.py

@@ -0,0 +1,168 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2020, The Monero Project.
+# Copyright (c) 2020, dsc@xmr.pm
+
+import json
+import asyncio
+import random
+from typing import Union
+
+
+class FeatherTask:
+    """
+    The base class of many recurring tasks for this
+    project. This abstracts away some functionality:
+
+    1. Tasks are automatically cached in Redis if the `_cache_key` is set.
+    2. The task result is propagated to connected websocket clients if
+       `_websocket_cmd` is set.
+    3. Inheritors should implement the `task()` method.
+    4. Inheritors can optionally implement the `done()` method.
+    """
+    def __init__(self, interval: int):
+        """
+        :param interval: secs
+        """
+        self.interval = interval
+
+        # propogate to websocket clients?
+        self._websocket_cmd: str = None
+
+        # redis
+        self._cache_key: str = None
+        self._cache_expiry: int = None
+
+        # logging
+        self._qualname: str = f"{self.__class__.__module__}.{self.__class__.__name__}"
+
+        self._active = True
+        self._running = False
+
+    async def start(self, *args, **kwargs):
+        from wowlet_backend.factory import app, connected_websockets
+        if not self._active:
+            # invalid task
+            return
+
+        app.logger.info(f"Starting task {self._qualname}")
+        sleep = lambda: asyncio.sleep(random.randrange(self.interval - 5,
+                                                       self.interval + 5))
+        while True:
+            if not self._active:
+                # invalid task
+                return
+
+            if self._running:
+                # task already running, wait for completion
+                await asyncio.sleep(5)
+                continue
+
+            try:
+                self._running = True
+                result: dict = await self.task(*args, **kwargs)
+                if not result:
+                    raise Exception("No result")
+            except Exception as ex:
+                app.logger.error(f"{self._qualname} - {ex}")
+
+                # if the task failed we can attempt to use an old value from the cache.
+                if not self._cache_key:
+                    app.logger.warning(f"{self._qualname} - No cache key for task, skipping")
+                    await sleep()
+                    self._running = False
+                    continue
+
+                app.logger.info(f"{self._qualname} - trying cache")
+                result = await self.cache_get(self._cache_key)
+                if result:
+                    app.logger.warning(f"serving cached result for {self._qualname}")
+                else:
+                    app.logger.error(f"{self._qualname} - cache lookup failed, fix me")
+                    await sleep()
+                    self._running = False
+                    continue
+
+            # optional: propogate result to websocket peers
+            if self._websocket_cmd and result:
+                # but only when there is a change
+                normalize = lambda k: json.dumps(k, sort_keys=True, indent=4)
+                propagate = True
+
+                cached = await self.cache_get(self._cache_key)
+                if cached:
+                    if normalize(cached) == normalize(result):
+                        propagate = False
+
+                if propagate:
+                    for queue in connected_websockets:
+                        await queue.put({
+                            "cmd": self._websocket_cmd,
+                            "data": result
+                        })
+
+            # optional: cache the result
+            if self._cache_key and result:
+                await self.cache_set(self._cache_key, result, self._cache_expiry)
+
+            # optional: call completion function
+            if 'done' in self.__class__.__dict__:
+                await self.done(result)
+
+            await sleep()
+            self._running = False
+
+    async def task(self, *args, **kwargs):
+        raise NotImplementedError()
+
+    async def done(self, *args, **kwargs):
+        """overload this method to execute this function after
+        completion of `task`. Results from `task` are parameters
+        for `done`."""
+        raise NotImplementedError()
+
+    async def end(self, result: dict):
+        raise NotImplementedError()
+
+    async def cache_json_get(self, key: str, path="."):
+        from wowlet_backend.factory import app, cache
+
+        try:
+            data = await cache.execute('JSON.GET', key, path)
+            if data:
+                return json.loads(data)
+        except Exception as ex:
+            app.logger.error(f"Redis error: {ex}")
+
+    async def cache_get(self, key: str) -> dict:
+        from wowlet_backend.factory import app, cache
+
+        try:
+            data = await cache.get(key)
+            if not data:
+                return {}
+            return json.loads(data)
+        except Exception as ex:
+            app.logger.error(f"Redis GET error with key '{key}': {ex}")
+
+    async def cache_set(self, key, val: Union[dict, int], expiry: int = 0) -> bool:
+        from wowlet_backend.factory import app, cache
+        try:
+            data = json.dumps(val)
+            if isinstance(expiry, int) and expiry > 0:
+                await cache.setex(key, expiry, data)
+            else:
+                await cache.set(key, data)
+            return True
+        except Exception as ex:
+            app.logger.error(f"Redis SET error with key '{key}': {ex}")
+
+
+from wowlet_backend.tasks.proposals import FundingProposalsTask
+from wowlet_backend.tasks.historical_prices import HistoricalPriceTask
+from wowlet_backend.tasks.blockheight import BlockheightTask
+from wowlet_backend.tasks.rates_fiat import FiatRatesTask
+from wowlet_backend.tasks.rates_crypto import CryptoRatesTask
+from wowlet_backend.tasks.reddit import RedditTask
+from wowlet_backend.tasks.rpc_nodes import RPCNodeCheckTask
+from wowlet_backend.tasks.xmrig import XmrigTask
+from wowlet_backend.tasks.xmrto import XmrToTask

wowlet_backend/tasks/blockheight.py

@@ -0,0 +1,161 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2020, The Monero Project.
+# Copyright (c) 2020, dsc@xmr.pm
+
+import re
+from typing import Union
+from collections import Counter
+from functools import partial
+
+import settings
+from wowlet_backend.utils import httpget, popularity_contest
+from wowlet_backend.tasks import FeatherTask
+
+
+class BlockheightTask(FeatherTask):
+    """
+    Fetch latest blockheight using webcrawling. We pick the most popular
+    height from a list of websites. Arguably this approach has benefits
+    over querying a (local) Monero RPC instance, as that requires
+    maintenance, while this solution assumes that (at least) 2 websites
+    reports the correct height.
+    """
+    def __init__(self, interval: int = 60):
+        super(BlockheightTask, self).__init__(interval)
+
+        self._cache_key = "blockheights"
+        self._cache_expiry = 90
+
+        self._websocket_cmd = "blockheights"
+
+        self._fns = {
+            "xmr": {
+                "mainnet": [
+                    self._blockchair,
+                    partial(self._onion_explorer, url="https://xmrchain.net/"),
+                    partial(self._onion_explorer, url="https://community.xmr.to/explorer/mainnet/"),
+                    partial(self._onion_explorer, url="https://monero.exan.tech/")
+                ],
+                "stagenet": [
+                    partial(self._onion_explorer, url="https://stagenet.xmrchain.net/"),
+                    partial(self._onion_explorer, url="https://community.xmr.to/explorer/stagenet/"),
+                    partial(self._onion_explorer, url="https://monero-stagenet.exan.tech/")
+                ]
+            },
+            "wow": {
+                "mainnet": [
+                    partial(self._onion_explorer, url="https://explore.wownero.com/"),
+                ]
+            },
+            "aeon": {
+                "mainnet": [
+                    partial(self._onion_explorer, url="https://aeonblockexplorer.com/"),
+                ],
+                "stagenet": [
+                    partial(self._onion_explorer, url="http://162.210.173.151:8083/"),
+                ]
+            },
+            "trtl": {
+                "mainnet": [
+                    self._turtlenode,
+                    self._turtlenetwork,
+                    self._l33d4n
+                ]
+            },
+            "xhv": {
+                "mainnet": [
+                    partial(self._onion_explorer, url="https://explorer.havenprotocol.org/")
+                ],
+                "stagenet": [
+                    partial(self._onion_explorer, url="https://explorer.stagenet.havenprotocol.org/page/1")
+                ]
+            },
+            "loki": {
+                "mainnet": [
+                    partial(self._onion_explorer, url="https://lokiblocks.com/")
+                ],
+                "testnet": [
+                    partial(self._onion_explorer, url="https://lokitestnet.com/")
+                ]
+            }
+        }
+
+    async def task(self) -> Union[dict, None]:
+        from wowlet_backend.factory import app
+        coin_network_types = ["mainnet", "stagenet", "testnet"]
+        data = {t: 0 for t in coin_network_types}
+
+        for coin_network_type in coin_network_types:
+            if coin_network_type not in self._fns[settings.COIN_SYMBOL]:
+                continue
+
+            heights = []
+            for fn in self._fns[settings.COIN_SYMBOL][coin_network_type]:
+                fn_name = fn.func.__name__ if isinstance(fn, partial) else fn.__name__
+
+                try:
+                    result = await fn()
+                    heights.append(result)
+                except Exception as ex:
+                    app.logger.error(f"blockheight fetch failed from {fn_name}(): {ex}")
+                    continue
+
+            if heights:
+                data[coin_network_type] = popularity_contest(heights)
+
+        if data["mainnet"] == 0:  # only care about mainnet
+            app.logger.error(f"Failed to parse latest blockheight!")
+            return
+
+        return data
+
+    async def _blockchair(self) -> int:
+        re_blockheight = r"<a href=\".*\">(\d+)</a>"
+
+        url = "https://blockchair.com/monero"
+        content = await httpget(url, json=False, raise_for_status=True)
+
+        height = re.findall(re_blockheight, content)
+        height = max(map(int, height))
+        return height
+
+    async def _wownero(self) -> int:
+        url = "https://explore.wownero.com/"
+        return await BlockheightTask._onion_explorer(url)
+
+    async def _turtlenode(self) -> int:
+        url = "https://public.turtlenode.net/info"
+        blob = await httpget(url, json=True, raise_for_status=True)
+        height = int(blob.get("height", 0))
+        if height <= 0:
+            raise Exception("bad height")
+        return height
+
+    async def _turtlenetwork(self) -> int:
+        url = "https://tnnode2.turtlenetwork.eu/blocks/height"
+        blob = await httpget(url, json=True, raise_for_status=True)
+        height = int(blob.get("height", 0))
+        if height <= 0:
+            raise Exception("bad height")
+        return height
+
+    async def _l33d4n(self):
+        url = "https://blockapi.turtlepay.io/block/header/top"
+        blob = await httpget(url, json=True, raise_for_status=True)
+        height = int(blob.get("height", 0))
+        if height <= 0:
+            raise Exception("bad height")
+        return height
+
+    @staticmethod
+    async def _onion_explorer(url):
+        """
+        Pages that are based on:
+        https://github.com/moneroexamples/onion-monero-blockchain-explorer
+        """
+        re_blockheight = r"block\/(\d+)\"\>"
+        content = await httpget(url, json=False)
+
+        height = re.findall(re_blockheight, content)
+        height = max(map(int, height))
+        return height

wowlet_backend/tasks/historical_prices.py

@@ -0,0 +1,114 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2020, The Monero Project.
+# Copyright (c) 2020, dsc@xmr.pm
+
+import asyncio
+import os
+import json
+from typing import List, Union
+from datetime import datetime
+
+import aiofiles
+
+import settings
+from wowlet_backend.utils import httpget
+from wowlet_backend.tasks import FeatherTask
+
+
+class HistoricalPriceTask(FeatherTask):
+    """
+    This class manages a historical price (USD) database, saved in a
+    textfile at `self._path`. A Feather wallet instance will ask
+    for the historical fiat price database on startup (but only
+    in chunks of a month for anti-fingerprinting reasons).
+
+    The task in this class simply keeps the fiat database
+    up-to-date locally.
+    """
+    def __init__(self, interval: int = 43200):
+        super(HistoricalPriceTask, self).__init__(interval)
+
+        self._cache_key = f"historical_fiat"
+        self._path = f"data/historical_prices_{settings.COIN_SYMBOL}.json"
+        self._http_endpoint = f"https://www.coingecko.com/price_charts/{settings.COIN_NAME}/usd/max.json"
+
+        self._year_genesis = int(settings.COIN_GENESIS_DATE[:4])
+
+        asyncio.create_task(self._load())
+
+    async def task(self) -> Union[dict, None]:
+        content = await httpget(self._http_endpoint, json=True, raise_for_status=False)
+        if "stats" not in content:
+            raise Exception()
+
+        stats: List[List] = content.get('stats', [])  # [[timestamp,USD],]
+        if not stats:
+            return
+
+        data = {
+            year: {
+                month: {} for month in range(1, 13)
+            } for year in range(self._year_genesis, datetime.now().year + 1)
+        }
+
+        # timestamp:USD
+        daily_price_blob = {day[0]: day[1] for day in stats}
+
+        # normalize
+        for timestamp, usd in daily_price_blob.items():
+            _date = datetime.fromtimestamp(timestamp / 1000)
+            data[_date.year].setdefault(_date.month, {})
+            data[_date.year][_date.month][_date.day] = usd
+
+        # update local database
+        await self._write(data)
+        return data
+
+    async def _load(self) -> None:
+        if not os.path.exists(self._path):
+            return
+
+        async with aiofiles.open(self._path, mode="r") as f:
+            content = await f.read()
+            blob = json.loads(content)
+
+            # ¯\_(ツ)_/¯
+            blob = {int(k): {
+                int(_k): {
+                    int(__k): __v for __k, __v in _v.items()
+                } for _k, _v in v.items()
+            } for k, v in blob.items()}
+
+            await self.cache_set(self._cache_key, blob)
+
+    async def _write(self, blob: dict) -> None:
+        data = json.dumps(blob, sort_keys=True, indent=4)
+        async with aiofiles.open(self._path, mode="w") as f:
+            await f.write(data)
+
+    @staticmethod
+    async def get(year: int, month: int = None) -> Union[dict, None]:
+        """This function is called when a Feather wallet client asks
+        for (a range of) historical fiat information. It returns the
+        data filtered by the parameters."""
+        from wowlet_backend.factory import cache
+
+        blob = await cache.get("historical_fiat")
+        blob = json.loads(blob)
+        if year not in blob:
+            return
+
+        rtn = {}
+        if not month:
+            for _m, days in blob[year].items():
+                for day, price in days.items():
+                    rtn[datetime(year, _m, day).strftime('%Y%m%d')] = price
+            return rtn
+
+        if month not in blob[year]:
+            return
+
+        for day, price in blob[year][month].items():
+            rtn[datetime(year, month, day).strftime('%Y%m%d')] = price
+
+        return rtn

wowlet_backend/tasks/proposals.py

@@ -0,0 +1,138 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2020, The Monero Project.
+# Copyright (c) 2020, dsc@xmr.pm
+
+from bs4 import BeautifulSoup
+from typing import List
+
+import settings
+from wowlet_backend.utils import httpget
+from wowlet_backend.tasks import FeatherTask
+
+
+class FundingProposalsTask(FeatherTask):
+    """Fetch funding proposals made by the community."""
+    def __init__(self, interval: int = 600):
+        from wowlet_backend.factory import app
+        super(FundingProposalsTask, self).__init__(interval)
+
+        self._cache_key = "funding_proposals"
+        self._cache_expiry = self.interval * 1000
+
+        # url
+        self._http_endpoints = {
+            "xmr": "https://ccs.getmonero.org",
+            "wow": "https://funding.wownero.com"
+        }
+
+        if settings.COIN_SYMBOL not in self._http_endpoints:
+            app.logger.warning(f"Missing proposal URL for {settings.COIN_SYMBOL.upper()}, ignoring update task")
+            self._active = False
+
+        self._http_endpoint = self._http_endpoints[settings.COIN_SYMBOL]
+        if self._http_endpoint.endswith("/"):
+            self._http_endpoint = self._http_endpoint[:-1]
+
+        # websocket
+        self._websocket_cmd = "funding_proposals"
+        self._websocket_cmds = {
+            "xmr": "ccs",
+            "wow": "wfs"
+        }
+
+        if settings.COIN_SYMBOL not in self._websocket_cmds:
+            app.logger.warning(f"Missing websocket cmd for {settings.COIN_SYMBOL.upper()}, ignoring update task")
+            self._active = False
+
+        self._websocket_cmd = self._websocket_cmds[settings.COIN_SYMBOL]
+
+    async def task(self):
+        if settings.COIN_SYMBOL == "xmr":
+            return await self._xmr()
+        elif settings.COIN_SYMBOL == "wow":
+            return await self._wfs()
+
+    async def _xmr(self) -> List[dict]:
+        # CCS API is lacking;
+        # - API returns more `FUNDING-REQUIRED` proposals than there are on the website
+        # - API does not allow filtering
+        # - API sometimes breaks; https://hackerone.com/reports/934231
+        # we'll web scrape instead
+        from wowlet_backend.factory import app
+
+        content = await httpget(f"{self._http_endpoint}/funding-required/", json=False)
+        soup = BeautifulSoup(content, "html.parser")
+
+        listings = []
+        for listing in soup.findAll("a", {"class": "ffs-idea"}):
+            try:
+                item = {
+                    "state": "FUNDING-REQUIRED",
+                    "author": listing.find("p", {"class": "author-list"}).text,
+                    "date": listing.find("p", {"class": "date-list"}).text,
+                    "title": listing.find("h3").text,
+                    "raised_amount": float(listing.find("span", {"class": "progress-number-funded"}).text),
+                    "target_amount": float(listing.find("span", {"class": "progress-number-goal"}).text),
+                    "contributors": 0,
+                    "url": f"{self._http_endpoint}{listing.attrs['href']}"
+                }
+                item["percentage_funded"] = item["raised_amount"] * (100 / item["target_amount"])
+                if item["percentage_funded"] >= 100:
+                    item["percentage_funded"] = 100.0
+                try:
+                    item["contributors"] = int(listing.find("p", {"class": "contributor"}).text.split(" ")[0])
+                except:
+                    pass
+
+                href = listing.attrs['href']
+
+                try:
+                    content = await httpget(f"{self._http_endpoint}{href}", json=False)
+                    try:
+                        soup2 = BeautifulSoup(content, "html.parser")
+                    except Exception as ex:
+                        app.logger.error(f"error parsing ccs HTML page: {ex}")
+                        continue
+
+                    try:
+                        instructions = soup2.find("div", {"class": "instructions"})
+                        if not instructions:
+                            raise Exception("could not parse div.instructions, page probably broken")
+                        address = instructions.find("p", {"class": "string"}).text
+                        if not address.strip():
+                            raise Exception(f"error fetching ccs HTML: could not parse address")
+                        item["address"] = address.strip()
+                    except Exception as ex:
+                        app.logger.error(f"error parsing ccs address from HTML: {ex}")
+                        continue
+                except Exception as ex:
+                    app.logger.error(f"error fetching ccs HTML: {ex}")
+                    continue
+                listings.append(item)
+            except Exception as ex:
+                app.logger.error(f"error parsing a ccs item: {ex}")
+
+        return listings
+
+    async def _wfs(self) -> List[dict]:
+        """https://git.wownero.com/wownero/wownero-funding-system"""
+        blob = await httpget(f"{self._http_endpoint}/api/1/proposals?offset=0&limit=10&status=2", json=True)
+        if "data" not in blob:
+            raise Exception("invalid json response")
+
+        listings = []
+        for p in blob['data']:
+            item = {
+                "address": p["addr_donation"],
+                "url": f"{self._http_endpoint}/proposal/{p['id']}",
+                "state": "FUNDING-REQUIRED",
+                "date": p['date_posted'],
+                "title": p['headline'],
+                'target_amount': p['funds_target'],
+                'raised_amount': round(p['funds_target'] / 100 * p['funded_pct'], 2),
+                'contributors': 0,
+                'percentage_funded': round(p['funded_pct'], 2),
+                'author': p['user']
+            }
+            listings.append(item)
+        return listings

wowlet_backend/tasks/rates_crypto.py

@@ -0,0 +1,59 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2020, The Monero Project.
+# Copyright (c) 2020, dsc@xmr.pm
+
+from typing import List, Union
+
+import settings
+from wowlet_backend.utils import httpget
+from wowlet_backend.tasks import FeatherTask
+
+
+class CryptoRatesTask(FeatherTask):
+    def __init__(self, interval: int = 180):
+        super(CryptoRatesTask, self).__init__(interval)
+
+        self._cache_key = "crypto_rates"
+        self._cache_expiry = self.interval * 10
+
+        self._websocket_cmd = "crypto_rates"
+
+        self._http_api_gecko = "https://api.coingecko.com/api/v3"
+
+    async def task(self) -> Union[List[dict], None]:
+        """Fetch USD prices for various coins"""
+        from wowlet_backend.factory import app
+
+        url = f"{self._http_api_gecko}/coins/markets?vs_currency=usd"
+        rates = await httpget(url, json=True)
+
+        # normalize object, too many useless keys
+        rates = [{
+            "id": r["id"],
+            "symbol": r["symbol"],
+            "image": r["image"],
+            "name": r["name"],
+            "current_price": r["current_price"],
+            "price_change_percentage_24h": r["price_change_percentage_24h"]
+        } for r in rates]
+
+        # additional coins as defined by `settings.CRYPTO_RATES_COINS_EXTRA`
+        for coin, symbol in settings.CRYPTO_RATES_COINS_EXTRA.items():
+            url = f"{self._http_api_gecko}/simple/price?ids={coin}&vs_currencies=usd"
+            try:
+                data = await httpget(url, json=True)
+                if coin not in data or "usd" not in data[coin]:
+                    continue
+
+                rates.append({
+                    "id": coin,
+                    "symbol": symbol,
+                    "image": "",
+                    "name": coin.capitalize(),
+                    "current_price": data[coin]["usd"],
+                    "price_change_percentage_24h": 0.0
+                })
+            except Exception as ex:
+                app.logger.error(f"extra coin: {coin}; {ex}")
+
+        return rates

wowlet_backend/tasks/rates_fiat.py

@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2020, The Monero Project.
+# Copyright (c) 2020, dsc@xmr.pm
+
+from wowlet_backend.utils import httpget
+from wowlet_backend.tasks import FeatherTask
+
+
+class FiatRatesTask(FeatherTask):
+    def __init__(self, interval: int = 600):
+        super(FiatRatesTask, self).__init__(interval)
+
+        self._cache_key = "fiat_rates"
+        self._cache_expiry = self.interval * 10
+
+        self._websocket_cmd = "fiat_rates"
+
+        self._http_endpoint = "https://api.exchangeratesapi.io/latest?base=USD"
+
+    async def task(self):
+        """Fetch fiat rates"""
+        result = await httpget(self._http_endpoint, json=True)
+        return result

wowlet_backend/tasks/reddit.py

@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2020, The Monero Project.
+# Copyright (c) 2020, dsc@xmr.pm
+
+import html
+import settings
+from wowlet_backend.utils import httpget
+from wowlet_backend.tasks import FeatherTask
+
+
+class RedditTask(FeatherTask):
+    def __init__(self, interval: int = 900):
+        from wowlet_backend.factory import app
+        super(RedditTask, self).__init__(interval)
+
+        self._cache_key = "reddit"
+        self._cache_expiry = self.interval * 10
+
+        self._websocket_cmd = "reddit"
+
+        self._http_endpoints = {
+            "xmr": "https://www.reddit.com/r/monero",
+            "wow": "https://www.reddit.com/r/wownero",
+            "aeon": "https://www.reddit.com/r/aeon",
+            "trtl": "https://www.reddit.com/r/TRTL",
+            "xhv": "https://www.reddit.com/r/havenprotocol",
+            "loki": "https://www.reddit.com/r/LokiProject"
+        }
+
+        if settings.COIN_SYMBOL not in self._http_endpoints:
+            app.logger.warning(f"Missing Reddit URL for {settings.COIN_SYMBOL.upper()}, ignoring update task")
+            self._active = False
+
+        self._http_endpoint = self._http_endpoints[settings.COIN_SYMBOL]
+        if self._http_endpoint.endswith("/"):
+            self._http_endpoint = self._http_endpoint[:-1]
+
+    async def task(self):
+        from wowlet_backend.factory import app
+
+        url = f"{self._http_endpoint}/new.json?limit=15"
+        try:
+            blob = await httpget(url, json=True, raise_for_status=True)
+        except Exception as ex:
+            app.logger.error(f"failed fetching '{url}' {ex}")
+            raise
+
+        blob = [{
+            'title': html.unescape(z['data']['title']),
+            'author': z['data']['author'],
+            'url': "https://old.reddit.com" + z['data']['permalink'],  # legacy
+            'permalink': z['data']['permalink'],
+            'comments': z['data']['num_comments']
+        } for z in blob['data']['children']]
+        if not blob:
+            raise Exception("no content")
+
+        return blob

wowlet_backend/tasks/rpc_nodes.py

@@ -0,0 +1,116 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2020, The Monero Project.
+# Copyright (c) 2020, dsc@xmr.pm
+
+import json
+from typing import List
+
+import settings
+from wowlet_backend.utils import httpget, popularity_contest
+from wowlet_backend.tasks import FeatherTask
+
+
+class RPCNodeCheckTask(FeatherTask):
+    def __init__(self, interval: int = 60):
+        super(RPCNodeCheckTask, self).__init__(interval)
+
+        self._cache_key = "rpc_nodes"
+        self._cache_expiry = None
+
+        self._websocket_cmd = "nodes"
+
+        self._http_timeout = 5
+        self._http_timeout_onion = 10
+
+    async def task(self) -> List[dict]:
+        """Check RPC nodes status"""
+        from wowlet_backend.factory import app, cache
+
+        try:
+            heights = json.loads(await cache.get("blockheights"))
+        except:
+            heights = {}
+
+        rpc_nodes = await self.cache_json_get("nodes")
+
+        nodes = []
+        for network_type_coin, _ in rpc_nodes.items():
+            data = []
+
+            for network_type, _nodes in _.items():
+                for node in _nodes:
+                    try:
+                        blob = await self.node_check(node, network_type=network_type)
+                        data.append(blob)
+                    except Exception as ex:
+                        app.logger.warning(f"node {node} not reachable; {ex}")
+                        data.append(self._bad_node({
+                            "address": node,
+                            "nettype": network_type_coin,
+                            "type": network_type,
+                            "height": 0
+                        }, reason="unreachable"))
+
+            # not neccesary for stagenet/testnet nodes to be validated
+            if network_type_coin != "mainnet":
+                nodes += data
+                continue
+
+            if not data:
+                continue
+
+            # Filter out nodes affected by < v0.17.1.3 sybil attack
+            data = list(map(lambda _node: _node if _node['target_height'] <= _node['height']
+                            else self._bad_node(_node, reason="+2_attack"), data))
+
+            allowed_offset = 3
+            valid_heights = []
+            # current_blockheight = heights.get(network_type_coin, 0)
+
+            # popularity contest
+            common_height = popularity_contest([z['height'] for z in data if z['height'] != 0])
+            valid_heights = range(common_height + allowed_offset, common_height - allowed_offset, -1)
+
+            data = list(map(lambda _node: _node if _node['height'] in valid_heights
+                            else self._bad_node(_node, reason="out_of_sync"), data))
+            nodes += data
+        return nodes
+
+    async def node_check(self, node, network_type: str) -> dict:
+        """Call /get_info on the RPC, return JSON"""
+        opts = {
+            "timeout": self._http_timeout,
+            "json": True
+        }
+
+        if network_type == "tor":
+            opts["socks5"] = settings.TOR_SOCKS_PROXY
+            opts["timeout"] = self._http_timeout_onion
+
+        blob = await httpget(f"http://{node}/get_info", **opts)
+        for expect in ["nettype", "height", "target_height"]:
+            if expect not in blob:
+                raise Exception(f"Invalid JSON response from RPC; expected key '{expect}'")
+
+        height = int(blob.get("height", 0))
+        target_height = int(blob.get("target_height", 0))
+
+        return {
+            "address": node,
+            "height": height,
+            "target_height": target_height,
+            "online": True,
+            "nettype": blob["nettype"],
+            "type": network_type
+        }
+
+    def _bad_node(self, node: dict, reason=""):
+        return {
+            "address": node['address'],
+            "height": node['height'],
+            "target_height": 0,
+            "online": False,
+            "nettype": node['nettype'],
+            "type": node['type'],
+            "reason": reason
+        }

wowlet_backend/tasks/xmrig.py

@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2020, The Monero Project.
+# Copyright (c) 2020, dsc@xmr.pm
+
+from dateutil.parser import parse
+
+import settings
+from wowlet_backend.utils import httpget
+from wowlet_backend.tasks import FeatherTask
+
+
+class XmrigTask(FeatherTask):
+    """Fetches the latest XMRig releases using Github's API"""
+    def __init__(self, interval: int = 43200):
+        super(XmrigTask, self).__init__(interval)
+
+        self._cache_key = "xmrig"
+        self._cache_expiry = self.interval * 10
+
+        self._websocket_cmd = "xmrig"
+
+        self._http_endpoint = "https://api.github.com/repos/xmrig/xmrig/releases"
+
+    async def task(self) -> dict:
+        blob = await httpget(self._http_endpoint)
+        if not isinstance(blob, list) or not blob:
+            raise Exception(f"Invalid JSON response for {self._http_endpoint}")
+        blob = blob[0]
+
+        # only uploaded assets
+        assets = list(filter(lambda k: k['state'] == 'uploaded', blob['assets']))
+
+        # only archives
+        assets = list(filter(lambda k: k['name'].endswith(('.tar.gz', '.zip')), assets))
+
+        version = blob['tag_name']
+        data = {}
+
+        # sort by OS
+        for asset in assets:
+            operating_system = "linux"
+            if "msvc" in asset['name'] or "win64" in asset['name']:
+                operating_system = "windows"
+            elif "macos" in asset["name"]:
+                operating_system = "macos"
+
+            data.setdefault(operating_system, [])
+            data[operating_system].append({
+                "name": asset["name"],
+                "created_at": parse(asset["created_at"]).strftime("%Y-%m-%d"),
+                "url": f"https://github.com/xmrig/xmrig/releases/download/{version}/{asset['name']}",
+                "download_count": int(asset["download_count"])
+            })
+
+        return {
+            "version": version,
+            "assets": data
+        }

wowlet_backend/tasks/xmrto.py

@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2020, The Monero Project.
+# Copyright (c) 2020, dsc@xmr.pm
+
+import settings
+from wowlet_backend.utils import httpget
+from wowlet_backend.tasks import FeatherTask
+
+
+class XmrToTask(FeatherTask):
+    def __init__(self, interval: int = 30):
+        super(XmrToTask, self).__init__(interval)
+
+        self._cache_key = "xmrto_rates"
+        self._cache_expiry = self.interval * 10
+
+        if settings.COIN_MODE == 'stagenet':
+            self._http_endpoint = "https://test.xmr.to/api/v3/xmr2btc/order_parameter_query/"
+        else:
+            self._http_endpoint = "https://xmr.to/api/v3/xmr2btc/order_parameter_query/"
+
+    async def task(self):
+        result = await httpget(self._http_endpoint)
+        if "error" in result:
+            raise Exception(f"${result['error']} ${result['error_msg']}")
+        return result

wowlet_backend/utils.py

@@ -0,0 +1,130 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2020, The Monero Project.
+# Copyright (c) 2020, dsc@xmr.pm
+
+import json
+import asyncio
+import os
+import random
+from datetime import datetime
+from collections import Counter
+from functools import wraps
+from typing import List, Union
+
+import psutil
+import aiohttp
+from aiohttp_socks import ProxyConnector
+
+import settings
+
+
+def print_banner():
+    print(f"""\033[91m
+      █████▒▓█████ ▄▄▄     ▄▄▄█████▓ ██░ ██ ▓█████  ██▀███  
+    ▓██   ▒ ▓█   ▀▒████▄   ▓  ██▒ ▓▒▓██░ ██▒▓█   ▀ ▓██ ▒ ██▒
+    ▒████ ░ ▒███  ▒██  ▀█▄ ▒ ▓██░ ▒░▒██▀▀██░▒███   ▓██ ░▄█ ▒
+    ░▓█▒  ░ ▒▓█  ▄░██▄▄▄▄██░ ▓██▓ ░ ░▓█ ░██ ▒▓█  ▄ ▒██▀▀█▄  
+    ░▒█░    ░▒████▒▓█   ▓██▒ ▒██▒ ░ ░▓█▒░██▓░▒████▒░██▓ ▒██▒
+     ▒ ░    ░░ ▒░ ░▒▒   ▓▒█░ ▒ ░░    ▒ ░░▒░▒░░ ▒░ ░░ ▒▓ ░▒▓░
+     ░       ░ ░  ░ ▒   ▒▒ ░   ░     ▒ ░▒░ ░ ░ ░  ░  ░▒ ░ ▒░
+     ░ ░       ░    ░   ▒    ░       ░  ░░ ░   ░     ░░   ░ 
+               ░  ░     ░  ░         ░  ░  ░   ░  ░   ░  {settings.COIN_SYMBOL}\033[0m
+    """.strip())
+
+
+def collect_websocket(func):
+    @wraps(func)
+    async def wrapper(*args, **kwargs):
+        from wowlet_backend.factory import connected_websockets
+        queue = asyncio.Queue()
+        connected_websockets.add(queue)
+        try:
+            return await func(queue, *args, **kwargs)
+        finally:
+            connected_websockets.remove(queue)
+    return wrapper
+
+
+async def httpget(url: str, json=True, timeout: int = 5, socks5: str = None, raise_for_status=True):
+    headers = {"User-Agent": random_agent()}
+    opts = {"timeout": aiohttp.ClientTimeout(total=timeout)}
+    if socks5:
+        opts['connector'] = ProxyConnector.from_url(socks5)
+
+    async with aiohttp.ClientSession(**opts) as session:
+        async with session.get(url, headers=headers) as response:
+            if raise_for_status:
+                response.raise_for_status()
+
+            result = await response.json() if json else await response.text()
+            if result is None or (isinstance(result, str) and result == ''):
+                raise Exception("empty response from request")
+            return result
+
+
+def random_agent():
+    from wowlet_backend.factory import user_agents
+    return random.choice(user_agents)
+
+
+async def feather_data():
+    """A collection of data collected by
+    `FeatherTask`, for Feather wallet clients."""
+    from wowlet_backend.factory import cache, now
+    data = await cache.get("data")
+    if data:
+        data = json.loads(data)
+        return data
+
+    keys = ["blockheights", "funding_proposals", "crypto_rates", "fiat_rates", "reddit", "rpc_nodes", "xmrig", "xmrto_rates"]
+    data = {keys[i]: json.loads(val) if val else None for i, val in enumerate(await cache.mget(*keys))}
+
+    # @TODO: for backward-compat reasons we're including some legacy keys which can be removed after 1.0 release
+    data['nodes'] = data['rpc_nodes']
+    data['ccs'] = data['funding_proposals']
+    data['wfs'] = data['funding_proposals']
+
+    # start caching when application lifetime is more than 20 seconds
+    if (datetime.now() - now).total_seconds() > 20:
+        await cache.setex("data", 30, json.dumps(data))
+    return data
+
+
+def popularity_contest(lst: List[int]) -> Union[int, None]:
+    """Return most common occurrences of List[int]. If
+    there are no duplicates, return max() instead.
+    """
+    if not lst:
+        return
+    if len(set(lst)) == len(lst):
+        return max(lst)
+    return Counter(lst).most_common(1)[0][0]
+
+
+def current_worker_thread_is_primary() -> bool:
+    """
+    ASGI server (Hypercorn) may start multiple
+    worker threads, but we only want one feather-ws
+    instance to schedule `FeatherTask` tasks at an
+    interval. Therefor this function determines if the
+    current instance is responsible for the
+    recurring Feather tasks.
+    """
+    from wowlet_backend.factory import app
+
+    current_pid = os.getpid()
+    parent_pid = os.getppid()
+    app.logger.debug(f"current_pid: {current_pid}, "
+                     f"parent_pid: {parent_pid}")
+
+    if parent_pid == 0:
+        return True
+
+    parent = psutil.Process(parent_pid)
+    if parent.name() != "hypercorn":
+        return True
+
+    lowest_pid = min(c.pid for c in parent.children(recursive=True) if c.name() == "hypercorn")
+    if current_pid == lowest_pid:
+        return True
+

wowlet_backend/wsparse.py

@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2020, The Monero Project.
+# Copyright (c) 2020, dsc@xmr.pm
+
+
+class WebsocketParse:
+    @staticmethod
+    async def parser(cmd: str, data=None):
+        if cmd == "txFiatHistory":
+            return await WebsocketParse.txFiatHistory(data)
+
+    @staticmethod
+    async def txFiatHistory(data=None):
+        if not data or not isinstance(data, dict):
+            return
+        if "year" not in data or not isinstance(data['year'], int):
+            return
+        if "month" in data and not isinstance(data['month'], int):
+            return
+
+        year = data.get('year')
+        month = data.get('month')
+
+        from wowlet_backend.tasks.historical_prices import HistoricalPriceTask
+        return await HistoricalPriceTask.get(year, month)