YellWOWpages API - distribute contacts to WS clients
This commit is contained in:
parent
c3138a6fb3
commit
7e64dcd133
|
@ -1,5 +1,5 @@
|
||||||
quart
|
quart
|
||||||
aioredis
|
aioredis>=2.0.0
|
||||||
aiohttp
|
aiohttp
|
||||||
aiofiles
|
aiofiles
|
||||||
quart_session
|
quart_session
|
||||||
|
|
|
@ -14,21 +14,18 @@ DEBUG = bool_env(os.environ.get("WOWLET_DEBUG", False))
|
||||||
HOST = os.environ.get("WOWLET_HOST", "127.0.0.1")
|
HOST = os.environ.get("WOWLET_HOST", "127.0.0.1")
|
||||||
PORT = int(os.environ.get("WOWLET_PORT", 1337))
|
PORT = int(os.environ.get("WOWLET_PORT", 1337))
|
||||||
|
|
||||||
REDIS_ADDRESS = os.environ.get("WOWLET_REDIS_ADDRESS", "redis://localhost")
|
REDIS_HOST = os.environ.get("WOWLET_REDIS_HOST", "localhost")
|
||||||
|
REDIS_PORT = os.environ.get("WOWLET_REDIS_PORT", 6379)
|
||||||
REDIS_PASSWORD = os.environ.get("WOWLET_REDIS_PASSWORD")
|
REDIS_PASSWORD = os.environ.get("WOWLET_REDIS_PASSWORD")
|
||||||
|
|
||||||
COIN_NAME = os.environ.get("WOWLET_COIN_NAME", "monero").lower() # as per coingecko
|
COIN_NAME = os.environ.get("WOWLET_COIN_NAME", "wownero").lower() # as per coingecko
|
||||||
COIN_SYMBOL = os.environ.get("WOWLET_COIN_SYMBOL", "xmr").lower() # as per coingecko
|
COIN_SYMBOL = os.environ.get("WOWLET_COIN_SYMBOL", "wow").lower() # as per coingecko
|
||||||
COIN_GENESIS_DATE = os.environ.get("WOWLET_COIN_GENESIS_DATE", "20140418")
|
COIN_GENESIS_DATE = os.environ.get("WOWLET_COIN_GENESIS_DATE", "20180401")
|
||||||
COIN_MODE = os.environ.get("WOWLET_COIN_MODE", "mainnet").lower()
|
COIN_MODE = os.environ.get("WOWLET_COIN_MODE", "mainnet").lower()
|
||||||
|
|
||||||
TOR_SOCKS_PROXY = os.environ.get("WOWLET_TOR_SOCKS_PROXY", "socks5://127.0.0.1:9050")
|
TOR_SOCKS_PROXY = os.environ.get("WOWLET_TOR_SOCKS_PROXY", "socks5://127.0.0.1:9050")
|
||||||
|
|
||||||
# while fetching USD price from coingecko, also include these extra coins:
|
# while fetching USD price from coingecko, also include these extra coins:
|
||||||
CRYPTO_RATES_COINS_EXTRA = {
|
CRYPTO_RATES_COINS_EXTRA = {
|
||||||
"wownero": "wow",
|
"wownero": "wow"
|
||||||
"aeon": "aeon",
|
|
||||||
"turtlecoin": "trtl",
|
|
||||||
"haven": "xhv",
|
|
||||||
"loki": "loki"
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,7 +12,7 @@ from quart import Quart
|
||||||
from quart_session import Session
|
from quart_session import Session
|
||||||
import aioredis
|
import aioredis
|
||||||
|
|
||||||
from wowlet_backend.utils import current_worker_thread_is_primary, print_banner
|
from wowlet_backend.utils import print_banner
|
||||||
import settings
|
import settings
|
||||||
|
|
||||||
now = datetime.now()
|
now = datetime.now()
|
||||||
|
@ -20,7 +20,6 @@ app: Quart = None
|
||||||
cache = None
|
cache = None
|
||||||
user_agents: List[str] = None
|
user_agents: List[str] = None
|
||||||
broadcast = MultisubscriberQueue()
|
broadcast = MultisubscriberQueue()
|
||||||
_is_primary_worker_thread = False
|
|
||||||
|
|
||||||
|
|
||||||
async def _setup_nodes(app: Quart):
|
async def _setup_nodes(app: Quart):
|
||||||
|
@ -39,14 +38,16 @@ async def _setup_user_agents(app: Quart):
|
||||||
async def _setup_cache(app: Quart):
|
async def _setup_cache(app: Quart):
|
||||||
global cache
|
global cache
|
||||||
# Each coin has it's own Redis DB index; `redis-cli -n $INDEX`
|
# Each coin has it's own Redis DB index; `redis-cli -n $INDEX`
|
||||||
db = {"xmr": 0, "wow": 1, "aeon": 2, "trtl": 3, "msr": 4, "xhv": 5, "loki": 6}[settings.COIN_SYMBOL]
|
db = {"wow": 0, "xmr": 1, "aeon": 2, "trtl": 3, "msr": 4, "xhv": 5, "loki": 6}[settings.COIN_SYMBOL]
|
||||||
|
|
||||||
data = {
|
data = {
|
||||||
"address": settings.REDIS_ADDRESS,
|
"host": settings.REDIS_HOST,
|
||||||
|
"port": settings.REDIS_PORT,
|
||||||
"db": db,
|
"db": db,
|
||||||
"password": settings.REDIS_PASSWORD if settings.REDIS_PASSWORD else None
|
"password": settings.REDIS_PASSWORD if settings.REDIS_PASSWORD else None
|
||||||
}
|
}
|
||||||
|
|
||||||
cache = await aioredis.create_redis_pool(**data)
|
cache = await aioredis.Redis(**data)
|
||||||
app.config['SESSION_TYPE'] = 'redis'
|
app.config['SESSION_TYPE'] = 'redis'
|
||||||
app.config['SESSION_REDIS'] = cache
|
app.config['SESSION_REDIS'] = cache
|
||||||
Session(app)
|
Session(app)
|
||||||
|
@ -54,13 +55,11 @@ async def _setup_cache(app: Quart):
|
||||||
|
|
||||||
async def _setup_tasks(app: Quart):
|
async def _setup_tasks(app: Quart):
|
||||||
"""Schedules a series of tasks at an interval."""
|
"""Schedules a series of tasks at an interval."""
|
||||||
if not _is_primary_worker_thread:
|
|
||||||
return
|
|
||||||
|
|
||||||
from wowlet_backend.tasks import (
|
from wowlet_backend.tasks import (
|
||||||
BlockheightTask, HistoricalPriceTask, FundingProposalsTask,
|
BlockheightTask, HistoricalPriceTask, FundingProposalsTask,
|
||||||
CryptoRatesTask, FiatRatesTask, RedditTask, RPCNodeCheckTask,
|
CryptoRatesTask, FiatRatesTask, RedditTask, RPCNodeCheckTask,
|
||||||
XmrigTask, SuchWowTask, WowletReleasesTask, ForumThreadsTask)
|
XmrigTask, SuchWowTask, WowletReleasesTask, ForumThreadsTask,
|
||||||
|
YellWowTask)
|
||||||
|
|
||||||
asyncio.create_task(BlockheightTask().start())
|
asyncio.create_task(BlockheightTask().start())
|
||||||
asyncio.create_task(HistoricalPriceTask().start())
|
asyncio.create_task(HistoricalPriceTask().start())
|
||||||
|
@ -72,6 +71,7 @@ async def _setup_tasks(app: Quart):
|
||||||
asyncio.create_task(SuchWowTask().start())
|
asyncio.create_task(SuchWowTask().start())
|
||||||
asyncio.create_task(WowletReleasesTask().start())
|
asyncio.create_task(WowletReleasesTask().start())
|
||||||
asyncio.create_task(ForumThreadsTask().start())
|
asyncio.create_task(ForumThreadsTask().start())
|
||||||
|
asyncio.create_task(YellWowTask().start())
|
||||||
|
|
||||||
if settings.COIN_SYMBOL in ["xmr", "wow"]:
|
if settings.COIN_SYMBOL in ["xmr", "wow"]:
|
||||||
asyncio.create_task(FundingProposalsTask().start())
|
asyncio.create_task(FundingProposalsTask().start())
|
||||||
|
@ -101,11 +101,7 @@ def create_app():
|
||||||
|
|
||||||
@app.before_serving
|
@app.before_serving
|
||||||
async def startup():
|
async def startup():
|
||||||
global _is_primary_worker_thread
|
print_banner()
|
||||||
_is_primary_worker_thread = current_worker_thread_is_primary()
|
|
||||||
|
|
||||||
if _is_primary_worker_thread:
|
|
||||||
print_banner()
|
|
||||||
|
|
||||||
await _setup_cache(app)
|
await _setup_cache(app)
|
||||||
await _setup_nodes(app)
|
await _setup_nodes(app)
|
||||||
|
|
|
@ -9,14 +9,14 @@ import json
|
||||||
from quart import websocket, jsonify, send_from_directory
|
from quart import websocket, jsonify, send_from_directory
|
||||||
|
|
||||||
import settings
|
import settings
|
||||||
from wowlet_backend.factory import app
|
from wowlet_backend.factory import app, broadcast
|
||||||
from wowlet_backend.wsparse import WebsocketParse
|
from wowlet_backend.wsparse import WebsocketParse
|
||||||
from wowlet_backend.utils import broadcast, feather_data
|
from wowlet_backend.utils import wowlet_data
|
||||||
|
|
||||||
|
|
||||||
@app.route("/")
|
@app.route("/")
|
||||||
async def root():
|
async def root():
|
||||||
data = await feather_data()
|
data = await wowlet_data()
|
||||||
return jsonify(data)
|
return jsonify(data)
|
||||||
|
|
||||||
|
|
||||||
|
@ -29,7 +29,7 @@ async def suchwow(name: str):
|
||||||
|
|
||||||
@app.websocket('/ws')
|
@app.websocket('/ws')
|
||||||
async def ws():
|
async def ws():
|
||||||
data = await feather_data()
|
data = await wowlet_data()
|
||||||
|
|
||||||
# blast available data on connect
|
# blast available data on connect
|
||||||
for task_key, task_value in data.items():
|
for task_key, task_value in data.items():
|
||||||
|
@ -54,9 +54,8 @@ async def ws():
|
||||||
continue
|
continue
|
||||||
|
|
||||||
async def tx():
|
async def tx():
|
||||||
while True:
|
async for _data in broadcast.subscribe():
|
||||||
data = await broadcast.get()
|
payload = json.dumps(_data).encode()
|
||||||
payload = json.dumps(data).encode()
|
|
||||||
await websocket.send(payload)
|
await websocket.send(payload)
|
||||||
|
|
||||||
# bidirectional async rx and tx loops
|
# bidirectional async rx and tx loops
|
||||||
|
|
|
@ -167,3 +167,4 @@ from wowlet_backend.tasks.xmrig import XmrigTask
|
||||||
from wowlet_backend.tasks.suchwow import SuchWowTask
|
from wowlet_backend.tasks.suchwow import SuchWowTask
|
||||||
from wowlet_backend.tasks.wowlet import WowletReleasesTask
|
from wowlet_backend.tasks.wowlet import WowletReleasesTask
|
||||||
from wowlet_backend.tasks.forum import ForumThreadsTask
|
from wowlet_backend.tasks.forum import ForumThreadsTask
|
||||||
|
from wowlet_backend.tasks.yellow import YellWowTask
|
||||||
|
|
|
@ -0,0 +1,28 @@
|
||||||
|
# SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
# Copyright (c) 2022, The Monero Project.
|
||||||
|
# Copyright (c) 2022, dsc@xmr.pm
|
||||||
|
|
||||||
|
from dateutil.parser import parse
|
||||||
|
|
||||||
|
import settings
|
||||||
|
from wowlet_backend.utils import httpget
|
||||||
|
from wowlet_backend.tasks import WowletTask
|
||||||
|
|
||||||
|
|
||||||
|
class YellWowTask(WowletTask):
|
||||||
|
"""Fetches yellwowpages usernames/addresses"""
|
||||||
|
def __init__(self, interval: int = 3600):
|
||||||
|
super(YellWowTask, self).__init__(interval)
|
||||||
|
|
||||||
|
self._cache_key = "yellwow"
|
||||||
|
self._cache_expiry = self.interval * 10
|
||||||
|
|
||||||
|
self._websocket_cmd = "yellwow"
|
||||||
|
|
||||||
|
self._http_endpoint = "https://yellow.wownero.com/api/user/"
|
||||||
|
|
||||||
|
async def task(self) -> list[dict]:
|
||||||
|
blob = await httpget(self._http_endpoint)
|
||||||
|
if not isinstance(blob, list) or not blob:
|
||||||
|
raise Exception(f"Invalid JSON response for {self._http_endpoint}")
|
||||||
|
return blob
|
|
@ -1,6 +1,6 @@
|
||||||
# SPDX-License-Identifier: BSD-3-Clause
|
# SPDX-License-Identifier: BSD-3-Clause
|
||||||
# Copyright (c) 2020, The Monero Project.
|
# Copyright (c) 2022, The Monero Project.
|
||||||
# Copyright (c) 2020, dsc@xmr.pm
|
# Copyright (c) 2022, dsc@xmr.pm
|
||||||
|
|
||||||
import re
|
import re
|
||||||
import json
|
import json
|
||||||
|
@ -41,19 +41,6 @@ def print_banner():
|
||||||
""".strip())
|
""".strip())
|
||||||
|
|
||||||
|
|
||||||
def collect_websocket(func):
|
|
||||||
@wraps(func)
|
|
||||||
async def wrapper(*args, **kwargs):
|
|
||||||
from wowlet_backend.factory import connected_websockets
|
|
||||||
queue = asyncio.Queue()
|
|
||||||
connected_websockets.add(queue)
|
|
||||||
try:
|
|
||||||
return await func(queue, *args, **kwargs)
|
|
||||||
finally:
|
|
||||||
connected_websockets.remove(queue)
|
|
||||||
return wrapper
|
|
||||||
|
|
||||||
|
|
||||||
async def httpget(url: str, json=True, timeout: int = 5, socks5: str = None, raise_for_status=True, verify_tls=True):
|
async def httpget(url: str, json=True, timeout: int = 5, socks5: str = None, raise_for_status=True, verify_tls=True):
|
||||||
headers = {"User-Agent": random_agent()}
|
headers = {"User-Agent": random_agent()}
|
||||||
opts = {"timeout": aiohttp.ClientTimeout(total=timeout)}
|
opts = {"timeout": aiohttp.ClientTimeout(total=timeout)}
|
||||||
|
@ -76,23 +63,30 @@ def random_agent():
|
||||||
return random.choice(user_agents)
|
return random.choice(user_agents)
|
||||||
|
|
||||||
|
|
||||||
async def feather_data():
|
async def wowlet_data():
|
||||||
"""A collection of data collected by
|
"""A collection of data collected by the various wowlet tasks"""
|
||||||
`FeatherTask`, for Feather wallet clients."""
|
|
||||||
from wowlet_backend.factory import cache, now
|
from wowlet_backend.factory import cache, now
|
||||||
data = await cache.get("data")
|
data = await cache.get("data")
|
||||||
if data:
|
if data:
|
||||||
data = json.loads(data)
|
data = json.loads(data)
|
||||||
return data
|
return data
|
||||||
|
|
||||||
keys = ["blockheights", "funding_proposals", "crypto_rates", "fiat_rates", "reddit", "rpc_nodes", "xmrig", "xmrto_rates", "suchwow", "forum", "wowlet_releases"]
|
keys = [
|
||||||
|
"blockheights",
|
||||||
|
"funding_proposals",
|
||||||
|
"crypto_rates",
|
||||||
|
"fiat_rates",
|
||||||
|
"reddit",
|
||||||
|
"rpc_nodes",
|
||||||
|
"xmrig",
|
||||||
|
"xmrto_rates",
|
||||||
|
"suchwow",
|
||||||
|
"forum",
|
||||||
|
"wowlet_releases",
|
||||||
|
"yellwow"
|
||||||
|
]
|
||||||
data = {keys[i]: json.loads(val) if val else None for i, val in enumerate(await cache.mget(*keys))}
|
data = {keys[i]: json.loads(val) if val else None for i, val in enumerate(await cache.mget(*keys))}
|
||||||
|
|
||||||
# @TODO: for backward-compat reasons we're including some legacy keys which can be removed after 1.0 release
|
|
||||||
data['nodes'] = data['rpc_nodes']
|
|
||||||
data['ccs'] = data['funding_proposals']
|
|
||||||
data['wfs'] = data['funding_proposals']
|
|
||||||
|
|
||||||
# start caching when application lifetime is more than 20 seconds
|
# start caching when application lifetime is more than 20 seconds
|
||||||
if (datetime.now() - now).total_seconds() > 20:
|
if (datetime.now() - now).total_seconds() > 20:
|
||||||
await cache.setex("data", 30, json.dumps(data))
|
await cache.setex("data", 30, json.dumps(data))
|
||||||
|
@ -110,34 +104,6 @@ def popularity_contest(lst: List[int]) -> Union[int, None]:
|
||||||
return Counter(lst).most_common(1)[0][0]
|
return Counter(lst).most_common(1)[0][0]
|
||||||
|
|
||||||
|
|
||||||
def current_worker_thread_is_primary() -> bool:
|
|
||||||
"""
|
|
||||||
ASGI server (Hypercorn) may start multiple
|
|
||||||
worker threads, but we only want one feather-ws
|
|
||||||
instance to schedule `FeatherTask` tasks at an
|
|
||||||
interval. Therefor this function determines if the
|
|
||||||
current instance is responsible for the
|
|
||||||
recurring Feather tasks.
|
|
||||||
"""
|
|
||||||
from wowlet_backend.factory import app
|
|
||||||
|
|
||||||
current_pid = os.getpid()
|
|
||||||
parent_pid = os.getppid()
|
|
||||||
app.logger.debug(f"current_pid: {current_pid}, "
|
|
||||||
f"parent_pid: {parent_pid}")
|
|
||||||
|
|
||||||
if parent_pid == 0:
|
|
||||||
return True
|
|
||||||
|
|
||||||
parent = psutil.Process(parent_pid)
|
|
||||||
if parent.name() != "hypercorn":
|
|
||||||
return True
|
|
||||||
|
|
||||||
lowest_pid = min(c.pid for c in parent.children(recursive=True) if c.name() == "hypercorn")
|
|
||||||
if current_pid == lowest_pid:
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
async def image_resize(buffer: bytes, max_bounding_box: int = 512, quality: int = 70) -> bytes:
|
async def image_resize(buffer: bytes, max_bounding_box: int = 512, quality: int = 70) -> bytes:
|
||||||
"""
|
"""
|
||||||
- Resize if the image is too large
|
- Resize if the image is too large
|
||||||
|
|
Loading…
Reference in New Issue