Replace /.tokens with /.health and /.accounts

This commit is contained in:
Zed 2023-10-31 12:04:32 +00:00
parent 089275826c
commit 4120558649
7 changed files with 86 additions and 49 deletions

View file

@ -23,7 +23,7 @@ redisMaxConnections = 30
hmacKey = "secretkey" # random key for cryptographic signing of video urls hmacKey = "secretkey" # random key for cryptographic signing of video urls
base64Media = false # use base64 encoding for proxied media urls base64Media = false # use base64 encoding for proxied media urls
enableRSS = true # set this to false to disable RSS feeds enableRSS = true # set this to false to disable RSS feeds
enableDebug = false # enable request logs and debug endpoints (/.tokens) enableDebug = false # enable request logs and debug endpoints (/.accounts)
proxy = "" # http/https url, SOCKS proxies are not supported proxy = "" # http/https url, SOCKS proxies are not supported
proxyAuth = "" proxyAuth = ""
tokenCount = 10 tokenCount = 10

View file

@ -1,7 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
import httpclient, asyncdispatch, options, strutils, uri, times, math, tables import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
import jsony, packedjson, zippy, oauth1 import jsony, packedjson, zippy, oauth1
import types, tokens, consts, parserutils, http_pool import types, auth, consts, parserutils, http_pool
import experimental/types/common import experimental/types/common
const const
@ -120,7 +120,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
except OSError as e: except OSError as e:
raise e raise e
except Exception as e: except Exception as e:
let id = if account.isNil: "null" else: account.id let id = if account.isNil: "null" else: $account.id
echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", id, ", url: ", url echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", id, ", url: ", url
raise rateLimitError() raise rateLimitError()
finally: finally:

View file

@ -1,5 +1,5 @@
#SPDX-License-Identifier: AGPL-3.0-only #SPDX-License-Identifier: AGPL-3.0-only
import asyncdispatch, times, json, random, strutils, tables, sets, os import asyncdispatch, times, json, random, strutils, tables, intsets, os
import types import types
import experimental/parser/guestaccount import experimental/parser/guestaccount
@ -7,6 +7,21 @@ import experimental/parser/guestaccount
const const
maxConcurrentReqs = 2 maxConcurrentReqs = 2
dayInSeconds = 24 * 60 * 60 dayInSeconds = 24 * 60 * 60
apiMaxReqs: Table[Api, int] = {
Api.search: 50,
Api.tweetDetail: 150,
Api.photoRail: 180,
Api.userTweets: 500,
Api.userTweetsAndReplies: 500,
Api.userMedia: 500,
Api.userRestId: 500,
Api.userScreenName: 500,
Api.tweetResult: 500,
Api.list: 500,
Api.listTweets: 500,
Api.listMembers: 500,
Api.listBySlug: 500
}.toTable
var var
accountPool: seq[GuestAccount] accountPool: seq[GuestAccount]
@ -15,20 +30,64 @@ var
template log(str: varargs[string, `$`]) = template log(str: varargs[string, `$`]) =
if enableLogging: echo "[accounts] ", str.join("") if enableLogging: echo "[accounts] ", str.join("")
proc getPoolJson*(): JsonNode = proc getAccountPoolHealth*(): JsonNode =
var
list = newJObject()
totalReqs = 0
totalPending = 0
limited: HashSet[string]
reqsPerApi: Table[string, int]
let now = epochTime().int let now = epochTime().int
for account in accountPool: var
totalPending.inc(account.pending) totalReqs = 0
limited: IntSet
reqsPerApi: Table[string, int]
oldest = now
newest = 0
average = 0
var includeAccount = false for account in accountPool:
# Twitter snowflake conversion
let created = ((account.id shr 22) + 1288834974657) div 1000
if created > newest:
newest = created
if created < oldest:
oldest = created
average.inc created
for api in account.apis.keys:
let
apiStatus = account.apis[api]
reqs = apiMaxReqs[api] - apiStatus.remaining
reqsPerApi.mgetOrPut($api, 0).inc reqs
totalReqs.inc reqs
if apiStatus.limited:
limited.incl account.id
if accountPool.len > 0:
average = average div accountPool.len
else:
oldest = 0
average = 0
return %*{
"accounts": %*{
"total": accountPool.len,
"active": accountPool.len - limited.card,
"limited": limited.card,
"oldest": $fromUnix(oldest),
"newest": $fromUnix(newest),
"average": $fromUnix(average)
},
"requests": %*{
"total": totalReqs,
"apis": reqsPerApi
}
}
proc getAccountPoolDebug*(): JsonNode =
let now = epochTime().int
var list = newJObject()
for account in accountPool:
let accountJson = %*{ let accountJson = %*{
"apis": newJObject(), "apis": newJObject(),
"pending": account.pending, "pending": account.pending,
@ -47,37 +106,11 @@ proc getPoolJson*(): JsonNode =
if apiStatus.limited: if apiStatus.limited:
obj["limited"] = %true obj["limited"] = %true
limited.incl account.id
accountJson{"apis", $api} = obj accountJson{"apis", $api} = obj
includeAccount = true list[$account.id] = accountJson
let return %list
maxReqs =
case api
of Api.search: 50
of Api.tweetDetail: 150
of Api.photoRail: 180
of Api.userTweets, Api.userTweetsAndReplies, Api.userMedia,
Api.userRestId, Api.userScreenName,
Api.tweetResult,
Api.list, Api.listTweets, Api.listMembers, Api.listBySlug: 500
reqs = maxReqs - apiStatus.remaining
reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs
totalReqs.inc(reqs)
if includeAccount:
list[account.id] = accountJson
return %*{
"amount": accountPool.len,
"limited": limited.card,
"requests": totalReqs,
"pending": totalPending,
"apis": reqsPerApi,
"accounts": list
}
proc rateLimitError*(): ref RateLimitError = proc rateLimitError*(): ref RateLimitError =
newException(RateLimitError, "rate limited") newException(RateLimitError, "rate limited")

View file

@ -1,3 +1,4 @@
import std/strutils
import jsony import jsony
import ../types/guestaccount import ../types/guestaccount
from ../../types import GuestAccount from ../../types import GuestAccount
@ -5,7 +6,7 @@ from ../../types import GuestAccount
proc toGuestAccount(account: RawAccount): GuestAccount = proc toGuestAccount(account: RawAccount): GuestAccount =
let id = account.oauthToken[0 ..< account.oauthToken.find('-')] let id = account.oauthToken[0 ..< account.oauthToken.find('-')]
result = GuestAccount( result = GuestAccount(
id: id, id: parseBiggestInt(id),
oauthToken: account.oauthToken, oauthToken: account.oauthToken,
oauthSecret: account.oauthTokenSecret oauthSecret: account.oauthTokenSecret
) )

View file

@ -6,7 +6,7 @@ from os import getEnv
import jester import jester
import types, config, prefs, formatters, redis_cache, http_pool, tokens import types, config, prefs, formatters, redis_cache, http_pool, auth
import views/[general, about] import views/[general, about]
import routes/[ import routes/[
preferences, timeline, status, media, search, rss, list, debug, preferences, timeline, status, media, search, rss, list, debug,

View file

@ -1,10 +1,13 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
import jester import jester
import router_utils import router_utils
import ".."/[tokens, types] import ".."/[auth, types]
proc createDebugRouter*(cfg: Config) = proc createDebugRouter*(cfg: Config) =
router debug: router debug:
get "/.tokens": get "/.health":
respJson getAccountPoolHealth()
get "/.accounts":
cond cfg.enableDebug cond cfg.enableDebug
respJson getPoolJson() respJson getAccountPoolDebug()

View file

@ -36,7 +36,7 @@ type
limitedAt*: int limitedAt*: int
GuestAccount* = ref object GuestAccount* = ref object
id*: string id*: BiggestInt
oauthToken*: string oauthToken*: string
oauthSecret*: string oauthSecret*: string
pending*: int pending*: int