Replace /.tokens with /.health and /.accounts
This commit is contained in:
parent
089275826c
commit
4120558649
7 changed files with 86 additions and 49 deletions
|
@ -23,7 +23,7 @@ redisMaxConnections = 30
|
|||
hmacKey = "secretkey" # random key for cryptographic signing of video urls
|
||||
base64Media = false # use base64 encoding for proxied media urls
|
||||
enableRSS = true # set this to false to disable RSS feeds
|
||||
enableDebug = false # enable request logs and debug endpoints (/.tokens)
|
||||
enableDebug = false # enable request logs and debug endpoints (/.accounts)
|
||||
proxy = "" # http/https url, SOCKS proxies are not supported
|
||||
proxyAuth = ""
|
||||
tokenCount = 10
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
|
||||
import jsony, packedjson, zippy, oauth1
|
||||
import types, tokens, consts, parserutils, http_pool
|
||||
import types, auth, consts, parserutils, http_pool
|
||||
import experimental/types/common
|
||||
|
||||
const
|
||||
|
@ -120,7 +120,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
|
|||
except OSError as e:
|
||||
raise e
|
||||
except Exception as e:
|
||||
let id = if account.isNil: "null" else: account.id
|
||||
let id = if account.isNil: "null" else: $account.id
|
||||
echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", id, ", url: ", url
|
||||
raise rateLimitError()
|
||||
finally:
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
#SPDX-License-Identifier: AGPL-3.0-only
|
||||
import asyncdispatch, times, json, random, strutils, tables, sets, os
|
||||
import asyncdispatch, times, json, random, strutils, tables, intsets, os
|
||||
import types
|
||||
import experimental/parser/guestaccount
|
||||
|
||||
|
@ -7,6 +7,21 @@ import experimental/parser/guestaccount
|
|||
const
|
||||
maxConcurrentReqs = 2
|
||||
dayInSeconds = 24 * 60 * 60
|
||||
apiMaxReqs: Table[Api, int] = {
|
||||
Api.search: 50,
|
||||
Api.tweetDetail: 150,
|
||||
Api.photoRail: 180,
|
||||
Api.userTweets: 500,
|
||||
Api.userTweetsAndReplies: 500,
|
||||
Api.userMedia: 500,
|
||||
Api.userRestId: 500,
|
||||
Api.userScreenName: 500,
|
||||
Api.tweetResult: 500,
|
||||
Api.list: 500,
|
||||
Api.listTweets: 500,
|
||||
Api.listMembers: 500,
|
||||
Api.listBySlug: 500
|
||||
}.toTable
|
||||
|
||||
var
|
||||
accountPool: seq[GuestAccount]
|
||||
|
@ -15,20 +30,64 @@ var
|
|||
template log(str: varargs[string, `$`]) =
|
||||
if enableLogging: echo "[accounts] ", str.join("")
|
||||
|
||||
proc getPoolJson*(): JsonNode =
|
||||
var
|
||||
list = newJObject()
|
||||
totalReqs = 0
|
||||
totalPending = 0
|
||||
limited: HashSet[string]
|
||||
reqsPerApi: Table[string, int]
|
||||
|
||||
proc getAccountPoolHealth*(): JsonNode =
|
||||
let now = epochTime().int
|
||||
|
||||
for account in accountPool:
|
||||
totalPending.inc(account.pending)
|
||||
var
|
||||
totalReqs = 0
|
||||
limited: IntSet
|
||||
reqsPerApi: Table[string, int]
|
||||
oldest = now
|
||||
newest = 0
|
||||
average = 0
|
||||
|
||||
var includeAccount = false
|
||||
for account in accountPool:
|
||||
# Twitter snowflake conversion
|
||||
let created = ((account.id shr 22) + 1288834974657) div 1000
|
||||
|
||||
if created > newest:
|
||||
newest = created
|
||||
if created < oldest:
|
||||
oldest = created
|
||||
average.inc created
|
||||
|
||||
for api in account.apis.keys:
|
||||
let
|
||||
apiStatus = account.apis[api]
|
||||
reqs = apiMaxReqs[api] - apiStatus.remaining
|
||||
|
||||
reqsPerApi.mgetOrPut($api, 0).inc reqs
|
||||
totalReqs.inc reqs
|
||||
|
||||
if apiStatus.limited:
|
||||
limited.incl account.id
|
||||
|
||||
if accountPool.len > 0:
|
||||
average = average div accountPool.len
|
||||
else:
|
||||
oldest = 0
|
||||
average = 0
|
||||
|
||||
return %*{
|
||||
"accounts": %*{
|
||||
"total": accountPool.len,
|
||||
"active": accountPool.len - limited.card,
|
||||
"limited": limited.card,
|
||||
"oldest": $fromUnix(oldest),
|
||||
"newest": $fromUnix(newest),
|
||||
"average": $fromUnix(average)
|
||||
},
|
||||
"requests": %*{
|
||||
"total": totalReqs,
|
||||
"apis": reqsPerApi
|
||||
}
|
||||
}
|
||||
|
||||
proc getAccountPoolDebug*(): JsonNode =
|
||||
let now = epochTime().int
|
||||
var list = newJObject()
|
||||
|
||||
for account in accountPool:
|
||||
let accountJson = %*{
|
||||
"apis": newJObject(),
|
||||
"pending": account.pending,
|
||||
|
@ -47,37 +106,11 @@ proc getPoolJson*(): JsonNode =
|
|||
|
||||
if apiStatus.limited:
|
||||
obj["limited"] = %true
|
||||
limited.incl account.id
|
||||
|
||||
accountJson{"apis", $api} = obj
|
||||
includeAccount = true
|
||||
list[$account.id] = accountJson
|
||||
|
||||
let
|
||||
maxReqs =
|
||||
case api
|
||||
of Api.search: 50
|
||||
of Api.tweetDetail: 150
|
||||
of Api.photoRail: 180
|
||||
of Api.userTweets, Api.userTweetsAndReplies, Api.userMedia,
|
||||
Api.userRestId, Api.userScreenName,
|
||||
Api.tweetResult,
|
||||
Api.list, Api.listTweets, Api.listMembers, Api.listBySlug: 500
|
||||
reqs = maxReqs - apiStatus.remaining
|
||||
|
||||
reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs
|
||||
totalReqs.inc(reqs)
|
||||
|
||||
if includeAccount:
|
||||
list[account.id] = accountJson
|
||||
|
||||
return %*{
|
||||
"amount": accountPool.len,
|
||||
"limited": limited.card,
|
||||
"requests": totalReqs,
|
||||
"pending": totalPending,
|
||||
"apis": reqsPerApi,
|
||||
"accounts": list
|
||||
}
|
||||
return %list
|
||||
|
||||
proc rateLimitError*(): ref RateLimitError =
|
||||
newException(RateLimitError, "rate limited")
|
|
@ -1,3 +1,4 @@
|
|||
import std/strutils
|
||||
import jsony
|
||||
import ../types/guestaccount
|
||||
from ../../types import GuestAccount
|
||||
|
@ -5,7 +6,7 @@ from ../../types import GuestAccount
|
|||
proc toGuestAccount(account: RawAccount): GuestAccount =
|
||||
let id = account.oauthToken[0 ..< account.oauthToken.find('-')]
|
||||
result = GuestAccount(
|
||||
id: id,
|
||||
id: parseBiggestInt(id),
|
||||
oauthToken: account.oauthToken,
|
||||
oauthSecret: account.oauthTokenSecret
|
||||
)
|
||||
|
|
|
@ -6,7 +6,7 @@ from os import getEnv
|
|||
|
||||
import jester
|
||||
|
||||
import types, config, prefs, formatters, redis_cache, http_pool, tokens
|
||||
import types, config, prefs, formatters, redis_cache, http_pool, auth
|
||||
import views/[general, about]
|
||||
import routes/[
|
||||
preferences, timeline, status, media, search, rss, list, debug,
|
||||
|
|
|
@ -1,10 +1,13 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
import jester
|
||||
import router_utils
|
||||
import ".."/[tokens, types]
|
||||
import ".."/[auth, types]
|
||||
|
||||
proc createDebugRouter*(cfg: Config) =
|
||||
router debug:
|
||||
get "/.tokens":
|
||||
get "/.health":
|
||||
respJson getAccountPoolHealth()
|
||||
|
||||
get "/.accounts":
|
||||
cond cfg.enableDebug
|
||||
respJson getPoolJson()
|
||||
respJson getAccountPoolDebug()
|
||||
|
|
|
@ -36,7 +36,7 @@ type
|
|||
limitedAt*: int
|
||||
|
||||
GuestAccount* = ref object
|
||||
id*: string
|
||||
id*: BiggestInt
|
||||
oauthToken*: string
|
||||
oauthSecret*: string
|
||||
pending*: int
|
||||
|
|
Loading…
Reference in a new issue