2021-12-27 01:37:38 +00:00
|
|
|
# SPDX-License-Identifier: AGPL-3.0-only
|
2022-01-05 21:48:45 +00:00
|
|
|
import asyncdispatch, httpclient, times, sequtils, json, random
|
|
|
|
import strutils, tables
|
2021-12-26 05:49:27 +00:00
|
|
|
import zippy
|
2022-01-17 03:13:27 +00:00
|
|
|
import types, consts, http_pool
|
2020-06-01 00:16:24 +00:00
|
|
|
|
2021-01-18 06:47:51 +00:00
|
|
|
const
|
2022-01-05 22:38:46 +00:00
|
|
|
maxConcurrentReqs = 5 # max requests at a time per token, to avoid race conditions
|
|
|
|
maxLastUse = 1.hours # if a token is unused for 60 minutes, it expires
|
2022-01-16 16:56:45 +00:00
|
|
|
maxAge = 2.hours + 55.minutes # tokens expire after 3 hours
|
2021-01-18 06:47:51 +00:00
|
|
|
failDelay = initDuration(minutes=30)
|
|
|
|
|
2020-07-09 07:18:14 +00:00
|
|
|
var
|
2022-01-02 10:21:03 +00:00
|
|
|
clientPool: HttpPool
|
|
|
|
tokenPool: seq[Token]
|
2020-07-09 07:18:14 +00:00
|
|
|
lastFailed: Time
|
2021-01-13 13:32:26 +00:00
|
|
|
|
2022-01-05 23:42:18 +00:00
|
|
|
proc getPoolJson*(): JsonNode =
|
|
|
|
var
|
|
|
|
list = newJObject()
|
|
|
|
totalReqs = 0
|
|
|
|
totalPending = 0
|
|
|
|
reqsPerApi: Table[string, int]
|
|
|
|
|
2022-01-05 21:49:16 +00:00
|
|
|
for token in tokenPool:
|
2022-01-05 23:42:18 +00:00
|
|
|
totalPending.inc(token.pending)
|
2022-01-05 21:49:16 +00:00
|
|
|
list[token.tok] = %*{
|
|
|
|
"apis": newJObject(),
|
2022-01-05 22:38:46 +00:00
|
|
|
"pending": token.pending,
|
2022-01-05 21:49:16 +00:00
|
|
|
"init": $token.init,
|
|
|
|
"lastUse": $token.lastUse
|
|
|
|
}
|
|
|
|
|
|
|
|
for api in token.apis.keys:
|
2022-01-05 23:19:09 +00:00
|
|
|
list[token.tok]["apis"][$api] = %token.apis[api]
|
2022-01-05 23:42:18 +00:00
|
|
|
|
|
|
|
let
|
|
|
|
maxReqs =
|
|
|
|
case api
|
|
|
|
of Api.listBySlug, Api.list: 500
|
|
|
|
of Api.timeline: 187
|
|
|
|
else: 180
|
|
|
|
reqs = maxReqs - token.apis[api].remaining
|
|
|
|
|
|
|
|
reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs
|
|
|
|
totalReqs.inc(reqs)
|
|
|
|
|
|
|
|
return %*{
|
|
|
|
"amount": tokenPool.len,
|
|
|
|
"requests": totalReqs,
|
|
|
|
"pending": totalPending,
|
|
|
|
"apis": reqsPerApi,
|
|
|
|
"tokens": list
|
|
|
|
}
|
2021-01-13 13:32:26 +00:00
|
|
|
|
|
|
|
proc rateLimitError*(): ref RateLimitError =
|
2022-01-05 21:48:45 +00:00
|
|
|
newException(RateLimitError, "rate limited")
|
2020-06-01 00:16:24 +00:00
|
|
|
|
|
|
|
proc fetchToken(): Future[Token] {.async.} =
|
2021-01-18 06:47:51 +00:00
|
|
|
if getTime() - lastFailed < failDelay:
|
2021-01-13 13:32:26 +00:00
|
|
|
raise rateLimitError()
|
2020-07-09 07:18:14 +00:00
|
|
|
|
2020-11-07 20:31:03 +00:00
|
|
|
let headers = newHttpHeaders({
|
|
|
|
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
2021-12-26 05:49:27 +00:00
|
|
|
"accept-encoding": "gzip",
|
2020-11-07 20:31:03 +00:00
|
|
|
"accept-language": "en-US,en;q=0.5",
|
|
|
|
"connection": "keep-alive",
|
|
|
|
"authorization": auth
|
|
|
|
})
|
2020-06-01 00:16:24 +00:00
|
|
|
|
2020-06-02 18:37:55 +00:00
|
|
|
try:
|
2022-01-05 21:48:45 +00:00
|
|
|
let
|
|
|
|
resp = clientPool.use(headers): await c.postContent(activate)
|
|
|
|
tokNode = parseJson(uncompress(resp))["guest_token"]
|
|
|
|
tok = tokNode.getStr($(tokNode.getInt))
|
|
|
|
time = getTime()
|
2020-06-01 00:16:24 +00:00
|
|
|
|
2022-01-05 21:48:45 +00:00
|
|
|
return Token(tok: tok, init: time, lastUse: time)
|
2020-06-24 13:02:34 +00:00
|
|
|
except Exception as e:
|
2020-07-09 07:18:14 +00:00
|
|
|
lastFailed = getTime()
|
2021-01-07 21:31:29 +00:00
|
|
|
echo "fetching token failed: ", e.msg
|
2020-06-01 00:16:24 +00:00
|
|
|
|
2022-01-05 21:48:45 +00:00
|
|
|
proc expired(token: Token): bool =
|
2020-06-19 07:45:24 +00:00
|
|
|
let time = getTime()
|
2022-01-05 21:48:45 +00:00
|
|
|
token.init < time - maxAge or token.lastUse < time - maxLastUse
|
2020-06-01 00:16:24 +00:00
|
|
|
|
2022-01-05 21:48:45 +00:00
|
|
|
proc isLimited(token: Token; api: Api): bool =
|
|
|
|
if token.isNil or token.expired:
|
|
|
|
return true
|
|
|
|
|
|
|
|
if api in token.apis:
|
|
|
|
let limit = token.apis[api]
|
2022-01-05 23:19:09 +00:00
|
|
|
return (limit.remaining <= 10 and limit.reset > epochTime().int)
|
2022-01-05 21:48:45 +00:00
|
|
|
else:
|
|
|
|
return false
|
2020-06-01 00:16:24 +00:00
|
|
|
|
2022-01-05 22:38:46 +00:00
|
|
|
proc isReady(token: Token; api: Api): bool =
|
|
|
|
not (token.isNil or token.pending > maxConcurrentReqs or token.isLimited(api))
|
|
|
|
|
|
|
|
proc release*(token: Token; used=false; invalid=false) =
|
|
|
|
if token.isNil: return
|
|
|
|
if invalid or token.expired:
|
2021-01-18 06:47:51 +00:00
|
|
|
let idx = tokenPool.find(token)
|
|
|
|
if idx > -1: tokenPool.delete(idx)
|
2022-01-05 22:38:46 +00:00
|
|
|
elif used:
|
|
|
|
dec token.pending
|
|
|
|
token.lastUse = getTime()
|
2020-06-01 00:16:24 +00:00
|
|
|
|
2022-01-05 21:48:45 +00:00
|
|
|
proc getToken*(api: Api): Future[Token] {.async.} =
|
2020-06-01 00:16:24 +00:00
|
|
|
for i in 0 ..< tokenPool.len:
|
2022-01-05 22:38:46 +00:00
|
|
|
if result.isReady(api): break
|
2021-01-18 06:47:51 +00:00
|
|
|
release(result)
|
2021-01-13 13:32:26 +00:00
|
|
|
result = tokenPool.sample()
|
2020-06-01 00:16:24 +00:00
|
|
|
|
2022-01-05 22:38:46 +00:00
|
|
|
if not result.isReady(api):
|
2021-01-18 06:47:51 +00:00
|
|
|
release(result)
|
2020-06-01 00:16:24 +00:00
|
|
|
result = await fetchToken()
|
2021-01-13 13:32:26 +00:00
|
|
|
tokenPool.add result
|
|
|
|
|
2022-01-05 22:38:46 +00:00
|
|
|
if not result.isNil:
|
|
|
|
inc result.pending
|
|
|
|
else:
|
2021-01-13 13:32:26 +00:00
|
|
|
raise rateLimitError()
|
|
|
|
|
2022-01-05 21:48:45 +00:00
|
|
|
proc setRateLimit*(token: Token; api: Api; remaining, reset: int) =
|
2022-01-05 22:38:46 +00:00
|
|
|
# avoid undefined behavior in race conditions
|
|
|
|
if api in token.apis:
|
|
|
|
let limit = token.apis[api]
|
|
|
|
if limit.reset >= reset and limit.remaining < remaining:
|
|
|
|
return
|
|
|
|
|
|
|
|
token.apis[api] = RateLimit(remaining: remaining, reset: reset)
|
2022-01-05 21:48:45 +00:00
|
|
|
|
2020-06-01 00:16:24 +00:00
|
|
|
proc poolTokens*(amount: int) {.async.} =
|
|
|
|
var futs: seq[Future[Token]]
|
|
|
|
for i in 0 ..< amount:
|
|
|
|
futs.add fetchToken()
|
|
|
|
|
|
|
|
for token in futs:
|
2021-01-13 13:32:26 +00:00
|
|
|
var newToken: Token
|
|
|
|
|
|
|
|
try: newToken = await token
|
|
|
|
except: discard
|
|
|
|
|
2022-01-05 21:48:45 +00:00
|
|
|
if not newToken.isNil:
|
2021-01-13 13:32:26 +00:00
|
|
|
tokenPool.add newToken
|
2020-06-01 00:16:24 +00:00
|
|
|
|
|
|
|
proc initTokenPool*(cfg: Config) {.async.} =
|
2020-11-07 20:31:03 +00:00
|
|
|
clientPool = HttpPool()
|
|
|
|
|
2020-06-01 00:16:24 +00:00
|
|
|
while true:
|
2022-01-05 21:48:45 +00:00
|
|
|
if tokenPool.countIt(not it.isLimited(Api.timeline)) < cfg.minTokens:
|
2020-06-01 11:54:45 +00:00
|
|
|
await poolTokens(min(4, cfg.minTokens - tokenPool.len))
|
|
|
|
await sleepAsync(2000)
|