Track rate limits, reset after 24 hours

This commit is contained in:
Zed 2023-08-20 11:56:42 +02:00
parent bbd68e6840
commit 3d8858f0d8
3 changed files with 33 additions and 7 deletions

View file

@ -1,5 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-only
import httpclient, asyncdispatch, options, strutils, uri, times, math
import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
import jsony, packedjson, zippy, oauth1
import types, tokens, consts, parserutils, http_pool
import experimental/types/common
@ -129,6 +129,16 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
release(account, invalid=true)
raise rateLimitError()
if body.startsWith("{\"errors"):
let errors = body.fromJson(Errors)
if errors in {invalidToken, badToken}:
echo "fetch error: ", errors
release(account, invalid=true)
raise rateLimitError()
elif errors in {rateLimited}:
account.apis[api].limited = true
echo "rate limited, api: ", $api, ", reqs left: ", account.apis[api].remaining, ", id: ", account.id
proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
fetchImpl result:
if not (result.startsWith('{') or result.startsWith('[')):

View file

@ -3,7 +3,9 @@ import asyncdispatch, times, json, random, strutils, tables
import types
# max requests at a time per account to avoid race conditions
const maxConcurrentReqs = 5
const
maxConcurrentReqs = 5
dayInSeconds = 24 * 60 * 60
var
accountPool: seq[GuestAccount]
@ -19,7 +21,7 @@ proc getPoolJson*(): JsonNode =
totalPending = 0
reqsPerApi: Table[string, int]
let now = epochTime()
let now = epochTime().int
for account in accountPool:
totalPending.inc(account.pending)
@ -29,10 +31,17 @@ proc getPoolJson*(): JsonNode =
}
for api in account.apis.keys:
if (now.int - account.apis[api].reset) / 60 > 15:
continue
let obj = %*{}
if account.apis[api].limited:
obj["limited"] = %true
list[account.id]["apis"][$api] = %account.apis[api].remaining
if account.apis[api].reset > now.int:
obj["remaining"] = %account.apis[api].remaining
list[account.id]["apis"][$api] = obj
if "remaining" notin obj:
continue
let
maxReqs =
@ -65,7 +74,12 @@ proc isLimited(account: GuestAccount; api: Api): bool =
if api in account.apis:
let limit = account.apis[api]
return (limit.remaining <= 10 and limit.reset > epochTime().int)
if limit.limited and (epochTime().int - limit.limitedAt) > dayInSeconds:
account.apis[api].limited = false
echo "account limit reset, api: ", api, ", id: ", account.id
return limit.limited or (limit.remaining <= 10 and limit.reset > epochTime().int)
else:
return false

View file

@ -32,6 +32,8 @@ type
RateLimit* = object
remaining*: int
reset*: int
limited*: bool
limitedAt*: int
GuestAccount* = ref object
id*: string