Replace /.tokens with /.health and /.accounts
This commit is contained in:
parent
089275826c
commit
4120558649
7 changed files with 86 additions and 49 deletions
|
@ -23,7 +23,7 @@ redisMaxConnections = 30
|
||||||
hmacKey = "secretkey" # random key for cryptographic signing of video urls
|
hmacKey = "secretkey" # random key for cryptographic signing of video urls
|
||||||
base64Media = false # use base64 encoding for proxied media urls
|
base64Media = false # use base64 encoding for proxied media urls
|
||||||
enableRSS = true # set this to false to disable RSS feeds
|
enableRSS = true # set this to false to disable RSS feeds
|
||||||
enableDebug = false # enable request logs and debug endpoints (/.tokens)
|
enableDebug = false # enable request logs and debug endpoints (/.accounts)
|
||||||
proxy = "" # http/https url, SOCKS proxies are not supported
|
proxy = "" # http/https url, SOCKS proxies are not supported
|
||||||
proxyAuth = ""
|
proxyAuth = ""
|
||||||
tokenCount = 10
|
tokenCount = 10
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
# SPDX-License-Identifier: AGPL-3.0-only
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
|
import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
|
||||||
import jsony, packedjson, zippy, oauth1
|
import jsony, packedjson, zippy, oauth1
|
||||||
import types, tokens, consts, parserutils, http_pool
|
import types, auth, consts, parserutils, http_pool
|
||||||
import experimental/types/common
|
import experimental/types/common
|
||||||
|
|
||||||
const
|
const
|
||||||
|
@ -120,7 +120,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
|
||||||
except OSError as e:
|
except OSError as e:
|
||||||
raise e
|
raise e
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
let id = if account.isNil: "null" else: account.id
|
let id = if account.isNil: "null" else: $account.id
|
||||||
echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", id, ", url: ", url
|
echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", id, ", url: ", url
|
||||||
raise rateLimitError()
|
raise rateLimitError()
|
||||||
finally:
|
finally:
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
#SPDX-License-Identifier: AGPL-3.0-only
|
#SPDX-License-Identifier: AGPL-3.0-only
|
||||||
import asyncdispatch, times, json, random, strutils, tables, sets, os
|
import asyncdispatch, times, json, random, strutils, tables, intsets, os
|
||||||
import types
|
import types
|
||||||
import experimental/parser/guestaccount
|
import experimental/parser/guestaccount
|
||||||
|
|
||||||
|
@ -7,6 +7,21 @@ import experimental/parser/guestaccount
|
||||||
const
|
const
|
||||||
maxConcurrentReqs = 2
|
maxConcurrentReqs = 2
|
||||||
dayInSeconds = 24 * 60 * 60
|
dayInSeconds = 24 * 60 * 60
|
||||||
|
apiMaxReqs: Table[Api, int] = {
|
||||||
|
Api.search: 50,
|
||||||
|
Api.tweetDetail: 150,
|
||||||
|
Api.photoRail: 180,
|
||||||
|
Api.userTweets: 500,
|
||||||
|
Api.userTweetsAndReplies: 500,
|
||||||
|
Api.userMedia: 500,
|
||||||
|
Api.userRestId: 500,
|
||||||
|
Api.userScreenName: 500,
|
||||||
|
Api.tweetResult: 500,
|
||||||
|
Api.list: 500,
|
||||||
|
Api.listTweets: 500,
|
||||||
|
Api.listMembers: 500,
|
||||||
|
Api.listBySlug: 500
|
||||||
|
}.toTable
|
||||||
|
|
||||||
var
|
var
|
||||||
accountPool: seq[GuestAccount]
|
accountPool: seq[GuestAccount]
|
||||||
|
@ -15,20 +30,64 @@ var
|
||||||
template log(str: varargs[string, `$`]) =
|
template log(str: varargs[string, `$`]) =
|
||||||
if enableLogging: echo "[accounts] ", str.join("")
|
if enableLogging: echo "[accounts] ", str.join("")
|
||||||
|
|
||||||
proc getPoolJson*(): JsonNode =
|
proc getAccountPoolHealth*(): JsonNode =
|
||||||
var
|
|
||||||
list = newJObject()
|
|
||||||
totalReqs = 0
|
|
||||||
totalPending = 0
|
|
||||||
limited: HashSet[string]
|
|
||||||
reqsPerApi: Table[string, int]
|
|
||||||
|
|
||||||
let now = epochTime().int
|
let now = epochTime().int
|
||||||
|
|
||||||
for account in accountPool:
|
var
|
||||||
totalPending.inc(account.pending)
|
totalReqs = 0
|
||||||
|
limited: IntSet
|
||||||
|
reqsPerApi: Table[string, int]
|
||||||
|
oldest = now
|
||||||
|
newest = 0
|
||||||
|
average = 0
|
||||||
|
|
||||||
var includeAccount = false
|
for account in accountPool:
|
||||||
|
# Twitter snowflake conversion
|
||||||
|
let created = ((account.id shr 22) + 1288834974657) div 1000
|
||||||
|
|
||||||
|
if created > newest:
|
||||||
|
newest = created
|
||||||
|
if created < oldest:
|
||||||
|
oldest = created
|
||||||
|
average.inc created
|
||||||
|
|
||||||
|
for api in account.apis.keys:
|
||||||
|
let
|
||||||
|
apiStatus = account.apis[api]
|
||||||
|
reqs = apiMaxReqs[api] - apiStatus.remaining
|
||||||
|
|
||||||
|
reqsPerApi.mgetOrPut($api, 0).inc reqs
|
||||||
|
totalReqs.inc reqs
|
||||||
|
|
||||||
|
if apiStatus.limited:
|
||||||
|
limited.incl account.id
|
||||||
|
|
||||||
|
if accountPool.len > 0:
|
||||||
|
average = average div accountPool.len
|
||||||
|
else:
|
||||||
|
oldest = 0
|
||||||
|
average = 0
|
||||||
|
|
||||||
|
return %*{
|
||||||
|
"accounts": %*{
|
||||||
|
"total": accountPool.len,
|
||||||
|
"active": accountPool.len - limited.card,
|
||||||
|
"limited": limited.card,
|
||||||
|
"oldest": $fromUnix(oldest),
|
||||||
|
"newest": $fromUnix(newest),
|
||||||
|
"average": $fromUnix(average)
|
||||||
|
},
|
||||||
|
"requests": %*{
|
||||||
|
"total": totalReqs,
|
||||||
|
"apis": reqsPerApi
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
proc getAccountPoolDebug*(): JsonNode =
|
||||||
|
let now = epochTime().int
|
||||||
|
var list = newJObject()
|
||||||
|
|
||||||
|
for account in accountPool:
|
||||||
let accountJson = %*{
|
let accountJson = %*{
|
||||||
"apis": newJObject(),
|
"apis": newJObject(),
|
||||||
"pending": account.pending,
|
"pending": account.pending,
|
||||||
|
@ -47,37 +106,11 @@ proc getPoolJson*(): JsonNode =
|
||||||
|
|
||||||
if apiStatus.limited:
|
if apiStatus.limited:
|
||||||
obj["limited"] = %true
|
obj["limited"] = %true
|
||||||
limited.incl account.id
|
|
||||||
|
|
||||||
accountJson{"apis", $api} = obj
|
accountJson{"apis", $api} = obj
|
||||||
includeAccount = true
|
list[$account.id] = accountJson
|
||||||
|
|
||||||
let
|
return %list
|
||||||
maxReqs =
|
|
||||||
case api
|
|
||||||
of Api.search: 50
|
|
||||||
of Api.tweetDetail: 150
|
|
||||||
of Api.photoRail: 180
|
|
||||||
of Api.userTweets, Api.userTweetsAndReplies, Api.userMedia,
|
|
||||||
Api.userRestId, Api.userScreenName,
|
|
||||||
Api.tweetResult,
|
|
||||||
Api.list, Api.listTweets, Api.listMembers, Api.listBySlug: 500
|
|
||||||
reqs = maxReqs - apiStatus.remaining
|
|
||||||
|
|
||||||
reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs
|
|
||||||
totalReqs.inc(reqs)
|
|
||||||
|
|
||||||
if includeAccount:
|
|
||||||
list[account.id] = accountJson
|
|
||||||
|
|
||||||
return %*{
|
|
||||||
"amount": accountPool.len,
|
|
||||||
"limited": limited.card,
|
|
||||||
"requests": totalReqs,
|
|
||||||
"pending": totalPending,
|
|
||||||
"apis": reqsPerApi,
|
|
||||||
"accounts": list
|
|
||||||
}
|
|
||||||
|
|
||||||
proc rateLimitError*(): ref RateLimitError =
|
proc rateLimitError*(): ref RateLimitError =
|
||||||
newException(RateLimitError, "rate limited")
|
newException(RateLimitError, "rate limited")
|
|
@ -1,3 +1,4 @@
|
||||||
|
import std/strutils
|
||||||
import jsony
|
import jsony
|
||||||
import ../types/guestaccount
|
import ../types/guestaccount
|
||||||
from ../../types import GuestAccount
|
from ../../types import GuestAccount
|
||||||
|
@ -5,7 +6,7 @@ from ../../types import GuestAccount
|
||||||
proc toGuestAccount(account: RawAccount): GuestAccount =
|
proc toGuestAccount(account: RawAccount): GuestAccount =
|
||||||
let id = account.oauthToken[0 ..< account.oauthToken.find('-')]
|
let id = account.oauthToken[0 ..< account.oauthToken.find('-')]
|
||||||
result = GuestAccount(
|
result = GuestAccount(
|
||||||
id: id,
|
id: parseBiggestInt(id),
|
||||||
oauthToken: account.oauthToken,
|
oauthToken: account.oauthToken,
|
||||||
oauthSecret: account.oauthTokenSecret
|
oauthSecret: account.oauthTokenSecret
|
||||||
)
|
)
|
||||||
|
|
|
@ -6,7 +6,7 @@ from os import getEnv
|
||||||
|
|
||||||
import jester
|
import jester
|
||||||
|
|
||||||
import types, config, prefs, formatters, redis_cache, http_pool, tokens
|
import types, config, prefs, formatters, redis_cache, http_pool, auth
|
||||||
import views/[general, about]
|
import views/[general, about]
|
||||||
import routes/[
|
import routes/[
|
||||||
preferences, timeline, status, media, search, rss, list, debug,
|
preferences, timeline, status, media, search, rss, list, debug,
|
||||||
|
|
|
@ -1,10 +1,13 @@
|
||||||
# SPDX-License-Identifier: AGPL-3.0-only
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
import jester
|
import jester
|
||||||
import router_utils
|
import router_utils
|
||||||
import ".."/[tokens, types]
|
import ".."/[auth, types]
|
||||||
|
|
||||||
proc createDebugRouter*(cfg: Config) =
|
proc createDebugRouter*(cfg: Config) =
|
||||||
router debug:
|
router debug:
|
||||||
get "/.tokens":
|
get "/.health":
|
||||||
|
respJson getAccountPoolHealth()
|
||||||
|
|
||||||
|
get "/.accounts":
|
||||||
cond cfg.enableDebug
|
cond cfg.enableDebug
|
||||||
respJson getPoolJson()
|
respJson getAccountPoolDebug()
|
||||||
|
|
|
@ -36,7 +36,7 @@ type
|
||||||
limitedAt*: int
|
limitedAt*: int
|
||||||
|
|
||||||
GuestAccount* = ref object
|
GuestAccount* = ref object
|
||||||
id*: string
|
id*: BiggestInt
|
||||||
oauthToken*: string
|
oauthToken*: string
|
||||||
oauthSecret*: string
|
oauthSecret*: string
|
||||||
pending*: int
|
pending*: int
|
||||||
|
|
Loading…
Reference in a new issue