diff --git a/src/apiutils.nim b/src/apiutils.nim index 2574485..404c824 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only -import httpclient, asyncdispatch, options, strutils, uri, times, math, tables -import jsony, packedjson, zippy, oauth1 -import types, auth, consts, parserutils, http_pool +import httpclient, asyncdispatch, options, strutils, uri, times, tables +import jsony, packedjson, zippy +import types, tokens, consts, parserutils, http_pool import experimental/types/common import config @@ -30,30 +30,12 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor=""; else: result &= ("cursor", cursor) -proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string = - let - encodedUrl = url.replace(",", "%2C").replace("+", "%20") - params = OAuth1Parameters( - consumerKey: consumerKey, - signatureMethod: "HMAC-SHA1", - timestamp: $int(round(epochTime())), - nonce: "0", - isIncludeVersionToHeader: true, - token: oauthToken - ) - signature = getSignature(HttpGet, encodedUrl, "", params, consumerSecret, oauthTokenSecret) - - params.signature = percentEncode(signature) - - return getOauth1RequestHeader(params)["authorization"] - -proc genHeaders*(url, oauthToken, oauthTokenSecret: string): HttpHeaders = - let header = getOauthHeader(url, oauthToken, oauthTokenSecret) - +proc genHeaders*(token: Token = nil): HttpHeaders = result = newHttpHeaders({ "connection": "keep-alive", - "authorization": header, + "authorization": auth, "content-type": "application/json", + "x-guest-token": if token == nil: "" else: token.tok, "x-twitter-active-user": "yes", "authority": "api.twitter.com", "accept-encoding": "gzip", @@ -62,25 +44,29 @@ proc genHeaders*(url, oauthToken, oauthTokenSecret: string): HttpHeaders = "DNT": "1" }) -template updateAccount() = +template updateToken() = if resp.headers.hasKey(rlRemaining): let remaining = parseInt(resp.headers[rlRemaining]) reset = parseInt(resp.headers[rlReset]) - account.setRateLimit(api, remaining, reset) + token.setRateLimit(api, remaining, reset) template fetchImpl(result, additional_headers, fetchBody) {.dirty.} = once: pool = HttpPool() - var account = await getGuestAccount(api) - if account.oauthToken.len == 0: - echo "[accounts] Empty oauth token, account: ", account.id + var token = await getToken(api) + if token.tok.len == 0: raise rateLimitError() + if len(cfg.cookieHeader) != 0: + additional_headers.add("Cookie", cfg.cookieHeader) + if len(cfg.xCsrfToken) != 0: + additional_headers.add("x-csrf-token", cfg.xCsrfToken) + try: var resp: AsyncResponse - var headers = genHeaders($url, account.oauthToken, account.oauthSecret) + var headers = genHeaders(token) for key, value in additional_headers.pairs(): headers.add(key, value) pool.use(headers): @@ -101,7 +87,7 @@ template fetchImpl(result, additional_headers, fetchBody) {.dirty.} = let remaining = parseInt(resp.headers[rlRemaining]) reset = parseInt(resp.headers[rlReset]) - account.setRateLimit(api, remaining, reset) + token.setRateLimit(api, remaining, reset) if result.len > 0: if resp.headers.getOrDefault("content-encoding") == "gzip": @@ -111,34 +97,36 @@ template fetchImpl(result, additional_headers, fetchBody) {.dirty.} = let errors = result.fromJson(Errors) if errors in {expiredToken, badToken, authorizationError}: echo "fetch error: ", errors - invalidate(account) + release(token, invalid=true) raise rateLimitError() elif errors in {rateLimited}: # rate limit hit, resets after 24 hours - setLimited(account, api) + #setLimited(account, api) raise rateLimitError() elif result.startsWith("429 Too Many Requests"): - echo "[accounts] 429 error, API: ", api, ", account: ", account.id - account.apis[api].remaining = 0 + echo "[accounts] 429 error, API: ", api, ", token: ", token[] + #account.apis[api].remaining = 0 # rate limit hit, resets after the 15 minute window raise rateLimitError() fetchBody + release(token, used=true) + if resp.status == $Http400: raise newException(InternalError, $url) except InternalError as e: raise e except BadClientError as e: + release(token, used=true) raise e except OSError as e: raise e except Exception as e: - let id = if account.isNil: "null" else: $account.id - echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", id, ", url: ", url + echo "error: ", e.name, ", msg: ", e.msg, ", token: ", token[], ", url: ", url + if "length" notin e.msg and "descriptor" notin e.msg: + release(token, invalid=true) raise rateLimitError() - finally: - release(account) template retry(bod) = try: @@ -148,7 +136,6 @@ template retry(bod) = bod proc fetch*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[JsonNode] {.async.} = - retry: var body: string fetchImpl(body, additional_headers): @@ -158,10 +145,12 @@ proc fetch*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders echo resp.status, ": ", body, " --- url: ", url result = newJNull() + updateToken() + let error = result.getError - if error in {expiredToken, badToken, authorizationError}: - echo "fetchBody error: ", error - invalidate(account) + if error in {expiredToken, badToken}: + echo "fetch error: ", result.getError + release(token, invalid=true) raise rateLimitError() proc fetchRaw*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[string] {.async.} = @@ -170,3 +159,12 @@ proc fetchRaw*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHead if not (result.startsWith('{') or result.startsWith('[')): echo resp.status, ": ", result, " --- url: ", url result.setLen(0) + + updateToken() + + if result.startsWith("{\"errors"): + let errors = result.fromJson(Errors) + if errors in {expiredToken, badToken}: + echo "fetch error: ", errors + release(token, invalid=true) + raise rateLimitError() \ No newline at end of file diff --git a/src/config.nim b/src/config.nim index 2c216a2..fe4aba5 100644 --- a/src/config.nim +++ b/src/config.nim @@ -41,7 +41,9 @@ proc getConfig*(path: string): (Config, parseCfg.Config) = enableRss: cfg.get("Config", "enableRSS", true), enableDebug: cfg.get("Config", "enableDebug", false), proxy: cfg.get("Config", "proxy", ""), - proxyAuth: cfg.get("Config", "proxyAuth", "") + proxyAuth: cfg.get("Config", "proxyAuth", ""), + cookieHeader: cfg.get("Config", "cookieHeader", ""), + xCsrfToken: cfg.get("Config", "xCsrfToken", "") ) return (conf, cfg) diff --git a/src/consts.nim b/src/consts.nim index 0a24469..3b5b300 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -2,6 +2,8 @@ import uri, sequtils, strutils const + auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF" + consumerKey* = "3nVuSoBZnx6U4vzUxf5w" consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys" diff --git a/src/nitter.nim b/src/nitter.nim index 62a6026..0221f56 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -7,7 +7,7 @@ from os import getEnv import jester -import types, config, prefs, formatters, redis_cache, http_pool, auth +import types, config, prefs, formatters, redis_cache, http_pool import views/[general, about] import routes/[ preferences, timeline, status, media, search, rss, list, debug, @@ -16,10 +16,9 @@ import routes/[ const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances" const issuesUrl = "https://github.com/zedeus/nitter/issues" -let - accountsPath = getEnv("NITTER_ACCOUNTS_FILE", "./guest_accounts.json") +#let accountsPath = getEnv("NITTER_ACCOUNTS_FILE", "./guest_accounts.json") -initAccountPool(cfg, accountsPath) +# initAccountPool(cfg, accountsPath) if not cfg.enableDebug: # Silence Jester's query warning @@ -51,7 +50,7 @@ createSearchRouter(cfg) createMediaRouter(cfg) createEmbedRouter(cfg) createRssRouter(cfg) -createDebugRouter(cfg) +#createDebugRouter(cfg) settings: port = Port(cfg.port) @@ -91,7 +90,7 @@ routes: const link = a("another instance", href = instancesUrl) resp Http429, showError( &"Instance has been rate limited.
Use {link} or try again later.", cfg) - + extend home, "" extend follow, "" extend rss, "" @@ -103,5 +102,5 @@ routes: extend preferences, "" extend resolver, "" extend embed, "" - extend debug, "" + #extend debug, "" extend unsupported, "" diff --git a/src/routes/debug.nim b/src/routes/debug.nim index 895a285..47a8b22 100644 --- a/src/routes/debug.nim +++ b/src/routes/debug.nim @@ -1,13 +1,17 @@ # SPDX-License-Identifier: AGPL-3.0-only import jester import router_utils -import ".."/[auth, types] +import ".."/[tokens, types] proc createDebugRouter*(cfg: Config) = router debug: - get "/.health": - respJson getAccountPoolHealth() - - get "/.accounts": + get "/.tokens": cond cfg.enableDebug - respJson getAccountPoolDebug() + respJson getPoolJson() + + #get "/.health": + #respJson getAccountPoolHealth() + + #get "/.accounts": + #cond cfg.enableDebug + #respJson getAccountPoolDebug() diff --git a/src/tokens.nim b/src/tokens.nim new file mode 100644 index 0000000..f622e4d --- /dev/null +++ b/src/tokens.nim @@ -0,0 +1,168 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import asyncdispatch, httpclient, times, sequtils, json, random +import strutils, tables +import types, consts + +const + maxConcurrentReqs = 5 # max requests at a time per token, to avoid race conditions + maxLastUse = 1.hours # if a token is unused for 60 minutes, it expires + maxAge = 2.hours + 55.minutes # tokens expire after 3 hours + failDelay = initDuration(minutes=30) + +var + tokenPool: seq[Token] + lastFailed: Time + enableLogging = false + +let headers = newHttpHeaders({"authorization": auth}) + +template log(str) = + if enableLogging: echo "[tokens] ", str + +proc getPoolJson*(): JsonNode = + var + list = newJObject() + totalReqs = 0 + totalPending = 0 + reqsPerApi: Table[string, int] + + for token in tokenPool: + totalPending.inc(token.pending) + list[token.tok] = %*{ + "apis": newJObject(), + "pending": token.pending, + "init": $token.init, + "lastUse": $token.lastUse + } + + for api in token.apis.keys: + list[token.tok]["apis"][$api] = %token.apis[api] + + let + maxReqs = + case api + of Api.photoRail: 180 + #of Api.timeline: 187 + #of Api.userTweets, Api.userTimeline: 300 + of Api.userTweets: 300 + of Api.listMembers, Api.listBySlug, Api.list, Api.listTweets, + Api.userTweetsAndReplies, Api.userMedia, + Api.userRestId, Api.userScreenName, Api.tweetDetail, + Api.tweetResult, Api.search, Api.favorites, + Api.retweeters, Api.favoriters, Api.following, Api.followers: 500 + #of Api.userSearch: 900 + reqs = maxReqs - token.apis[api].remaining + + reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs + totalReqs.inc(reqs) + + return %*{ + "amount": tokenPool.len, + "requests": totalReqs, + "pending": totalPending, + "apis": reqsPerApi, + "tokens": list + } + +proc rateLimitError*(): ref RateLimitError = + newException(RateLimitError, "rate limited") + +proc fetchToken(): Future[Token] {.async.} = + if getTime() - lastFailed < failDelay: + raise rateLimitError() + + let client = newAsyncHttpClient(headers=headers) + + try: + let + resp = await client.postContent(activate) + tokNode = parseJson(resp)["guest_token"] + tok = tokNode.getStr($(tokNode.getInt)) + time = getTime() + + return Token(tok: tok, init: time, lastUse: time) + except Exception as e: + echo "[tokens] fetching token failed: ", e.msg + if "Try again" notin e.msg: + echo "[tokens] fetching tokens paused, resuming in 30 minutes" + lastFailed = getTime() + finally: + client.close() + +proc expired(token: Token): bool = + let time = getTime() + token.init < time - maxAge or token.lastUse < time - maxLastUse + +proc isLimited(token: Token; api: Api): bool = + if token.isNil or token.expired: + return true + + if api in token.apis: + let limit = token.apis[api] + return (limit.remaining <= 10 and limit.reset > epochTime().int) + else: + return false + +proc isReady(token: Token; api: Api): bool = + not (token.isNil or token.pending > maxConcurrentReqs or token.isLimited(api)) + +proc release*(token: Token; used=false; invalid=false) = + if token.isNil: return + if invalid or token.expired: + if invalid: log "discarding invalid token" + elif token.expired: log "discarding expired token" + + let idx = tokenPool.find(token) + if idx > -1: tokenPool.delete(idx) + elif used: + dec token.pending + token.lastUse = getTime() + +proc getToken*(api: Api): Future[Token] {.async.} = + for i in 0 ..< tokenPool.len: + if result.isReady(api): break + release(result) + result = tokenPool.sample() + + if not result.isReady(api): + release(result) + result = await fetchToken() + log "added new token to pool" + tokenPool.add result + + if not result.isNil: + inc result.pending + else: + raise rateLimitError() + +proc setRateLimit*(token: Token; api: Api; remaining, reset: int) = + # avoid undefined behavior in race conditions + if api in token.apis: + let limit = token.apis[api] + if limit.reset >= reset and limit.remaining < remaining: + return + + token.apis[api] = RateLimit(remaining: remaining, reset: reset) + +proc poolTokens*(amount: int) {.async.} = + var futs: seq[Future[Token]] + for i in 0 ..< amount: + futs.add fetchToken() + + for token in futs: + var newToken: Token + + try: newToken = await token + except: discard + + if not newToken.isNil: + log "added new token to pool" + tokenPool.add newToken + +proc initTokenPool*(cfg: Config) {.async.} = + enableLogging = cfg.enableDebug + + while true: + if tokenPool.countIt(not it.isLimited(Api.userTweets)) < cfg.minTokens: + await poolTokens(min(4, cfg.minTokens - tokenPool.len)) + await sleepAsync(2000) \ No newline at end of file diff --git a/src/types.nim b/src/types.nim index 7b4923c..d60248d 100644 --- a/src/types.nim +++ b/src/types.nim @@ -38,6 +38,13 @@ type limited*: bool limitedAt*: int + Token* = ref object + tok*: string + init*: Time + lastUse*: Time + pending*: int + apis*: Table[Api, RateLimit] + GuestAccount* = ref object id*: int64 oauthToken*: string @@ -274,6 +281,8 @@ type enableDebug*: bool proxy*: string proxyAuth*: string + cookieHeader*: string + xCsrfToken*: string rssCacheTime*: int listCacheTime*: int diff --git a/twitter_oauth.sh b/twitter_oauth.sh deleted file mode 100644 index f7cfba0..0000000 --- a/twitter_oauth.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# Grab oauth token for use with Nitter (requires Twitter account). -# results: {"oauth_token":"xxxxxxxxxx-xxxxxxxxx","oauth_token_secret":"xxxxxxxxxxxxxxxxxxxxx"} - -username="" -password="" - -if [[ -z "$username" || -z "$password" ]]; then - echo "needs username and password" - exit 1 -fi - -bearer_token='AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F' -guest_token=$(curl -s -XPOST https://api.twitter.com/1.1/guest/activate.json -H "Authorization: Bearer ${bearer_token}" | jq -r '.guest_token') -base_url='https://api.twitter.com/1.1/onboarding/task.json' -header=(-H "Authorization: Bearer ${bearer_token}" -H "User-Agent: TwitterAndroid/10.21.1" -H "Content-Type: application/json" -H "X-Guest-Token: ${guest_token}") - -# start flow -flow_1=$(curl -si -XPOST "${base_url}?flow_name=login" "${header[@]}") - -# get 'att', now needed in headers, and 'flow_token' from flow_1 -att=$(sed -En 's/^att: (.*)\r/\1/p' <<< "${flow_1}") -flow_token=$(sed -n '$p' <<< "${flow_1}" | jq -r .flow_token) - -# username -token_2=$(curl -s -XPOST "${base_url}" -H "att: ${att}" "${header[@]}" \ - -d '{"flow_token":"'"${flow_token}"'","subtask_inputs":[{"subtask_id":"LoginEnterUserIdentifierSSO","settings_list":{"setting_responses":[{"key":"user_identifier","response_data":{"text_data":{"result":"'"${username}"'"}}}],"link":"next_link"}}]}' | jq -r .flow_token) - -# password -token_3=$(curl -s -XPOST "${base_url}" -H "att: ${att}" "${header[@]}" \ - -d '{"flow_token":"'"${token_2}"'","subtask_inputs":[{"enter_password":{"password":"'"${password}"'","link":"next_link"},"subtask_id":"LoginEnterPassword"}]}' | jq -r .flow_token) - -# finally print oauth_token and secret -curl -s -XPOST "${base_url}" -H "att: ${att}" "${header[@]}" \ - -d '{"flow_token":"'"${token_3}"'","subtask_inputs":[{"check_logged_in_account":{"link":"AccountDuplicationCheck_false"},"subtask_id":"AccountDuplicationCheck"}]}' | \ - jq -c '.subtasks[0]|if(.open_account) then {oauth_token: .open_account.oauth_token, oauth_token_secret: .open_account.oauth_token_secret} else empty end' \ No newline at end of file