restore old auth because it actually works as oauth from guest tokens is removed
This commit is contained in:
parent
3bff6ec415
commit
29e4b240d6
8 changed files with 239 additions and 93 deletions
|
@ -1,7 +1,7 @@
|
||||||
# SPDX-License-Identifier: AGPL-3.0-only
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
|
import httpclient, asyncdispatch, options, strutils, uri, times, tables
|
||||||
import jsony, packedjson, zippy, oauth1
|
import jsony, packedjson, zippy
|
||||||
import types, auth, consts, parserutils, http_pool
|
import types, tokens, consts, parserutils, http_pool
|
||||||
import experimental/types/common
|
import experimental/types/common
|
||||||
import config
|
import config
|
||||||
|
|
||||||
|
@ -30,30 +30,12 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
|
||||||
else:
|
else:
|
||||||
result &= ("cursor", cursor)
|
result &= ("cursor", cursor)
|
||||||
|
|
||||||
proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
|
proc genHeaders*(token: Token = nil): HttpHeaders =
|
||||||
let
|
|
||||||
encodedUrl = url.replace(",", "%2C").replace("+", "%20")
|
|
||||||
params = OAuth1Parameters(
|
|
||||||
consumerKey: consumerKey,
|
|
||||||
signatureMethod: "HMAC-SHA1",
|
|
||||||
timestamp: $int(round(epochTime())),
|
|
||||||
nonce: "0",
|
|
||||||
isIncludeVersionToHeader: true,
|
|
||||||
token: oauthToken
|
|
||||||
)
|
|
||||||
signature = getSignature(HttpGet, encodedUrl, "", params, consumerSecret, oauthTokenSecret)
|
|
||||||
|
|
||||||
params.signature = percentEncode(signature)
|
|
||||||
|
|
||||||
return getOauth1RequestHeader(params)["authorization"]
|
|
||||||
|
|
||||||
proc genHeaders*(url, oauthToken, oauthTokenSecret: string): HttpHeaders =
|
|
||||||
let header = getOauthHeader(url, oauthToken, oauthTokenSecret)
|
|
||||||
|
|
||||||
result = newHttpHeaders({
|
result = newHttpHeaders({
|
||||||
"connection": "keep-alive",
|
"connection": "keep-alive",
|
||||||
"authorization": header,
|
"authorization": auth,
|
||||||
"content-type": "application/json",
|
"content-type": "application/json",
|
||||||
|
"x-guest-token": if token == nil: "" else: token.tok,
|
||||||
"x-twitter-active-user": "yes",
|
"x-twitter-active-user": "yes",
|
||||||
"authority": "api.twitter.com",
|
"authority": "api.twitter.com",
|
||||||
"accept-encoding": "gzip",
|
"accept-encoding": "gzip",
|
||||||
|
@ -62,25 +44,29 @@ proc genHeaders*(url, oauthToken, oauthTokenSecret: string): HttpHeaders =
|
||||||
"DNT": "1"
|
"DNT": "1"
|
||||||
})
|
})
|
||||||
|
|
||||||
template updateAccount() =
|
template updateToken() =
|
||||||
if resp.headers.hasKey(rlRemaining):
|
if resp.headers.hasKey(rlRemaining):
|
||||||
let
|
let
|
||||||
remaining = parseInt(resp.headers[rlRemaining])
|
remaining = parseInt(resp.headers[rlRemaining])
|
||||||
reset = parseInt(resp.headers[rlReset])
|
reset = parseInt(resp.headers[rlReset])
|
||||||
account.setRateLimit(api, remaining, reset)
|
token.setRateLimit(api, remaining, reset)
|
||||||
|
|
||||||
template fetchImpl(result, additional_headers, fetchBody) {.dirty.} =
|
template fetchImpl(result, additional_headers, fetchBody) {.dirty.} =
|
||||||
once:
|
once:
|
||||||
pool = HttpPool()
|
pool = HttpPool()
|
||||||
|
|
||||||
var account = await getGuestAccount(api)
|
var token = await getToken(api)
|
||||||
if account.oauthToken.len == 0:
|
if token.tok.len == 0:
|
||||||
echo "[accounts] Empty oauth token, account: ", account.id
|
|
||||||
raise rateLimitError()
|
raise rateLimitError()
|
||||||
|
|
||||||
|
if len(cfg.cookieHeader) != 0:
|
||||||
|
additional_headers.add("Cookie", cfg.cookieHeader)
|
||||||
|
if len(cfg.xCsrfToken) != 0:
|
||||||
|
additional_headers.add("x-csrf-token", cfg.xCsrfToken)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
var resp: AsyncResponse
|
var resp: AsyncResponse
|
||||||
var headers = genHeaders($url, account.oauthToken, account.oauthSecret)
|
var headers = genHeaders(token)
|
||||||
for key, value in additional_headers.pairs():
|
for key, value in additional_headers.pairs():
|
||||||
headers.add(key, value)
|
headers.add(key, value)
|
||||||
pool.use(headers):
|
pool.use(headers):
|
||||||
|
@ -101,7 +87,7 @@ template fetchImpl(result, additional_headers, fetchBody) {.dirty.} =
|
||||||
let
|
let
|
||||||
remaining = parseInt(resp.headers[rlRemaining])
|
remaining = parseInt(resp.headers[rlRemaining])
|
||||||
reset = parseInt(resp.headers[rlReset])
|
reset = parseInt(resp.headers[rlReset])
|
||||||
account.setRateLimit(api, remaining, reset)
|
token.setRateLimit(api, remaining, reset)
|
||||||
|
|
||||||
if result.len > 0:
|
if result.len > 0:
|
||||||
if resp.headers.getOrDefault("content-encoding") == "gzip":
|
if resp.headers.getOrDefault("content-encoding") == "gzip":
|
||||||
|
@ -111,34 +97,36 @@ template fetchImpl(result, additional_headers, fetchBody) {.dirty.} =
|
||||||
let errors = result.fromJson(Errors)
|
let errors = result.fromJson(Errors)
|
||||||
if errors in {expiredToken, badToken, authorizationError}:
|
if errors in {expiredToken, badToken, authorizationError}:
|
||||||
echo "fetch error: ", errors
|
echo "fetch error: ", errors
|
||||||
invalidate(account)
|
release(token, invalid=true)
|
||||||
raise rateLimitError()
|
raise rateLimitError()
|
||||||
elif errors in {rateLimited}:
|
elif errors in {rateLimited}:
|
||||||
# rate limit hit, resets after 24 hours
|
# rate limit hit, resets after 24 hours
|
||||||
setLimited(account, api)
|
#setLimited(account, api)
|
||||||
raise rateLimitError()
|
raise rateLimitError()
|
||||||
elif result.startsWith("429 Too Many Requests"):
|
elif result.startsWith("429 Too Many Requests"):
|
||||||
echo "[accounts] 429 error, API: ", api, ", account: ", account.id
|
echo "[accounts] 429 error, API: ", api, ", token: ", token[]
|
||||||
account.apis[api].remaining = 0
|
#account.apis[api].remaining = 0
|
||||||
# rate limit hit, resets after the 15 minute window
|
# rate limit hit, resets after the 15 minute window
|
||||||
raise rateLimitError()
|
raise rateLimitError()
|
||||||
|
|
||||||
fetchBody
|
fetchBody
|
||||||
|
|
||||||
|
release(token, used=true)
|
||||||
|
|
||||||
if resp.status == $Http400:
|
if resp.status == $Http400:
|
||||||
raise newException(InternalError, $url)
|
raise newException(InternalError, $url)
|
||||||
except InternalError as e:
|
except InternalError as e:
|
||||||
raise e
|
raise e
|
||||||
except BadClientError as e:
|
except BadClientError as e:
|
||||||
|
release(token, used=true)
|
||||||
raise e
|
raise e
|
||||||
except OSError as e:
|
except OSError as e:
|
||||||
raise e
|
raise e
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
let id = if account.isNil: "null" else: $account.id
|
echo "error: ", e.name, ", msg: ", e.msg, ", token: ", token[], ", url: ", url
|
||||||
echo "error: ", e.name, ", msg: ", e.msg, ", accountId: ", id, ", url: ", url
|
if "length" notin e.msg and "descriptor" notin e.msg:
|
||||||
|
release(token, invalid=true)
|
||||||
raise rateLimitError()
|
raise rateLimitError()
|
||||||
finally:
|
|
||||||
release(account)
|
|
||||||
|
|
||||||
template retry(bod) =
|
template retry(bod) =
|
||||||
try:
|
try:
|
||||||
|
@ -148,7 +136,6 @@ template retry(bod) =
|
||||||
bod
|
bod
|
||||||
|
|
||||||
proc fetch*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[JsonNode] {.async.} =
|
proc fetch*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[JsonNode] {.async.} =
|
||||||
|
|
||||||
retry:
|
retry:
|
||||||
var body: string
|
var body: string
|
||||||
fetchImpl(body, additional_headers):
|
fetchImpl(body, additional_headers):
|
||||||
|
@ -158,10 +145,12 @@ proc fetch*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders
|
||||||
echo resp.status, ": ", body, " --- url: ", url
|
echo resp.status, ": ", body, " --- url: ", url
|
||||||
result = newJNull()
|
result = newJNull()
|
||||||
|
|
||||||
|
updateToken()
|
||||||
|
|
||||||
let error = result.getError
|
let error = result.getError
|
||||||
if error in {expiredToken, badToken, authorizationError}:
|
if error in {expiredToken, badToken}:
|
||||||
echo "fetchBody error: ", error
|
echo "fetch error: ", result.getError
|
||||||
invalidate(account)
|
release(token, invalid=true)
|
||||||
raise rateLimitError()
|
raise rateLimitError()
|
||||||
|
|
||||||
proc fetchRaw*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[string] {.async.} =
|
proc fetchRaw*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[string] {.async.} =
|
||||||
|
@ -170,3 +159,12 @@ proc fetchRaw*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHead
|
||||||
if not (result.startsWith('{') or result.startsWith('[')):
|
if not (result.startsWith('{') or result.startsWith('[')):
|
||||||
echo resp.status, ": ", result, " --- url: ", url
|
echo resp.status, ": ", result, " --- url: ", url
|
||||||
result.setLen(0)
|
result.setLen(0)
|
||||||
|
|
||||||
|
updateToken()
|
||||||
|
|
||||||
|
if result.startsWith("{\"errors"):
|
||||||
|
let errors = result.fromJson(Errors)
|
||||||
|
if errors in {expiredToken, badToken}:
|
||||||
|
echo "fetch error: ", errors
|
||||||
|
release(token, invalid=true)
|
||||||
|
raise rateLimitError()
|
|
@ -41,7 +41,9 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
|
||||||
enableRss: cfg.get("Config", "enableRSS", true),
|
enableRss: cfg.get("Config", "enableRSS", true),
|
||||||
enableDebug: cfg.get("Config", "enableDebug", false),
|
enableDebug: cfg.get("Config", "enableDebug", false),
|
||||||
proxy: cfg.get("Config", "proxy", ""),
|
proxy: cfg.get("Config", "proxy", ""),
|
||||||
proxyAuth: cfg.get("Config", "proxyAuth", "")
|
proxyAuth: cfg.get("Config", "proxyAuth", ""),
|
||||||
|
cookieHeader: cfg.get("Config", "cookieHeader", ""),
|
||||||
|
xCsrfToken: cfg.get("Config", "xCsrfToken", "")
|
||||||
)
|
)
|
||||||
|
|
||||||
return (conf, cfg)
|
return (conf, cfg)
|
||||||
|
|
|
@ -2,6 +2,8 @@
|
||||||
import uri, sequtils, strutils
|
import uri, sequtils, strutils
|
||||||
|
|
||||||
const
|
const
|
||||||
|
auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
|
||||||
|
|
||||||
consumerKey* = "3nVuSoBZnx6U4vzUxf5w"
|
consumerKey* = "3nVuSoBZnx6U4vzUxf5w"
|
||||||
consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
|
consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
|
||||||
|
|
||||||
|
|
|
@ -7,7 +7,7 @@ from os import getEnv
|
||||||
|
|
||||||
import jester
|
import jester
|
||||||
|
|
||||||
import types, config, prefs, formatters, redis_cache, http_pool, auth
|
import types, config, prefs, formatters, redis_cache, http_pool
|
||||||
import views/[general, about]
|
import views/[general, about]
|
||||||
import routes/[
|
import routes/[
|
||||||
preferences, timeline, status, media, search, rss, list, debug,
|
preferences, timeline, status, media, search, rss, list, debug,
|
||||||
|
@ -16,10 +16,9 @@ import routes/[
|
||||||
const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances"
|
const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances"
|
||||||
const issuesUrl = "https://github.com/zedeus/nitter/issues"
|
const issuesUrl = "https://github.com/zedeus/nitter/issues"
|
||||||
|
|
||||||
let
|
#let accountsPath = getEnv("NITTER_ACCOUNTS_FILE", "./guest_accounts.json")
|
||||||
accountsPath = getEnv("NITTER_ACCOUNTS_FILE", "./guest_accounts.json")
|
|
||||||
|
|
||||||
initAccountPool(cfg, accountsPath)
|
# initAccountPool(cfg, accountsPath)
|
||||||
|
|
||||||
if not cfg.enableDebug:
|
if not cfg.enableDebug:
|
||||||
# Silence Jester's query warning
|
# Silence Jester's query warning
|
||||||
|
@ -51,7 +50,7 @@ createSearchRouter(cfg)
|
||||||
createMediaRouter(cfg)
|
createMediaRouter(cfg)
|
||||||
createEmbedRouter(cfg)
|
createEmbedRouter(cfg)
|
||||||
createRssRouter(cfg)
|
createRssRouter(cfg)
|
||||||
createDebugRouter(cfg)
|
#createDebugRouter(cfg)
|
||||||
|
|
||||||
settings:
|
settings:
|
||||||
port = Port(cfg.port)
|
port = Port(cfg.port)
|
||||||
|
@ -91,7 +90,7 @@ routes:
|
||||||
const link = a("another instance", href = instancesUrl)
|
const link = a("another instance", href = instancesUrl)
|
||||||
resp Http429, showError(
|
resp Http429, showError(
|
||||||
&"Instance has been rate limited.<br>Use {link} or try again later.", cfg)
|
&"Instance has been rate limited.<br>Use {link} or try again later.", cfg)
|
||||||
|
|
||||||
extend home, ""
|
extend home, ""
|
||||||
extend follow, ""
|
extend follow, ""
|
||||||
extend rss, ""
|
extend rss, ""
|
||||||
|
@ -103,5 +102,5 @@ routes:
|
||||||
extend preferences, ""
|
extend preferences, ""
|
||||||
extend resolver, ""
|
extend resolver, ""
|
||||||
extend embed, ""
|
extend embed, ""
|
||||||
extend debug, ""
|
#extend debug, ""
|
||||||
extend unsupported, ""
|
extend unsupported, ""
|
||||||
|
|
|
@ -1,13 +1,17 @@
|
||||||
# SPDX-License-Identifier: AGPL-3.0-only
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
import jester
|
import jester
|
||||||
import router_utils
|
import router_utils
|
||||||
import ".."/[auth, types]
|
import ".."/[tokens, types]
|
||||||
|
|
||||||
proc createDebugRouter*(cfg: Config) =
|
proc createDebugRouter*(cfg: Config) =
|
||||||
router debug:
|
router debug:
|
||||||
get "/.health":
|
get "/.tokens":
|
||||||
respJson getAccountPoolHealth()
|
|
||||||
|
|
||||||
get "/.accounts":
|
|
||||||
cond cfg.enableDebug
|
cond cfg.enableDebug
|
||||||
respJson getAccountPoolDebug()
|
respJson getPoolJson()
|
||||||
|
|
||||||
|
#get "/.health":
|
||||||
|
#respJson getAccountPoolHealth()
|
||||||
|
|
||||||
|
#get "/.accounts":
|
||||||
|
#cond cfg.enableDebug
|
||||||
|
#respJson getAccountPoolDebug()
|
||||||
|
|
168
src/tokens.nim
Normal file
168
src/tokens.nim
Normal file
|
@ -0,0 +1,168 @@
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
import asyncdispatch, httpclient, times, sequtils, json, random
|
||||||
|
import strutils, tables
|
||||||
|
import types, consts
|
||||||
|
|
||||||
|
const
|
||||||
|
maxConcurrentReqs = 5 # max requests at a time per token, to avoid race conditions
|
||||||
|
maxLastUse = 1.hours # if a token is unused for 60 minutes, it expires
|
||||||
|
maxAge = 2.hours + 55.minutes # tokens expire after 3 hours
|
||||||
|
failDelay = initDuration(minutes=30)
|
||||||
|
|
||||||
|
var
|
||||||
|
tokenPool: seq[Token]
|
||||||
|
lastFailed: Time
|
||||||
|
enableLogging = false
|
||||||
|
|
||||||
|
let headers = newHttpHeaders({"authorization": auth})
|
||||||
|
|
||||||
|
template log(str) =
|
||||||
|
if enableLogging: echo "[tokens] ", str
|
||||||
|
|
||||||
|
proc getPoolJson*(): JsonNode =
|
||||||
|
var
|
||||||
|
list = newJObject()
|
||||||
|
totalReqs = 0
|
||||||
|
totalPending = 0
|
||||||
|
reqsPerApi: Table[string, int]
|
||||||
|
|
||||||
|
for token in tokenPool:
|
||||||
|
totalPending.inc(token.pending)
|
||||||
|
list[token.tok] = %*{
|
||||||
|
"apis": newJObject(),
|
||||||
|
"pending": token.pending,
|
||||||
|
"init": $token.init,
|
||||||
|
"lastUse": $token.lastUse
|
||||||
|
}
|
||||||
|
|
||||||
|
for api in token.apis.keys:
|
||||||
|
list[token.tok]["apis"][$api] = %token.apis[api]
|
||||||
|
|
||||||
|
let
|
||||||
|
maxReqs =
|
||||||
|
case api
|
||||||
|
of Api.photoRail: 180
|
||||||
|
#of Api.timeline: 187
|
||||||
|
#of Api.userTweets, Api.userTimeline: 300
|
||||||
|
of Api.userTweets: 300
|
||||||
|
of Api.listMembers, Api.listBySlug, Api.list, Api.listTweets,
|
||||||
|
Api.userTweetsAndReplies, Api.userMedia,
|
||||||
|
Api.userRestId, Api.userScreenName, Api.tweetDetail,
|
||||||
|
Api.tweetResult, Api.search, Api.favorites,
|
||||||
|
Api.retweeters, Api.favoriters, Api.following, Api.followers: 500
|
||||||
|
#of Api.userSearch: 900
|
||||||
|
reqs = maxReqs - token.apis[api].remaining
|
||||||
|
|
||||||
|
reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs
|
||||||
|
totalReqs.inc(reqs)
|
||||||
|
|
||||||
|
return %*{
|
||||||
|
"amount": tokenPool.len,
|
||||||
|
"requests": totalReqs,
|
||||||
|
"pending": totalPending,
|
||||||
|
"apis": reqsPerApi,
|
||||||
|
"tokens": list
|
||||||
|
}
|
||||||
|
|
||||||
|
proc rateLimitError*(): ref RateLimitError =
|
||||||
|
newException(RateLimitError, "rate limited")
|
||||||
|
|
||||||
|
proc fetchToken(): Future[Token] {.async.} =
|
||||||
|
if getTime() - lastFailed < failDelay:
|
||||||
|
raise rateLimitError()
|
||||||
|
|
||||||
|
let client = newAsyncHttpClient(headers=headers)
|
||||||
|
|
||||||
|
try:
|
||||||
|
let
|
||||||
|
resp = await client.postContent(activate)
|
||||||
|
tokNode = parseJson(resp)["guest_token"]
|
||||||
|
tok = tokNode.getStr($(tokNode.getInt))
|
||||||
|
time = getTime()
|
||||||
|
|
||||||
|
return Token(tok: tok, init: time, lastUse: time)
|
||||||
|
except Exception as e:
|
||||||
|
echo "[tokens] fetching token failed: ", e.msg
|
||||||
|
if "Try again" notin e.msg:
|
||||||
|
echo "[tokens] fetching tokens paused, resuming in 30 minutes"
|
||||||
|
lastFailed = getTime()
|
||||||
|
finally:
|
||||||
|
client.close()
|
||||||
|
|
||||||
|
proc expired(token: Token): bool =
|
||||||
|
let time = getTime()
|
||||||
|
token.init < time - maxAge or token.lastUse < time - maxLastUse
|
||||||
|
|
||||||
|
proc isLimited(token: Token; api: Api): bool =
|
||||||
|
if token.isNil or token.expired:
|
||||||
|
return true
|
||||||
|
|
||||||
|
if api in token.apis:
|
||||||
|
let limit = token.apis[api]
|
||||||
|
return (limit.remaining <= 10 and limit.reset > epochTime().int)
|
||||||
|
else:
|
||||||
|
return false
|
||||||
|
|
||||||
|
proc isReady(token: Token; api: Api): bool =
|
||||||
|
not (token.isNil or token.pending > maxConcurrentReqs or token.isLimited(api))
|
||||||
|
|
||||||
|
proc release*(token: Token; used=false; invalid=false) =
|
||||||
|
if token.isNil: return
|
||||||
|
if invalid or token.expired:
|
||||||
|
if invalid: log "discarding invalid token"
|
||||||
|
elif token.expired: log "discarding expired token"
|
||||||
|
|
||||||
|
let idx = tokenPool.find(token)
|
||||||
|
if idx > -1: tokenPool.delete(idx)
|
||||||
|
elif used:
|
||||||
|
dec token.pending
|
||||||
|
token.lastUse = getTime()
|
||||||
|
|
||||||
|
proc getToken*(api: Api): Future[Token] {.async.} =
|
||||||
|
for i in 0 ..< tokenPool.len:
|
||||||
|
if result.isReady(api): break
|
||||||
|
release(result)
|
||||||
|
result = tokenPool.sample()
|
||||||
|
|
||||||
|
if not result.isReady(api):
|
||||||
|
release(result)
|
||||||
|
result = await fetchToken()
|
||||||
|
log "added new token to pool"
|
||||||
|
tokenPool.add result
|
||||||
|
|
||||||
|
if not result.isNil:
|
||||||
|
inc result.pending
|
||||||
|
else:
|
||||||
|
raise rateLimitError()
|
||||||
|
|
||||||
|
proc setRateLimit*(token: Token; api: Api; remaining, reset: int) =
|
||||||
|
# avoid undefined behavior in race conditions
|
||||||
|
if api in token.apis:
|
||||||
|
let limit = token.apis[api]
|
||||||
|
if limit.reset >= reset and limit.remaining < remaining:
|
||||||
|
return
|
||||||
|
|
||||||
|
token.apis[api] = RateLimit(remaining: remaining, reset: reset)
|
||||||
|
|
||||||
|
proc poolTokens*(amount: int) {.async.} =
|
||||||
|
var futs: seq[Future[Token]]
|
||||||
|
for i in 0 ..< amount:
|
||||||
|
futs.add fetchToken()
|
||||||
|
|
||||||
|
for token in futs:
|
||||||
|
var newToken: Token
|
||||||
|
|
||||||
|
try: newToken = await token
|
||||||
|
except: discard
|
||||||
|
|
||||||
|
if not newToken.isNil:
|
||||||
|
log "added new token to pool"
|
||||||
|
tokenPool.add newToken
|
||||||
|
|
||||||
|
proc initTokenPool*(cfg: Config) {.async.} =
|
||||||
|
enableLogging = cfg.enableDebug
|
||||||
|
|
||||||
|
while true:
|
||||||
|
if tokenPool.countIt(not it.isLimited(Api.userTweets)) < cfg.minTokens:
|
||||||
|
await poolTokens(min(4, cfg.minTokens - tokenPool.len))
|
||||||
|
await sleepAsync(2000)
|
|
@ -38,6 +38,13 @@ type
|
||||||
limited*: bool
|
limited*: bool
|
||||||
limitedAt*: int
|
limitedAt*: int
|
||||||
|
|
||||||
|
Token* = ref object
|
||||||
|
tok*: string
|
||||||
|
init*: Time
|
||||||
|
lastUse*: Time
|
||||||
|
pending*: int
|
||||||
|
apis*: Table[Api, RateLimit]
|
||||||
|
|
||||||
GuestAccount* = ref object
|
GuestAccount* = ref object
|
||||||
id*: int64
|
id*: int64
|
||||||
oauthToken*: string
|
oauthToken*: string
|
||||||
|
@ -274,6 +281,8 @@ type
|
||||||
enableDebug*: bool
|
enableDebug*: bool
|
||||||
proxy*: string
|
proxy*: string
|
||||||
proxyAuth*: string
|
proxyAuth*: string
|
||||||
|
cookieHeader*: string
|
||||||
|
xCsrfToken*: string
|
||||||
|
|
||||||
rssCacheTime*: int
|
rssCacheTime*: int
|
||||||
listCacheTime*: int
|
listCacheTime*: int
|
||||||
|
|
|
@ -1,36 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
# Grab oauth token for use with Nitter (requires Twitter account).
|
|
||||||
# results: {"oauth_token":"xxxxxxxxxx-xxxxxxxxx","oauth_token_secret":"xxxxxxxxxxxxxxxxxxxxx"}
|
|
||||||
|
|
||||||
username=""
|
|
||||||
password=""
|
|
||||||
|
|
||||||
if [[ -z "$username" || -z "$password" ]]; then
|
|
||||||
echo "needs username and password"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
bearer_token='AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F'
|
|
||||||
guest_token=$(curl -s -XPOST https://api.twitter.com/1.1/guest/activate.json -H "Authorization: Bearer ${bearer_token}" | jq -r '.guest_token')
|
|
||||||
base_url='https://api.twitter.com/1.1/onboarding/task.json'
|
|
||||||
header=(-H "Authorization: Bearer ${bearer_token}" -H "User-Agent: TwitterAndroid/10.21.1" -H "Content-Type: application/json" -H "X-Guest-Token: ${guest_token}")
|
|
||||||
|
|
||||||
# start flow
|
|
||||||
flow_1=$(curl -si -XPOST "${base_url}?flow_name=login" "${header[@]}")
|
|
||||||
|
|
||||||
# get 'att', now needed in headers, and 'flow_token' from flow_1
|
|
||||||
att=$(sed -En 's/^att: (.*)\r/\1/p' <<< "${flow_1}")
|
|
||||||
flow_token=$(sed -n '$p' <<< "${flow_1}" | jq -r .flow_token)
|
|
||||||
|
|
||||||
# username
|
|
||||||
token_2=$(curl -s -XPOST "${base_url}" -H "att: ${att}" "${header[@]}" \
|
|
||||||
-d '{"flow_token":"'"${flow_token}"'","subtask_inputs":[{"subtask_id":"LoginEnterUserIdentifierSSO","settings_list":{"setting_responses":[{"key":"user_identifier","response_data":{"text_data":{"result":"'"${username}"'"}}}],"link":"next_link"}}]}' | jq -r .flow_token)
|
|
||||||
|
|
||||||
# password
|
|
||||||
token_3=$(curl -s -XPOST "${base_url}" -H "att: ${att}" "${header[@]}" \
|
|
||||||
-d '{"flow_token":"'"${token_2}"'","subtask_inputs":[{"enter_password":{"password":"'"${password}"'","link":"next_link"},"subtask_id":"LoginEnterPassword"}]}' | jq -r .flow_token)
|
|
||||||
|
|
||||||
# finally print oauth_token and secret
|
|
||||||
curl -s -XPOST "${base_url}" -H "att: ${att}" "${header[@]}" \
|
|
||||||
-d '{"flow_token":"'"${token_3}"'","subtask_inputs":[{"check_logged_in_account":{"link":"AccountDuplicationCheck_false"},"subtask_id":"AccountDuplicationCheck"}]}' | \
|
|
||||||
jq -c '.subtasks[0]|if(.open_account) then {oauth_token: .open_account.oauth_token, oauth_token_secret: .open_account.oauth_token_secret} else empty end'
|
|
Loading…
Reference in a new issue