GraphQL timeline (#812)
* Update deps * Replace profile timeline with GraphQL endpoint * Update GraphQL endpoint versions * Use GraphQL for profile media tab * Fix UserByRestId request * Improve routing, fixes #814 * Fix token pool JSON * Deduplicate GraphQL timeline endpoints * Update list endpoints * Use GraphQL for list tweets * Remove debug leftover * Replace old pinned tweet endpoint with GraphQL * Validate tweet ID * Minor token handling fix * Hide US-only commerce cards * Update config example * Remove http pool and gzip from token pool * Support tombstoned tweets in threads * Retry GraphQL timeout errors * Remove unnecessary 401 retry * Remove broken timeout retry * Update karax, use new bool attribute feature * Update card test * Fix odd edgecase with broken retweets * Replace search endpoints, switch Bearer token * Only parse user search if it's a list * Fix quoted tweet crash * Fix empty search query handling * Fix invalid user search errors again
This commit is contained in:
Zed
GitHub
parent
e2560dc1f1
commit
1ac389e7c7
29 changed files with 405 additions and 301 deletions
|
|
@ -1,11 +1,11 @@
|
|||
[Server]
|
||||
hostname = "nitter.net" # for generating links, change this to your own domain/ip
|
||||
title = "nitter"
|
||||
address = "0.0.0.0"
|
||||
port = 8080
|
||||
https = false # disable to enable cookies when not using https
|
||||
httpMaxConnections = 100
|
||||
staticDir = "./public"
|
||||
title = "nitter"
|
||||
hostname = "nitter.net"
|
||||
|
||||
[Cache]
|
||||
listMinutes = 240 # how long to cache list info (not the tweets, so keep it high)
|
||||
|
|
@ -13,9 +13,9 @@ rssMinutes = 10 # how long to cache rss queries
|
|||
redisHost = "localhost" # Change to "nitter-redis" if using docker-compose
|
||||
redisPort = 6379
|
||||
redisPassword = ""
|
||||
redisConnections = 20 # connection pool size
|
||||
redisConnections = 20 # minimum open connections in pool
|
||||
redisMaxConnections = 30
|
||||
# max, new connections are opened when none are available, but if the pool size
|
||||
# new connections are opened when none are available, but if the pool size
|
||||
# goes above this, they're closed when released. don't worry about this unless
|
||||
# you receive tons of requests per second
|
||||
|
||||
|
|
@ -23,15 +23,15 @@ redisMaxConnections = 30
|
|||
hmacKey = "secretkey" # random key for cryptographic signing of video urls
|
||||
base64Media = false # use base64 encoding for proxied media urls
|
||||
enableRSS = true # set this to false to disable RSS feeds
|
||||
enableDebug = false # enable request logs and debug endpoints
|
||||
enableDebug = false # enable request logs and debug endpoints (/.tokens)
|
||||
proxy = "" # http/https url, SOCKS proxies are not supported
|
||||
proxyAuth = ""
|
||||
tokenCount = 10
|
||||
# minimum amount of usable tokens. tokens are used to authorize API requests,
|
||||
# but they expire after ~1 hour, and have a limit of 187 requests.
|
||||
# the limit gets reset every 15 minutes, and the pool is filled up so there's
|
||||
# always at least $tokenCount usable tokens. again, only increase this if
|
||||
# you receive major bursts all the time
|
||||
# but they expire after ~1 hour, and have a limit of 500 requests per endpoint.
|
||||
# the limits reset every 15 minutes, and the pool is filled up so there's
|
||||
# always at least `tokenCount` usable tokens. only increase this if you receive
|
||||
# major bursts all the time and don't have a rate limiting setup via e.g. nginx
|
||||
|
||||
# Change default preferences here, see src/prefs_impl.nim for a complete list
|
||||
[Preferences]
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ bin = @["nitter"]
|
|||
|
||||
requires "nim >= 1.4.8"
|
||||
requires "jester#baca3f"
|
||||
requires "karax#9ee695b"
|
||||
requires "karax#5cf360c"
|
||||
requires "sass#7dfdd03"
|
||||
requires "nimcrypto#4014ef9"
|
||||
requires "markdown#158efe3"
|
||||
|
|
|
|||
156
src/api.nim
156
src/api.nim
|
|
@ -7,106 +7,82 @@ import experimental/parser as newParser
|
|||
proc getGraphUser*(username: string): Future[User] {.async.} =
|
||||
if username.len == 0: return
|
||||
let
|
||||
variables = """{
|
||||
"screen_name": "$1",
|
||||
"withSafetyModeUserFields": false,
|
||||
"withSuperFollowsUserFields": false
|
||||
}""" % [username]
|
||||
js = await fetchRaw(graphUser ? {"variables": variables}, Api.userScreenName)
|
||||
variables = %*{"screen_name": username}
|
||||
params = {"variables": $variables, "features": gqlFeatures}
|
||||
js = await fetchRaw(graphUser ? params, Api.userScreenName)
|
||||
result = parseGraphUser(js)
|
||||
|
||||
proc getGraphUserById*(id: string): Future[User] {.async.} =
|
||||
if id.len == 0 or id.any(c => not c.isDigit): return
|
||||
let
|
||||
variables = """{"userId": "$1", "withSuperFollowsUserFields": true}""" % [id]
|
||||
js = await fetchRaw(graphUserById ? {"variables": variables}, Api.userRestId)
|
||||
variables = %*{"userId": id}
|
||||
params = {"variables": $variables, "features": gqlFeatures}
|
||||
js = await fetchRaw(graphUserById ? params, Api.userRestId)
|
||||
result = parseGraphUser(js)
|
||||
|
||||
proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Timeline] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
variables = userTweetsVariables % [id, cursor]
|
||||
params = {"variables": variables, "features": gqlFeatures}
|
||||
(url, apiId) = case kind
|
||||
of TimelineKind.tweets: (graphUserTweets, Api.userTweets)
|
||||
of TimelineKind.replies: (graphUserTweetsAndReplies, Api.userTweetsAndReplies)
|
||||
of TimelineKind.media: (graphUserMedia, Api.userMedia)
|
||||
js = await fetch(url ? params, apiId)
|
||||
result = parseGraphTimeline(js, "user", after)
|
||||
|
||||
proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
variables = listTweetsVariables % [id, cursor]
|
||||
params = {"variables": variables, "features": gqlFeatures}
|
||||
js = await fetch(graphListTweets ? params, Api.listTweets)
|
||||
result = parseGraphTimeline(js, "list", after)
|
||||
|
||||
proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
|
||||
let
|
||||
variables = %*{"screenName": name, "listSlug": list, "withHighlightedLabel": false}
|
||||
url = graphListBySlug ? {"variables": $variables}
|
||||
result = parseGraphList(await fetch(url, Api.listBySlug))
|
||||
variables = %*{"screenName": name, "listSlug": list}
|
||||
params = {"variables": $variables, "features": gqlFeatures}
|
||||
result = parseGraphList(await fetch(graphListBySlug ? params, Api.listBySlug))
|
||||
|
||||
proc getGraphList*(id: string): Future[List] {.async.} =
|
||||
let
|
||||
variables = %*{"listId": id, "withHighlightedLabel": false}
|
||||
url = graphList ? {"variables": $variables}
|
||||
result = parseGraphList(await fetch(url, Api.list))
|
||||
variables = %*{"listId": id}
|
||||
params = {"variables": $variables, "features": gqlFeatures}
|
||||
result = parseGraphList(await fetch(graphListById ? params, Api.list))
|
||||
|
||||
proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
|
||||
if list.id.len == 0: return
|
||||
var
|
||||
variables = %*{
|
||||
"listId": list.id,
|
||||
"withSuperFollowsUserFields": false,
|
||||
"withBirdwatchPivots": false,
|
||||
"withDownvotePerspective": false,
|
||||
"withReactionsMetadata": false,
|
||||
"withReactionsPerspective": false,
|
||||
"withSuperFollowsTweetFields": false
|
||||
"withReactionsPerspective": false
|
||||
}
|
||||
if after.len > 0:
|
||||
variables["cursor"] = % after
|
||||
let url = graphListMembers ? {"variables": $variables}
|
||||
let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures}
|
||||
result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
|
||||
|
||||
proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
|
||||
proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
ps = genParams({"list_id": id, "ranking_mode": "reverse_chronological"}, after)
|
||||
url = listTimeline ? ps
|
||||
result = parseTimeline(await fetch(url, Api.timeline), after)
|
||||
|
||||
proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
|
||||
url = timeline / (id & ".json") ? ps
|
||||
result = parseTimeline(await fetch(url, Api.timeline), after)
|
||||
|
||||
proc getMediaTimeline*(id: string; after=""): Future[Timeline] {.async.} =
|
||||
if id.len == 0: return
|
||||
let url = mediaTimeline / (id & ".json") ? genParams(cursor=after)
|
||||
result = parseTimeline(await fetch(url, Api.timeline), after)
|
||||
|
||||
proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
|
||||
if name.len == 0: return
|
||||
let
|
||||
ps = genParams({"screen_name": name, "trim_user": "true"},
|
||||
count="18", ext=false)
|
||||
url = photoRail ? ps
|
||||
result = parsePhotoRail(await fetch(url, Api.timeline))
|
||||
|
||||
proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
|
||||
when T is User:
|
||||
const
|
||||
searchMode = ("result_filter", "user")
|
||||
parse = parseUsers
|
||||
fetchFunc = fetchRaw
|
||||
else:
|
||||
const
|
||||
searchMode = ("tweet_search_mode", "live")
|
||||
parse = parseTimeline
|
||||
fetchFunc = fetch
|
||||
|
||||
let q = genQueryParam(query)
|
||||
if q.len == 0 or q == emptyQuery:
|
||||
return Result[T](beginning: true, query: query)
|
||||
|
||||
let url = search ? genParams(searchParams & @[("q", q), searchMode], after)
|
||||
try:
|
||||
result = parse(await fetchFunc(url, Api.search), after)
|
||||
result.query = query
|
||||
except InternalError:
|
||||
return Result[T](beginning: true, query: query)
|
||||
variables = tweetResultVariables % id
|
||||
params = {"variables": variables, "features": gqlFeatures}
|
||||
js = await fetch(graphTweetResult ? params, Api.tweetResult)
|
||||
result = parseGraphTweetResult(js)
|
||||
|
||||
proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
variables = tweetVariables % [id, cursor]
|
||||
params = {"variables": variables, "features": tweetFeatures}
|
||||
params = {"variables": variables, "features": gqlFeatures}
|
||||
js = await fetch(graphTweet ? params, Api.tweetDetail)
|
||||
result = parseGraphConversation(js, id)
|
||||
|
||||
|
|
@ -119,9 +95,51 @@ proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
|
|||
if after.len > 0:
|
||||
result.replies = await getReplies(id, after)
|
||||
|
||||
proc getStatus*(id: string): Future[Tweet] {.async.} =
|
||||
let url = status / (id & ".json") ? genParams()
|
||||
result = parseStatus(await fetch(url, Api.status))
|
||||
proc getGraphSearch*(query: Query; after=""): Future[Result[Tweet]] {.async.} =
|
||||
let q = genQueryParam(query)
|
||||
if q.len == 0 or q == emptyQuery:
|
||||
return Result[Tweet](query: query, beginning: true)
|
||||
|
||||
var
|
||||
variables = %*{
|
||||
"rawQuery": q,
|
||||
"count": 20,
|
||||
"product": "Latest",
|
||||
"withDownvotePerspective": false,
|
||||
"withReactionsMetadata": false,
|
||||
"withReactionsPerspective": false
|
||||
}
|
||||
if after.len > 0:
|
||||
variables["cursor"] = % after
|
||||
let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
|
||||
result = parseGraphSearch(await fetch(url, Api.search), after)
|
||||
result.query = query
|
||||
|
||||
proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} =
|
||||
if query.text.len == 0:
|
||||
return Result[User](query: query, beginning: true)
|
||||
|
||||
var url = userSearch ? {
|
||||
"q": query.text,
|
||||
"skip_status": "1",
|
||||
"count": "20",
|
||||
"page": page
|
||||
}
|
||||
|
||||
result = parseUsers(await fetchRaw(url, Api.userSearch))
|
||||
result.query = query
|
||||
if page.len == 0:
|
||||
result.bottom = "2"
|
||||
elif page.allCharsInSet(Digits):
|
||||
result.bottom = $(parseInt(page) + 1)
|
||||
|
||||
proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
|
||||
if name.len == 0: return
|
||||
let
|
||||
ps = genParams({"screen_name": name, "trim_user": "true"},
|
||||
count="18", ext=false)
|
||||
url = photoRail ? ps
|
||||
result = parsePhotoRail(await fetch(url, Api.timeline))
|
||||
|
||||
proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
|
||||
let client = newAsyncHttpClient(maxRedirects=0)
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
|
|||
result &= p
|
||||
if ext:
|
||||
result &= ("ext", "mediaStats")
|
||||
result &= ("include_ext_alt_text", "true")
|
||||
result &= ("include_ext_media_availability", "true")
|
||||
result &= ("include_ext_alt_text", "1")
|
||||
result &= ("include_ext_media_availability", "1")
|
||||
if count.len > 0:
|
||||
result &= ("count", count)
|
||||
if cursor.len > 0:
|
||||
|
|
@ -44,7 +44,7 @@ proc genHeaders*(token: Token = nil): HttpHeaders =
|
|||
})
|
||||
|
||||
template updateToken() =
|
||||
if api != Api.search and resp.headers.hasKey(rlRemaining):
|
||||
if resp.headers.hasKey(rlRemaining):
|
||||
let
|
||||
remaining = parseInt(resp.headers[rlRemaining])
|
||||
reset = parseInt(resp.headers[rlReset])
|
||||
|
|
@ -67,14 +67,9 @@ template fetchImpl(result, fetchBody) {.dirty.} =
|
|||
|
||||
getContent()
|
||||
|
||||
# Twitter randomly returns 401 errors with an empty body quite often.
|
||||
# Retrying the request usually works.
|
||||
if resp.status == "401 Unauthorized" and result.len == 0:
|
||||
getContent()
|
||||
|
||||
if resp.status == $Http503:
|
||||
badClient = true
|
||||
raise newException(InternalError, result)
|
||||
raise newException(BadClientError, "Bad client")
|
||||
|
||||
if result.len > 0:
|
||||
if resp.headers.getOrDefault("content-encoding") == "gzip":
|
||||
|
|
@ -90,6 +85,9 @@ template fetchImpl(result, fetchBody) {.dirty.} =
|
|||
raise newException(InternalError, $url)
|
||||
except InternalError as e:
|
||||
raise e
|
||||
except BadClientError as e:
|
||||
release(token, used=true)
|
||||
raise e
|
||||
except Exception as e:
|
||||
echo "error: ", e.name, ", msg: ", e.msg, ", token: ", token[], ", url: ", url
|
||||
if "length" notin e.msg and "descriptor" notin e.msg:
|
||||
|
|
|
|||
129
src/consts.nim
129
src/consts.nim
|
|
@ -1,30 +1,28 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
import uri, sequtils
|
||||
import uri, sequtils, strutils
|
||||
|
||||
const
|
||||
auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
|
||||
auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
|
||||
|
||||
api = parseUri("https://api.twitter.com")
|
||||
activate* = $(api / "1.1/guest/activate.json")
|
||||
|
||||
userShow* = api / "1.1/users/show.json"
|
||||
photoRail* = api / "1.1/statuses/media_timeline.json"
|
||||
status* = api / "1.1/statuses/show"
|
||||
search* = api / "2/search/adaptive.json"
|
||||
|
||||
timelineApi = api / "2/timeline"
|
||||
timeline* = timelineApi / "profile"
|
||||
mediaTimeline* = timelineApi / "media"
|
||||
listTimeline* = timelineApi / "list.json"
|
||||
tweet* = timelineApi / "conversation"
|
||||
userSearch* = api / "1.1/users/search.json"
|
||||
|
||||
graphql = api / "graphql"
|
||||
graphTweet* = graphql / "6lWNh96EXDJCXl05SAtn_g/TweetDetail"
|
||||
graphUser* = graphql / "7mjxD3-C6BxitPMVQ6w0-Q/UserByScreenName"
|
||||
graphUserById* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
|
||||
graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
|
||||
graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
|
||||
graphListMembers* = graphql / "Ke6urWMeCV2UlKXGRy4sow/ListMembers"
|
||||
graphUser* = graphql / "8mPfHBetXOg-EHAyeVxUoA/UserByScreenName"
|
||||
graphUserById* = graphql / "nI8WydSd-X-lQIVo6bdktQ/UserByRestId"
|
||||
graphUserTweets* = graphql / "9rys0A7w1EyqVd2ME0QCJg/UserTweets"
|
||||
graphUserTweetsAndReplies* = graphql / "ehMCHF3Mkgjsfz_aImqOsg/UserTweetsAndReplies"
|
||||
graphUserMedia* = graphql / "MA_EP2a21zpzNWKRkaPBMg/UserMedia"
|
||||
graphTweet* = graphql / "6I7Hm635Q6ftv69L8VrSeQ/TweetDetail"
|
||||
graphTweetResult* = graphql / "rt-rHeSJ-2H9O9gxWQcPcg/TweetResultByRestId"
|
||||
graphSearchTimeline* = graphql / "gkjsKepM6gl_HmFWoWKfgg/SearchTimeline"
|
||||
graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId"
|
||||
graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug"
|
||||
graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers"
|
||||
graphListTweets* = graphql / "jZntL0oVJSdjhmPcdbw_eA/ListLatestTweetsTimeline"
|
||||
|
||||
timelineParams* = {
|
||||
"include_profile_interstitial_type": "0",
|
||||
|
|
@ -35,58 +33,85 @@ const
|
|||
"include_mute_edge": "0",
|
||||
"include_can_dm": "0",
|
||||
"include_can_media_tag": "1",
|
||||
"include_ext_is_blue_verified": "1",
|
||||
"skip_status": "1",
|
||||
"cards_platform": "Web-12",
|
||||
"include_cards": "1",
|
||||
"include_composer_source": "false",
|
||||
"include_composer_source": "0",
|
||||
"include_reply_count": "1",
|
||||
"tweet_mode": "extended",
|
||||
"include_entities": "true",
|
||||
"include_user_entities": "true",
|
||||
"include_ext_media_color": "false",
|
||||
"send_error_codes": "true",
|
||||
"simple_quoted_tweet": "true",
|
||||
"include_quote_count": "true"
|
||||
"include_entities": "1",
|
||||
"include_user_entities": "1",
|
||||
"include_ext_media_color": "0",
|
||||
"send_error_codes": "1",
|
||||
"simple_quoted_tweet": "1",
|
||||
"include_quote_count": "1"
|
||||
}.toSeq
|
||||
|
||||
searchParams* = {
|
||||
"query_source": "typed_query",
|
||||
"pc": "1",
|
||||
"spelling_corrections": "1"
|
||||
}.toSeq
|
||||
## top: nothing
|
||||
## latest: "tweet_search_mode: live"
|
||||
## user: "result_filter: user"
|
||||
## photos: "result_filter: photos"
|
||||
## videos: "result_filter: videos"
|
||||
gqlFeatures* = """{
|
||||
"blue_business_profile_image_shape_enabled": false,
|
||||
"freedom_of_speech_not_reach_fetch_enabled": false,
|
||||
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
|
||||
"interactive_text_enabled": false,
|
||||
"longform_notetweets_consumption_enabled": true,
|
||||
"longform_notetweets_richtext_consumption_enabled": true,
|
||||
"longform_notetweets_rich_text_read_enabled": false,
|
||||
"responsive_web_edit_tweet_api_enabled": false,
|
||||
"responsive_web_enhance_cards_enabled": false,
|
||||
"responsive_web_graphql_exclude_directive_enabled": true,
|
||||
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
|
||||
"responsive_web_graphql_timeline_navigation_enabled": false,
|
||||
"responsive_web_text_conversations_enabled": false,
|
||||
"responsive_web_twitter_blue_verified_badge_is_enabled": true,
|
||||
"spaces_2022_h2_clipping": true,
|
||||
"spaces_2022_h2_spaces_communities": true,
|
||||
"standardized_nudges_misinfo": false,
|
||||
"tweet_awards_web_tipping_enabled": false,
|
||||
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
|
||||
"tweetypie_unmention_optimization_enabled": false,
|
||||
"verified_phone_label_enabled": false,
|
||||
"vibe_api_enabled": false,
|
||||
"view_counts_everywhere_api_enabled": false
|
||||
}""".replace(" ", "").replace("\n", "")
|
||||
|
||||
tweetVariables* = """{
|
||||
"focalTweetId": "$1",
|
||||
$2
|
||||
"includePromotedContent": false,
|
||||
"withBirdwatchNotes": false,
|
||||
"includePromotedContent": false,
|
||||
"withDownvotePerspective": false,
|
||||
"withReactionsMetadata": false,
|
||||
"withReactionsPerspective": false,
|
||||
"withVoice": false
|
||||
}"""
|
||||
|
||||
tweetResultVariables* = """{
|
||||
"tweetId": "$1",
|
||||
"includePromotedContent": false,
|
||||
"withDownvotePerspective": false,
|
||||
"withReactionsMetadata": false,
|
||||
"withReactionsPerspective": false,
|
||||
"withVoice": false,
|
||||
"withCommunity": false
|
||||
}"""
|
||||
|
||||
userTweetsVariables* = """{
|
||||
"userId": "$1", $2
|
||||
"count": 20,
|
||||
"includePromotedContent": false,
|
||||
"withDownvotePerspective": false,
|
||||
"withReactionsMetadata": false,
|
||||
"withReactionsPerspective": false,
|
||||
"withSuperFollowsTweetFields": false,
|
||||
"withSuperFollowsUserFields": false,
|
||||
"withVoice": false,
|
||||
"withV2Timeline": true
|
||||
}"""
|
||||
|
||||
tweetFeatures* = """{
|
||||
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
|
||||
"responsive_web_graphql_timeline_navigation_enabled": false,
|
||||
"standardized_nudges_misinfo": false,
|
||||
"verified_phone_label_enabled": false,
|
||||
"responsive_web_twitter_blue_verified_badge_is_enabled": false,
|
||||
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
|
||||
"view_counts_everywhere_api_enabled": false,
|
||||
"responsive_web_edit_tweet_api_enabled": false,
|
||||
"tweetypie_unmention_optimization_enabled": false,
|
||||
"vibe_api_enabled": false,
|
||||
"longform_notetweets_consumption_enabled": true,
|
||||
"responsive_web_text_conversations_enabled": false,
|
||||
"responsive_web_enhance_cards_enabled": false,
|
||||
"interactive_text_enabled": false
|
||||
listTweetsVariables* = """{
|
||||
"listId": "$1", $2
|
||||
"count": 20,
|
||||
"includePromotedContent": false,
|
||||
"withDownvotePerspective": false,
|
||||
"withReactionsMetadata": false,
|
||||
"withReactionsPerspective": false,
|
||||
"withVoice": false
|
||||
}"""
|
||||
|
|
|
|||
|
|
@ -1,2 +1,2 @@
|
|||
import parser/[user, graphql, timeline]
|
||||
export user, graphql, timeline
|
||||
import parser/[user, graphql]
|
||||
export user, graphql
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ proc parseGraphUser*(json: string): User =
|
|||
|
||||
result = toUser raw.data.user.result.legacy
|
||||
result.id = raw.data.user.result.restId
|
||||
result.verified = result.verified or raw.data.user.result.isBlueVerified
|
||||
|
||||
proc parseGraphListMembers*(json, cursor: string): Result[User] =
|
||||
result = Result[User](
|
||||
|
|
|
|||
|
|
@ -1,30 +0,0 @@
|
|||
import std/[strutils, tables]
|
||||
import jsony
|
||||
import user, ../types/timeline
|
||||
from ../../types import Result, User
|
||||
|
||||
proc getId(id: string): string {.inline.} =
|
||||
let start = id.rfind("-")
|
||||
if start < 0: return id
|
||||
id[start + 1 ..< id.len]
|
||||
|
||||
proc parseUsers*(json: string; after=""): Result[User] =
|
||||
result = Result[User](beginning: after.len == 0)
|
||||
|
||||
let raw = json.fromJson(Search)
|
||||
if raw.timeline.instructions.len == 0:
|
||||
return
|
||||
|
||||
for i in raw.timeline.instructions:
|
||||
if i.addEntries.entries.len > 0:
|
||||
for e in i.addEntries.entries:
|
||||
let id = e.entryId.getId
|
||||
if e.entryId.startsWith("user"):
|
||||
if id in raw.globalObjects.users:
|
||||
result.content.add toUser raw.globalObjects.users[id]
|
||||
elif e.entryId.startsWith("cursor"):
|
||||
let cursor = e.content.operation.cursor
|
||||
if cursor.cursorType == "Top":
|
||||
result.top = cursor.value
|
||||
elif cursor.cursorType == "Bottom":
|
||||
result.bottom = cursor.value
|
||||
|
|
@ -84,6 +84,8 @@ proc parseUnifiedCard*(json: string): Card =
|
|||
component.parseMedia(card, result)
|
||||
of buttonGroup:
|
||||
discard
|
||||
of ComponentType.hidden:
|
||||
result.kind = CardKind.hidden
|
||||
of ComponentType.unknown:
|
||||
echo "ERROR: Unknown component type: ", json
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import std/[algorithm, unicode, re, strutils, strformat, options, nre]
|
|||
import jsony
|
||||
import utils, slices
|
||||
import ../types/user as userType
|
||||
from ../../types import User, Error
|
||||
from ../../types import Result, User, Error
|
||||
|
||||
let
|
||||
unRegex = re.re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
|
||||
|
|
@ -76,3 +76,12 @@ proc parseUser*(json: string; username=""): User =
|
|||
else: echo "[error - parseUser]: ", error
|
||||
|
||||
result = toUser json.fromJson(RawUser)
|
||||
|
||||
proc parseUsers*(json: string; after=""): Result[User] =
|
||||
result = Result[User](beginning: after.len == 0)
|
||||
|
||||
# starting with '{' means it's an error
|
||||
if json[0] == '[':
|
||||
let raw = json.fromJson(seq[RawUser])
|
||||
for user in raw:
|
||||
result.content.add user.toUser
|
||||
|
|
|
|||
|
|
@ -11,4 +11,5 @@ type
|
|||
UserResult = object
|
||||
legacy*: RawUser
|
||||
restId*: string
|
||||
isBlueVerified*: bool
|
||||
reason*: Option[string]
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ type
|
|||
twitterListDetails
|
||||
communityDetails
|
||||
mediaWithDetailsHorizontal
|
||||
hidden
|
||||
unknown
|
||||
|
||||
Component* = object
|
||||
|
|
@ -71,11 +72,11 @@ type
|
|||
Text = object
|
||||
content: string
|
||||
|
||||
HasTypeField = Component | Destination | MediaEntity | AppStoreData
|
||||
TypeField = Component | Destination | MediaEntity | AppStoreData
|
||||
|
||||
converter fromText*(text: Text): string = text.content
|
||||
|
||||
proc renameHook*(v: var HasTypeField; fieldName: var string) =
|
||||
proc renameHook*(v: var TypeField; fieldName: var string) =
|
||||
if fieldName == "type":
|
||||
fieldName = "kind"
|
||||
|
||||
|
|
@ -89,6 +90,7 @@ proc enumHook*(s: string; v: var ComponentType) =
|
|||
of "twitter_list_details": twitterListDetails
|
||||
of "community_details": communityDetails
|
||||
of "media_with_details_horizontal": mediaWithDetailsHorizontal
|
||||
of "commerce_drop_details": hidden
|
||||
else: echo "ERROR: Unknown enum value (ComponentType): ", s; unknown
|
||||
|
||||
proc enumHook*(s: string; v: var AppType) =
|
||||
|
|
|
|||
|
|
@ -42,5 +42,11 @@ template use*(pool: HttpPool; heads: HttpHeaders; body: untyped): untyped =
|
|||
except ProtocolError:
|
||||
# Twitter closed the connection, retry
|
||||
body
|
||||
except BadClientError:
|
||||
# Twitter returned 503, we need a new client
|
||||
pool.release(c, true)
|
||||
badClient = false
|
||||
c = pool.acquire(heads)
|
||||
body
|
||||
finally:
|
||||
pool.release(c, badClient)
|
||||
|
|
|
|||
|
|
@ -85,19 +85,23 @@ routes:
|
|||
resp Http500, showError(
|
||||
&"An error occurred, please {link} with the URL you tried to visit.", cfg)
|
||||
|
||||
error BadClientError:
|
||||
echo error.exc.name, ": ", error.exc.msg
|
||||
resp Http500, showError("Network error occured, please try again.", cfg)
|
||||
|
||||
error RateLimitError:
|
||||
const link = a("another instance", href = instancesUrl)
|
||||
resp Http429, showError(
|
||||
&"Instance has been rate limited.<br>Use {link} or try again later.", cfg)
|
||||
|
||||
extend unsupported, ""
|
||||
extend preferences, ""
|
||||
extend resolver, ""
|
||||
extend rss, ""
|
||||
extend status, ""
|
||||
extend search, ""
|
||||
extend timeline, ""
|
||||
extend list, ""
|
||||
extend status, ""
|
||||
extend media, ""
|
||||
extend list, ""
|
||||
extend preferences, ""
|
||||
extend resolver, ""
|
||||
extend embed, ""
|
||||
extend debug, ""
|
||||
extend unsupported, ""
|
||||
|
|
|
|||
126
src/parser.nim
126
src/parser.nim
|
|
@ -4,6 +4,8 @@ import packedjson, packedjson/deserialiser
|
|||
import types, parserutils, utils
|
||||
import experimental/parser/unifiedcard
|
||||
|
||||
proc parseGraphTweet(js: JsonNode): Tweet
|
||||
|
||||
proc parseUser(js: JsonNode; id=""): User =
|
||||
if js.isNull: return
|
||||
result = User(
|
||||
|
|
@ -19,13 +21,20 @@ proc parseUser(js: JsonNode; id=""): User =
|
|||
tweets: js{"statuses_count"}.getInt,
|
||||
likes: js{"favourites_count"}.getInt,
|
||||
media: js{"media_count"}.getInt,
|
||||
verified: js{"verified"}.getBool,
|
||||
verified: js{"verified"}.getBool or js{"ext_is_blue_verified"}.getBool,
|
||||
protected: js{"protected"}.getBool,
|
||||
joinDate: js{"created_at"}.getTime
|
||||
)
|
||||
|
||||
result.expandUserEntities(js)
|
||||
|
||||
proc parseGraphUser(js: JsonNode): User =
|
||||
let user = ? js{"user_results", "result"}
|
||||
result = parseUser(user{"legacy"})
|
||||
|
||||
if "is_blue_verified" in user:
|
||||
result.verified = true
|
||||
|
||||
proc parseGraphList*(js: JsonNode): List =
|
||||
if js.isNull: return
|
||||
|
||||
|
|
@ -38,11 +47,11 @@ proc parseGraphList*(js: JsonNode): List =
|
|||
result = List(
|
||||
id: list{"id_str"}.getStr,
|
||||
name: list{"name"}.getStr,
|
||||
username: list{"user", "legacy", "screen_name"}.getStr,
|
||||
userId: list{"user", "rest_id"}.getStr,
|
||||
username: list{"user_results", "result", "legacy", "screen_name"}.getStr,
|
||||
userId: list{"user_results", "result", "rest_id"}.getStr,
|
||||
description: list{"description"}.getStr,
|
||||
members: list{"member_count"}.getInt,
|
||||
banner: list{"custom_banner_media", "media_info", "url"}.getImageStr
|
||||
banner: list{"custom_banner_media", "media_info", "original_img_url"}.getImageStr
|
||||
)
|
||||
|
||||
proc parsePoll(js: JsonNode): Poll =
|
||||
|
|
@ -213,10 +222,18 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
|
|||
if js{"is_quote_status"}.getBool:
|
||||
result.quote = some Tweet(id: js{"quoted_status_id_str"}.getId)
|
||||
|
||||
# legacy
|
||||
with rt, js{"retweeted_status_id_str"}:
|
||||
result.retweet = some Tweet(id: rt.getId)
|
||||
return
|
||||
|
||||
# graphql
|
||||
with rt, js{"retweeted_status_result", "result"}:
|
||||
# needed due to weird edgecase where the actual tweet data isn't included
|
||||
if "legacy" in rt:
|
||||
result.retweet = some parseGraphTweet(rt)
|
||||
return
|
||||
|
||||
if jsCard.kind != JNull:
|
||||
let name = jsCard{"name"}.getStr
|
||||
if "poll" in name:
|
||||
|
|
@ -237,7 +254,10 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
|
|||
of "video":
|
||||
result.video = some(parseVideo(m))
|
||||
with user, m{"additional_media_info", "source_user"}:
|
||||
if user{"id"}.getInt > 0:
|
||||
result.attribution = some(parseUser(user))
|
||||
else:
|
||||
result.attribution = some(parseGraphUser(user))
|
||||
of "animated_gif":
|
||||
result.gif = some(parseGif(m))
|
||||
else: discard
|
||||
|
|
@ -299,19 +319,6 @@ proc parseGlobalObjects(js: JsonNode): GlobalObjects =
|
|||
tweet.user = result.users[tweet.user.id]
|
||||
result.tweets[k] = tweet
|
||||
|
||||
proc parseStatus*(js: JsonNode): Tweet =
|
||||
with e, js{"errors"}:
|
||||
if e.getError in {tweetNotFound, tweetUnavailable, tweetCensored, doesntExist,
|
||||
tweetNotAuthorized, suspended}:
|
||||
return
|
||||
|
||||
result = parseTweet(js, js{"card"})
|
||||
if not result.isNil:
|
||||
result.user = parseUser(js{"user"})
|
||||
|
||||
with quote, js{"quoted_status"}:
|
||||
result.quote = some parseStatus(js{"quoted_status"})
|
||||
|
||||
proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNode) =
|
||||
if js.kind != JArray or js.len == 0:
|
||||
return
|
||||
|
|
@ -352,7 +359,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
|
|||
result.top = e.getCursor
|
||||
elif "cursor-bottom" in entry:
|
||||
result.bottom = e.getCursor
|
||||
elif entry.startsWith("sq-C"):
|
||||
elif entry.startsWith("sq-cursor"):
|
||||
with cursor, e{"content", "operation", "cursor"}:
|
||||
if cursor{"cursorType"}.getStr == "Bottom":
|
||||
result.bottom = cursor{"value"}.getStr
|
||||
|
|
@ -373,9 +380,20 @@ proc parsePhotoRail*(js: JsonNode): PhotoRail =
|
|||
result.add GalleryPhoto(url: url, tweetId: $t.id)
|
||||
|
||||
proc parseGraphTweet(js: JsonNode): Tweet =
|
||||
if js.kind == JNull or js{"__typename"}.getStr == "TweetUnavailable":
|
||||
if js.kind == JNull:
|
||||
return Tweet(available: false)
|
||||
|
||||
case js{"__typename"}.getStr
|
||||
of "TweetUnavailable":
|
||||
return Tweet(available: false)
|
||||
of "TweetTombstone":
|
||||
return Tweet(
|
||||
available: false,
|
||||
text: js{"tombstone", "text"}.getTombstone
|
||||
)
|
||||
of "TweetWithVisibilityResults":
|
||||
return parseGraphTweet(js{"tweet"})
|
||||
|
||||
var jsCard = copy(js{"card", "legacy"})
|
||||
if jsCard.kind != JNull:
|
||||
var values = newJObject()
|
||||
|
|
@ -384,7 +402,7 @@ proc parseGraphTweet(js: JsonNode): Tweet =
|
|||
jsCard["binding_values"] = values
|
||||
|
||||
result = parseTweet(js{"legacy"}, jsCard)
|
||||
result.user = parseUser(js{"core", "user_results", "result", "legacy"})
|
||||
result.user = parseGraphUser(js{"core"})
|
||||
|
||||
with noteTweet, js{"note_tweet", "note_tweet_results", "result"}:
|
||||
result.expandNoteTweetEntities(noteTweet)
|
||||
|
|
@ -407,10 +425,14 @@ proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
|
|||
if t{"item", "itemContent", "tweetDisplayType"}.getStr == "SelfThread":
|
||||
result.self = true
|
||||
|
||||
proc parseGraphTweetResult*(js: JsonNode): Tweet =
|
||||
with tweet, js{"data", "tweetResult", "result"}:
|
||||
result = parseGraphTweet(tweet)
|
||||
|
||||
proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
|
||||
result = Conversation(replies: Result[Chain](beginning: true))
|
||||
|
||||
let instructions = ? js{"data", "threaded_conversation_with_injections_v2", "instructions"}
|
||||
let instructions = ? js{"data", "threaded_conversation_with_injections", "instructions"}
|
||||
if instructions.len == 0:
|
||||
return
|
||||
|
||||
|
|
@ -418,7 +440,8 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
|
|||
let entryId = e{"entryId"}.getStr
|
||||
# echo entryId
|
||||
if entryId.startsWith("tweet"):
|
||||
let tweet = parseGraphTweet(e{"content", "itemContent", "tweet_results", "result"})
|
||||
with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
|
||||
let tweet = parseGraphTweet(tweetResult)
|
||||
|
||||
if not tweet.available:
|
||||
tweet.id = parseBiggestInt(entryId.getId())
|
||||
|
|
@ -427,6 +450,18 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
|
|||
result.tweet = tweet
|
||||
else:
|
||||
result.before.content.add tweet
|
||||
elif entryId.startsWith("tombstone"):
|
||||
let id = entryId.getId()
|
||||
let tweet = Tweet(
|
||||
id: parseBiggestInt(id),
|
||||
available: false,
|
||||
text: e{"content", "itemContent", "tombstoneInfo", "richText"}.getTombstone
|
||||
)
|
||||
|
||||
if id == tweetId:
|
||||
result.tweet = tweet
|
||||
else:
|
||||
result.before.content.add tweet
|
||||
elif entryId.startsWith("conversationthread"):
|
||||
let (thread, self) = parseGraphThread(e)
|
||||
if self:
|
||||
|
|
@ -435,3 +470,50 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
|
|||
result.replies.content.add thread
|
||||
elif entryId.startsWith("cursor-bottom"):
|
||||
result.replies.bottom = e{"content", "itemContent", "value"}.getStr
|
||||
|
||||
proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Timeline =
|
||||
result = Timeline(beginning: after.len == 0)
|
||||
|
||||
let instructions =
|
||||
if root == "list": ? js{"data", "list", "tweets_timeline", "timeline", "instructions"}
|
||||
else: ? js{"data", "user", "result", "timeline_v2", "timeline", "instructions"}
|
||||
|
||||
if instructions.len == 0:
|
||||
return
|
||||
|
||||
for i in instructions:
|
||||
if i{"type"}.getStr == "TimelineAddEntries":
|
||||
for e in i{"entries"}:
|
||||
let entryId = e{"entryId"}.getStr
|
||||
if entryId.startsWith("tweet"):
|
||||
with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
|
||||
let tweet = parseGraphTweet(tweetResult)
|
||||
if not tweet.available:
|
||||
tweet.id = parseBiggestInt(entryId.getId())
|
||||
result.content.add tweet
|
||||
elif entryId.startsWith("cursor-bottom"):
|
||||
result.bottom = e{"content", "value"}.getStr
|
||||
|
||||
proc parseGraphSearch*(js: JsonNode; after=""): Timeline =
|
||||
result = Timeline(beginning: after.len == 0)
|
||||
|
||||
let instructions = js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"}
|
||||
if instructions.len == 0:
|
||||
return
|
||||
|
||||
for instruction in instructions:
|
||||
let typ = instruction{"type"}.getStr
|
||||
if typ == "TimelineAddEntries":
|
||||
for e in instructions[0]{"entries"}:
|
||||
let entryId = e{"entryId"}.getStr
|
||||
if entryId.startsWith("tweet"):
|
||||
with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
|
||||
let tweet = parseGraphTweet(tweetResult)
|
||||
if not tweet.available:
|
||||
tweet.id = parseBiggestInt(entryId.getId())
|
||||
result.content.add tweet
|
||||
elif entryId.startsWith("cursor-bottom"):
|
||||
result.bottom = e{"content", "value"}.getStr
|
||||
elif typ == "TimelineReplaceEntry":
|
||||
if instruction{"entry_id_to_replace"}.getStr.startsWith("cursor-bottom"):
|
||||
result.bottom = instruction{"entry", "content", "value"}.getStr
|
||||
|
|
|
|||
|
|
@ -130,7 +130,7 @@ proc getBanner*(js: JsonNode): string =
|
|||
return
|
||||
|
||||
proc getTombstone*(js: JsonNode): string =
|
||||
result = js{"tombstoneInfo", "richText", "text"}.getStr
|
||||
result = js{"text"}.getStr
|
||||
result.removeSuffix(" Learn more")
|
||||
|
||||
proc getMp4Resolution*(url: string): int =
|
||||
|
|
|
|||
|
|
@ -153,7 +153,7 @@ proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
|
|||
if tweet != redisNil:
|
||||
tweet.deserialize(Tweet)
|
||||
else:
|
||||
result = await getStatus($id)
|
||||
result = await getGraphTweetResult($id)
|
||||
if not result.isNil:
|
||||
await cache(result)
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ import jester
|
|||
import router_utils
|
||||
import ".."/[types, redis_cache, api]
|
||||
import ../views/[general, timeline, list]
|
||||
export getListTimeline, getGraphList
|
||||
|
||||
template respList*(list, timeline, title, vnode: typed) =
|
||||
if list.id.len == 0 or list.name.len == 0:
|
||||
|
|
@ -39,7 +38,7 @@ proc createListRouter*(cfg: Config) =
|
|||
let
|
||||
prefs = cookiePrefs()
|
||||
list = await getCachedList(id=(@"id"))
|
||||
timeline = await getListTimeline(list.id, getCursor())
|
||||
timeline = await getGraphListTweets(list.id, getCursor())
|
||||
vnode = renderTimelineTweets(timeline, prefs, request.path)
|
||||
respList(list, timeline, list.title, vnode)
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
|
|||
var q = query
|
||||
q.fromUser = names
|
||||
profile = Profile(
|
||||
tweets: await getSearch[Tweet](q, after),
|
||||
tweets: await getGraphSearch(q, after),
|
||||
# this is kinda dumb
|
||||
user: User(
|
||||
username: name,
|
||||
|
|
@ -78,7 +78,7 @@ proc createRssRouter*(cfg: Config) =
|
|||
if rss.cursor.len > 0:
|
||||
respRss(rss, "Search")
|
||||
|
||||
let tweets = await getSearch[Tweet](query, cursor)
|
||||
let tweets = await getGraphSearch(query, cursor)
|
||||
rss.cursor = tweets.bottom
|
||||
rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg)
|
||||
|
||||
|
|
@ -159,7 +159,7 @@ proc createRssRouter*(cfg: Config) =
|
|||
|
||||
let
|
||||
list = await getCachedList(id=id)
|
||||
timeline = await getListTimeline(list.id, cursor)
|
||||
timeline = await getGraphListTweets(list.id, cursor)
|
||||
rss.cursor = timeline.bottom
|
||||
rss.feed = renderListRss(timeline.content, list, cfg)
|
||||
|
||||
|
|
|
|||
|
|
@ -27,11 +27,15 @@ proc createSearchRouter*(cfg: Config) =
|
|||
of users:
|
||||
if "," in q:
|
||||
redirect("/" & q)
|
||||
let users = await getSearch[User](query, getCursor())
|
||||
var users: Result[User]
|
||||
try:
|
||||
users = await getUserSearch(query, getCursor())
|
||||
except InternalError:
|
||||
users = Result[User](beginning: true, query: query)
|
||||
resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title)
|
||||
of tweets:
|
||||
let
|
||||
tweets = await getSearch[Tweet](query, getCursor())
|
||||
tweets = await getGraphSearch(query, getCursor())
|
||||
rss = "/search/rss?" & genQueryUrl(query)
|
||||
resp renderMain(renderTweetSearch(tweets, prefs, getPath()),
|
||||
request, cfg, prefs, title, rss=rss)
|
||||
|
|
|
|||
|
|
@ -16,17 +16,21 @@ proc createStatusRouter*(cfg: Config) =
|
|||
router status:
|
||||
get "/@name/status/@id/?":
|
||||
cond '.' notin @"name"
|
||||
cond not @"id".any(c => not c.isDigit)
|
||||
let id = @"id"
|
||||
|
||||
if id.len > 19 or id.any(c => not c.isDigit):
|
||||
resp Http404, showError("Invalid tweet ID", cfg)
|
||||
|
||||
let prefs = cookiePrefs()
|
||||
|
||||
# used for the infinite scroll feature
|
||||
if @"scroll".len > 0:
|
||||
let replies = await getReplies(@"id", getCursor())
|
||||
let replies = await getReplies(id, getCursor())
|
||||
if replies.content.len == 0:
|
||||
resp Http404, ""
|
||||
resp $renderReplies(replies, prefs, getPath())
|
||||
|
||||
let conv = await getTweet(@"id", getCursor())
|
||||
let conv = await getTweet(id, getCursor())
|
||||
if conv == nil:
|
||||
echo "nil conv"
|
||||
|
||||
|
|
|
|||
|
|
@ -47,10 +47,10 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
|
|||
let
|
||||
timeline =
|
||||
case query.kind
|
||||
of posts: getTimeline(userId, after)
|
||||
of replies: getTimeline(userId, after, replies=true)
|
||||
of media: getMediaTimeline(userId, after)
|
||||
else: getSearch[Tweet](query, after)
|
||||
of posts: getGraphUserTweets(userId, TimelineKind.tweets, after)
|
||||
of replies: getGraphUserTweets(userId, TimelineKind.replies, after)
|
||||
of media: getGraphUserTweets(userId, TimelineKind.media, after)
|
||||
else: getGraphSearch(query, after)
|
||||
|
||||
rail =
|
||||
skipIf(skipRail or query.kind == media, @[]):
|
||||
|
|
@ -64,6 +64,7 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
|
|||
let tweet = await getCachedTweet(user.pinnedTweet)
|
||||
if not tweet.isNil:
|
||||
tweet.pinned = true
|
||||
tweet.user = user
|
||||
pinned = some tweet
|
||||
|
||||
result = Profile(
|
||||
|
|
@ -82,7 +83,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
|
|||
rss, after: string): Future[string] {.async.} =
|
||||
if query.fromUser.len != 1:
|
||||
let
|
||||
timeline = await getSearch[Tweet](query, after)
|
||||
timeline = await getGraphSearch(query, after)
|
||||
html = renderTweetSearch(timeline, prefs, getPath())
|
||||
return renderMain(html, request, cfg, prefs, "Multi", rss=rss)
|
||||
|
||||
|
|
@ -123,7 +124,7 @@ proc createTimelineRouter*(cfg: Config) =
|
|||
|
||||
get "/@name/?@tab?/?":
|
||||
cond '.' notin @"name"
|
||||
cond @"name" notin ["pic", "gif", "video"]
|
||||
cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"]
|
||||
cond @"tab" in ["with_replies", "media", "search", ""]
|
||||
let
|
||||
prefs = cookiePrefs()
|
||||
|
|
@ -137,7 +138,7 @@ proc createTimelineRouter*(cfg: Config) =
|
|||
# used for the infinite scroll feature
|
||||
if @"scroll".len > 0:
|
||||
if query.fromUser.len != 1:
|
||||
var timeline = await getSearch[Tweet](query, after)
|
||||
var timeline = await getGraphSearch(query, after)
|
||||
if timeline.content.len == 0: resp Http404
|
||||
timeline.beginning = true
|
||||
resp $renderTweetSearch(timeline, prefs, getPath())
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
import asyncdispatch, httpclient, times, sequtils, json, random
|
||||
import strutils, tables
|
||||
import zippy
|
||||
import types, consts, http_pool
|
||||
import types, consts
|
||||
|
||||
const
|
||||
maxConcurrentReqs = 5 # max requests at a time per token, to avoid race conditions
|
||||
|
|
@ -11,11 +10,12 @@ const
|
|||
failDelay = initDuration(minutes=30)
|
||||
|
||||
var
|
||||
clientPool: HttpPool
|
||||
tokenPool: seq[Token]
|
||||
lastFailed: Time
|
||||
enableLogging = false
|
||||
|
||||
let headers = newHttpHeaders({"authorization": auth})
|
||||
|
||||
template log(str) =
|
||||
if enableLogging: echo "[tokens] ", str
|
||||
|
||||
|
|
@ -41,10 +41,12 @@ proc getPoolJson*(): JsonNode =
|
|||
let
|
||||
maxReqs =
|
||||
case api
|
||||
of Api.listMembers, Api.listBySlug, Api.list,
|
||||
Api.userRestId, Api.userScreenName, Api.tweetDetail: 500
|
||||
of Api.timeline: 187
|
||||
else: 180
|
||||
of Api.listMembers, Api.listBySlug, Api.list, Api.listTweets,
|
||||
Api.userTweets, Api.userTweetsAndReplies, Api.userMedia,
|
||||
Api.userRestId, Api.userScreenName,
|
||||
Api.tweetDetail, Api.tweetResult, Api.search: 500
|
||||
of Api.userSearch: 900
|
||||
reqs = maxReqs - token.apis[api].remaining
|
||||
|
||||
reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs
|
||||
|
|
@ -65,18 +67,12 @@ proc fetchToken(): Future[Token] {.async.} =
|
|||
if getTime() - lastFailed < failDelay:
|
||||
raise rateLimitError()
|
||||
|
||||
let headers = newHttpHeaders({
|
||||
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
||||
"accept-encoding": "gzip",
|
||||
"accept-language": "en-US,en;q=0.5",
|
||||
"connection": "keep-alive",
|
||||
"authorization": auth
|
||||
})
|
||||
let client = newAsyncHttpClient(headers=headers)
|
||||
|
||||
try:
|
||||
let
|
||||
resp = clientPool.use(headers): await c.postContent(activate)
|
||||
tokNode = parseJson(uncompress(resp))["guest_token"]
|
||||
resp = await client.postContent(activate)
|
||||
tokNode = parseJson(resp)["guest_token"]
|
||||
tok = tokNode.getStr($(tokNode.getInt))
|
||||
time = getTime()
|
||||
|
||||
|
|
@ -86,6 +82,8 @@ proc fetchToken(): Future[Token] {.async.} =
|
|||
if "Try again" notin e.msg:
|
||||
echo "[tokens] fetching tokens paused, resuming in 30 minutes"
|
||||
lastFailed = getTime()
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
proc expired(token: Token): bool =
|
||||
let time = getTime()
|
||||
|
|
@ -158,7 +156,6 @@ proc poolTokens*(amount: int) {.async.} =
|
|||
tokenPool.add newToken
|
||||
|
||||
proc initTokenPool*(cfg: Config) {.async.} =
|
||||
clientPool = HttpPool()
|
||||
enableLogging = cfg.enableDebug
|
||||
|
||||
while true:
|
||||
|
|
|
|||
|
|
@ -7,19 +7,28 @@ genPrefsType()
|
|||
type
|
||||
RateLimitError* = object of CatchableError
|
||||
InternalError* = object of CatchableError
|
||||
BadClientError* = object of CatchableError
|
||||
|
||||
TimelineKind* {.pure.} = enum
|
||||
tweets
|
||||
replies
|
||||
media
|
||||
|
||||
Api* {.pure.} = enum
|
||||
tweetDetail
|
||||
userShow
|
||||
tweetResult
|
||||
timeline
|
||||
search
|
||||
tweet
|
||||
userSearch
|
||||
list
|
||||
listBySlug
|
||||
listMembers
|
||||
listTweets
|
||||
userRestId
|
||||
userScreenName
|
||||
status
|
||||
userTweets
|
||||
userTweetsAndReplies
|
||||
userMedia
|
||||
|
||||
RateLimit* = object
|
||||
remaining*: int
|
||||
|
|
@ -36,8 +45,10 @@ type
|
|||
null = 0
|
||||
noUserMatches = 17
|
||||
protectedUser = 22
|
||||
missingParams = 25
|
||||
couldntAuth = 32
|
||||
doesntExist = 34
|
||||
invalidParam = 47
|
||||
userNotFound = 50
|
||||
suspended = 63
|
||||
rateLimited = 88
|
||||
|
|
@ -150,6 +161,7 @@ type
|
|||
imageDirectMessage = "image_direct_message"
|
||||
audiospace = "audiospace"
|
||||
newsletterPublication = "newsletter_publication"
|
||||
hidden
|
||||
unknown
|
||||
|
||||
Card* = object
|
||||
|
|
|
|||
|
|
@ -59,8 +59,7 @@ proc buttonReferer*(action, text, path: string; class=""; `method`="post"): VNod
|
|||
proc genCheckbox*(pref, label: string; state: bool): VNode =
|
||||
buildHtml(label(class="pref-group checkbox-container")):
|
||||
text label
|
||||
if state: input(name=pref, `type`="checkbox", checked="")
|
||||
else: input(name=pref, `type`="checkbox")
|
||||
input(name=pref, `type`="checkbox", checked=state)
|
||||
span(class="checkbox")
|
||||
|
||||
proc genInput*(pref, label, state, placeholder: string; class=""; autofocus=true): VNode =
|
||||
|
|
@ -68,20 +67,15 @@ proc genInput*(pref, label, state, placeholder: string; class=""; autofocus=true
|
|||
buildHtml(tdiv(class=("pref-group pref-input " & class))):
|
||||
if label.len > 0:
|
||||
label(`for`=pref): text label
|
||||
if autofocus and state.len == 0:
|
||||
input(name=pref, `type`="text", placeholder=p, value=state, autofocus="")
|
||||
else:
|
||||
input(name=pref, `type`="text", placeholder=p, value=state)
|
||||
input(name=pref, `type`="text", placeholder=p, value=state, autofocus=(autofocus and state.len == 0))
|
||||
|
||||
proc genSelect*(pref, label, state: string; options: seq[string]): VNode =
|
||||
buildHtml(tdiv(class="pref-group pref-input")):
|
||||
label(`for`=pref): text label
|
||||
select(name=pref):
|
||||
for opt in options:
|
||||
if opt == state:
|
||||
option(value=opt, selected=""): text opt
|
||||
else:
|
||||
option(value=opt): text opt
|
||||
option(value=opt, selected=(opt == state)):
|
||||
text opt
|
||||
|
||||
proc genDate*(pref, state: string): VNode =
|
||||
buildHtml(span(class="date-input")):
|
||||
|
|
@ -93,12 +87,9 @@ proc genImg*(url: string; class=""): VNode =
|
|||
img(src=getPicUrl(url), class=class, alt="")
|
||||
|
||||
proc getTabClass*(query: Query; tab: QueryKind): string =
|
||||
result = "tab-item"
|
||||
if query.kind == tab:
|
||||
result &= " active"
|
||||
if query.kind == tab: "tab-item active"
|
||||
else: "tab-item"
|
||||
|
||||
proc getAvatarClass*(prefs: Prefs): string =
|
||||
if prefs.squareAvatars:
|
||||
"avatar"
|
||||
else:
|
||||
"avatar round"
|
||||
if prefs.squareAvatars: "avatar"
|
||||
else: "avatar round"
|
||||
|
|
|
|||
|
|
@ -63,12 +63,10 @@ proc renderSearchPanel*(query: Query): VNode =
|
|||
hiddenField("f", "tweets")
|
||||
genInput("q", "", query.text, "Enter search...", class="pref-inline")
|
||||
button(`type`="submit"): icon "search"
|
||||
if isPanelOpen(query):
|
||||
input(id="search-panel-toggle", `type`="checkbox", checked="")
|
||||
else:
|
||||
input(id="search-panel-toggle", `type`="checkbox")
|
||||
label(`for`="search-panel-toggle"):
|
||||
icon "down"
|
||||
|
||||
input(id="search-panel-toggle", `type`="checkbox", checked=isPanelOpen(query))
|
||||
label(`for`="search-panel-toggle"): icon "down"
|
||||
|
||||
tdiv(class="search-panel"):
|
||||
for f in @["filter", "exclude"]:
|
||||
span(class="search-title"): text capitalize(f)
|
||||
|
|
|
|||
|
|
@ -106,14 +106,10 @@ proc renderVideo*(video: Video; prefs: Prefs; path: string): VNode =
|
|||
else: vidUrl
|
||||
case playbackType
|
||||
of mp4:
|
||||
if prefs.muteVideos:
|
||||
video(poster=thumb, controls="", muted=""):
|
||||
source(src=source, `type`="video/mp4")
|
||||
else:
|
||||
video(poster=thumb, controls=""):
|
||||
video(poster=thumb, controls="", muted=prefs.muteVideos):
|
||||
source(src=source, `type`="video/mp4")
|
||||
of m3u8, vmap:
|
||||
video(poster=thumb, data-url=source, data-autoload="false")
|
||||
video(poster=thumb, data-url=source, data-autoload="false", muted=prefs.muteVideos)
|
||||
verbatim "<div class=\"video-overlay\" onclick=\"playVideo(this)\">"
|
||||
tdiv(class="overlay-circle"): span(class="overlay-triangle")
|
||||
verbatim "</div>"
|
||||
|
|
@ -127,14 +123,9 @@ proc renderGif(gif: Gif; prefs: Prefs): VNode =
|
|||
buildHtml(tdiv(class="attachments media-gif")):
|
||||
tdiv(class="gallery-gif", style={maxHeight: "unset"}):
|
||||
tdiv(class="attachment"):
|
||||
let thumb = getSmallPic(gif.thumb)
|
||||
let url = getPicUrl(gif.url)
|
||||
if prefs.autoplayGifs:
|
||||
video(class="gif", poster=thumb, controls="", autoplay="", muted="", loop=""):
|
||||
source(src=url, `type`="video/mp4")
|
||||
else:
|
||||
video(class="gif", poster=thumb, controls="", muted="", loop=""):
|
||||
source(src=url, `type`="video/mp4")
|
||||
video(class="gif", poster=getSmallPic(gif.thumb), autoplay=prefs.autoplayGifs,
|
||||
controls="", muted="", loop=""):
|
||||
source(src=getPicUrl(gif.url), `type`="video/mp4")
|
||||
|
||||
proc renderPoll(poll: Poll): VNode =
|
||||
buildHtml(tdiv(class="poll")):
|
||||
|
|
@ -328,7 +319,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
|
|||
if tweet.attribution.isSome:
|
||||
renderAttribution(tweet.attribution.get(), prefs)
|
||||
|
||||
if tweet.card.isSome:
|
||||
if tweet.card.isSome and tweet.card.get().kind != hidden:
|
||||
renderCard(tweet.card.get(), prefs, path)
|
||||
|
||||
if tweet.photos.len > 0:
|
||||
|
|
|
|||
|
|
@ -3,11 +3,6 @@ from parameterized import parameterized
|
|||
|
||||
|
||||
card = [
|
||||
['Thom_Wolf/status/1122466524860702729',
|
||||
'facebookresearch/fairseq',
|
||||
'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.',
|
||||
'github.com', True],
|
||||
|
||||
['nim_lang/status/1136652293510717440',
|
||||
'Version 0.20.0 released',
|
||||
'We are very proud to announce Nim version 0.20. This is a massive release, both literally and figuratively. It contains more than 1,000 commits and it marks our release candidate for version 1.0!',
|
||||
|
|
@ -25,6 +20,11 @@ card = [
|
|||
]
|
||||
|
||||
no_thumb = [
|
||||
['Thom_Wolf/status/1122466524860702729',
|
||||
'facebookresearch/fairseq',
|
||||
'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.',
|
||||
'github.com'],
|
||||
|
||||
['brent_p/status/1088857328680488961',
|
||||
'Hts Nim Sugar',
|
||||
'hts-nim is a library that allows one to use htslib via the nim programming language. Nim is a garbage-collected language that compiles to C and often has similar performance. I have become very...',
|
||||
|
|
|
|||
|
|
@ -17,11 +17,6 @@ protected = [
|
|||
|
||||
invalid = [['thisprofiledoesntexist'], ['%']]
|
||||
|
||||
banner_color = [
|
||||
['nim_lang', '22, 25, 32'],
|
||||
['rustlang', '35, 31, 32']
|
||||
]
|
||||
|
||||
banner_image = [
|
||||
['mobile_test', 'profile_banners%2F82135242%2F1384108037%2F1500x500']
|
||||
]
|
||||
|
|
@ -74,12 +69,6 @@ class ProfileTest(BaseTestCase):
|
|||
self.open_nitter('user')
|
||||
self.assert_text('User "user" has been suspended')
|
||||
|
||||
@parameterized.expand(banner_color)
|
||||
def test_banner_color(self, username, color):
|
||||
self.open_nitter(username)
|
||||
banner = self.find_element(Profile.banner + ' a')
|
||||
self.assertIn(color, banner.value_of_css_property('background-color'))
|
||||
|
||||
@parameterized.expand(banner_image)
|
||||
def test_banner_image(self, username, url):
|
||||
self.open_nitter(username)
|
||||
|
|
|
|||
Loading…
Reference in a new issue