Merge remote-tracking branch 'upstream/master'
This commit is contained in:
PrivacyDev
commit
b2beabf6cd
8 changed files with 180 additions and 166 deletions
28
src/api.nim
28
src/api.nim
|
|
@ -33,12 +33,12 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi
|
|||
js = await fetch(url ? params, apiId)
|
||||
result = parseGraphTimeline(js, "user", after)
|
||||
|
||||
proc getTimeline*(id: string; after=""; replies=false): Future[Profile] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
|
||||
url = oldUserTweets / (id & ".json") ? ps
|
||||
result = parseTimeline(await fetch(url, Api.timeline), after)
|
||||
# proc getTimeline*(id: string; after=""; replies=false): Future[Profile] {.async.} =
|
||||
# if id.len == 0: return
|
||||
# let
|
||||
# ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
|
||||
# url = oldUserTweets / (id & ".json") ? ps
|
||||
# result = parseTimeline(await fetch(url, Api.timeline), after)
|
||||
|
||||
proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
|
||||
if id.len == 0: return
|
||||
|
|
@ -166,20 +166,22 @@ proc getGraphSearch*(query: Query; after=""): Future[Profile] {.async.} =
|
|||
result.tweets.query = query
|
||||
|
||||
proc getTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
|
||||
let q = genQueryParam(query)
|
||||
var q = genQueryParam(query)
|
||||
|
||||
if q.len == 0 or q == emptyQuery:
|
||||
return Timeline(query: query, beginning: true)
|
||||
|
||||
if after.len > 0:
|
||||
q &= " max_id:" & after
|
||||
|
||||
let url = tweetSearch ? genParams({
|
||||
"q": q,
|
||||
"tweet_search_mode": "live",
|
||||
"max_id": after
|
||||
"q": q ,
|
||||
"modules": "status",
|
||||
"result_type": "recent",
|
||||
})
|
||||
|
||||
result = parseTweetSearch(await fetch(url, Api.search))
|
||||
result = parseTweetSearch(await fetch(url, Api.search), after)
|
||||
result.query = query
|
||||
if after.len == 0:
|
||||
result.beginning = true
|
||||
|
||||
proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} =
|
||||
if query.text.len == 0:
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
|
|||
for p in pars:
|
||||
result &= p
|
||||
if ext:
|
||||
result &= ("ext", "mediaStats")
|
||||
result &= ("ext", "mediaStats,isBlueVerified,isVerified,blue,blueVerified")
|
||||
result &= ("include_ext_alt_text", "1")
|
||||
result &= ("include_ext_media_availability", "1")
|
||||
if count.len > 0:
|
||||
|
|
|
|||
|
|
@ -12,9 +12,9 @@ const
|
|||
timelineApi = api / "2/timeline"
|
||||
favorites* = timelineApi / "favorites"
|
||||
userSearch* = api / "1.1/users/search.json"
|
||||
tweetSearch* = api / "1.1/search/tweets.json"
|
||||
tweetSearch* = api / "1.1/search/universal.json"
|
||||
|
||||
oldUserTweets* = api / "2/timeline/profile"
|
||||
# oldUserTweets* = api / "2/timeline/profile"
|
||||
|
||||
graphql = api / "graphql"
|
||||
graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery"
|
||||
|
|
@ -35,27 +35,28 @@ const
|
|||
graphFollowing* = graphql / "JPZiqKjET7_M1r5Tlr8pyA/Following"
|
||||
|
||||
timelineParams* = {
|
||||
"include_profile_interstitial_type": "0",
|
||||
"include_blocking": "0",
|
||||
"include_blocked_by": "0",
|
||||
"include_followed_by": "0",
|
||||
"include_want_retweets": "0",
|
||||
"include_mute_edge": "0",
|
||||
"include_can_dm": "0",
|
||||
"include_can_media_tag": "1",
|
||||
"include_ext_is_blue_verified": "1",
|
||||
"skip_status": "1",
|
||||
"cards_platform": "Web-12",
|
||||
"include_cards": "1",
|
||||
"include_composer_source": "0",
|
||||
"include_reply_count": "1",
|
||||
"cards_platform": "Web-13",
|
||||
"tweet_mode": "extended",
|
||||
"include_entities": "1",
|
||||
"include_user_entities": "1",
|
||||
"include_ext_media_color": "0",
|
||||
"ui_lang": "en-US",
|
||||
"send_error_codes": "1",
|
||||
"simple_quoted_tweet": "1",
|
||||
"include_quote_count": "1"
|
||||
"skip_status": "1",
|
||||
"include_blocked_by": "0",
|
||||
"include_blocking": "0",
|
||||
"include_can_dm": "0",
|
||||
"include_can_media_tag": "1",
|
||||
"include_cards": "1",
|
||||
"include_composer_source": "0",
|
||||
"include_entities": "1",
|
||||
"include_ext_is_blue_verified": "1",
|
||||
"include_ext_media_color": "0",
|
||||
"include_followed_by": "0",
|
||||
"include_mute_edge": "0",
|
||||
"include_profile_interstitial_type": "0",
|
||||
"include_quote_count": "1",
|
||||
"include_reply_count": "1",
|
||||
"include_user_entities": "1",
|
||||
"include_want_retweets": "0",
|
||||
}.toSeq
|
||||
|
||||
gqlFeatures* = """{
|
||||
|
|
@ -107,17 +108,17 @@ const
|
|||
"includeHasBirdwatchNotes": false
|
||||
}"""
|
||||
|
||||
oldUserTweetsVariables* = """{
|
||||
"userId": "$1", $2
|
||||
"count": 20,
|
||||
"includePromotedContent": false,
|
||||
"withDownvotePerspective": false,
|
||||
"withReactionsMetadata": false,
|
||||
"withReactionsPerspective": false,
|
||||
"withVoice": false,
|
||||
"withV2Timeline": true
|
||||
}
|
||||
"""
|
||||
# oldUserTweetsVariables* = """{
|
||||
# "userId": "$1", $2
|
||||
# "count": 20,
|
||||
# "includePromotedContent": false,
|
||||
# "withDownvotePerspective": false,
|
||||
# "withReactionsMetadata": false,
|
||||
# "withReactionsPerspective": false,
|
||||
# "withVoice": false,
|
||||
# "withV2Timeline": true
|
||||
# }
|
||||
# """
|
||||
|
||||
userTweetsVariables* = """{
|
||||
"rest_id": "$1", $2
|
||||
|
|
|
|||
194
src/parser.nim
194
src/parser.nim
|
|
@ -1,5 +1,5 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
import strutils, options, tables, times, math
|
||||
import strutils, options, times, math
|
||||
import packedjson, packedjson/deserialiser
|
||||
import types, parserutils, utils
|
||||
import experimental/parser/unifiedcard
|
||||
|
|
@ -302,110 +302,112 @@ proc parseLegacyTweet(js: JsonNode): Tweet =
|
|||
if result.quote.isSome:
|
||||
result.quote = some parseLegacyTweet(js{"quoted_status"})
|
||||
|
||||
proc parseTweetSearch*(js: JsonNode): Timeline =
|
||||
if js.kind == JNull or "statuses" notin js:
|
||||
return Timeline(beginning: true)
|
||||
proc parseTweetSearch*(js: JsonNode; after=""): Timeline =
|
||||
result.beginning = after.len == 0
|
||||
|
||||
for tweet in js{"statuses"}:
|
||||
let parsed = parseLegacyTweet(tweet)
|
||||
|
||||
if parsed.retweet.isSome:
|
||||
parsed.retweet = some parseLegacyTweet(tweet{"retweeted_status"})
|
||||
|
||||
result.content.add @[parsed]
|
||||
|
||||
let cursor = js{"search_metadata", "next_results"}.getStr
|
||||
if cursor.len > 0 and "max_id" in cursor:
|
||||
result.bottom = cursor[cursor.find("=") + 1 .. cursor.find("&q=")]
|
||||
|
||||
proc finalizeTweet(global: GlobalObjects; id: string): Tweet =
|
||||
let intId = if id.len > 0: parseBiggestInt(id) else: 0
|
||||
result = global.tweets.getOrDefault(id, Tweet(id: intId))
|
||||
|
||||
if result.quote.isSome:
|
||||
let quote = get(result.quote).id
|
||||
if $quote in global.tweets:
|
||||
result.quote = some global.tweets[$quote]
|
||||
else:
|
||||
result.quote = some Tweet()
|
||||
|
||||
if result.retweet.isSome:
|
||||
let rt = get(result.retweet).id
|
||||
if $rt in global.tweets:
|
||||
result.retweet = some finalizeTweet(global, $rt)
|
||||
else:
|
||||
result.retweet = some Tweet()
|
||||
|
||||
proc parsePin(js: JsonNode; global: GlobalObjects): Tweet =
|
||||
let pin = js{"pinEntry", "entry", "entryId"}.getStr
|
||||
if pin.len == 0: return
|
||||
|
||||
let id = pin.getId
|
||||
if id notin global.tweets: return
|
||||
|
||||
global.tweets[id].pinned = true
|
||||
return finalizeTweet(global, id)
|
||||
|
||||
proc parseGlobalObjects(js: JsonNode): GlobalObjects =
|
||||
result = GlobalObjects()
|
||||
let
|
||||
tweets = ? js{"globalObjects", "tweets"}
|
||||
users = ? js{"globalObjects", "users"}
|
||||
|
||||
for k, v in users:
|
||||
result.users[k] = parseUser(v, k)
|
||||
|
||||
for k, v in tweets:
|
||||
var tweet = parseTweet(v, v{"card"})
|
||||
if tweet.user.id in result.users:
|
||||
tweet.user = result.users[tweet.user.id]
|
||||
result.tweets[k] = tweet
|
||||
|
||||
proc parseInstructions(res: var Profile; global: GlobalObjects; js: JsonNode) =
|
||||
if js.kind != JArray or js.len == 0:
|
||||
if js.kind == JNull or "modules" notin js or js{"modules"}.len == 0:
|
||||
return
|
||||
|
||||
for i in js:
|
||||
if res.tweets.beginning and i{"pinEntry"}.notNull:
|
||||
with pin, parsePin(i, global):
|
||||
res.pinned = some pin
|
||||
for item in js{"modules"}:
|
||||
with tweet, item{"status", "data"}:
|
||||
let parsed = parseLegacyTweet(tweet)
|
||||
|
||||
with r, i{"replaceEntry", "entry"}:
|
||||
if "top" in r{"entryId"}.getStr:
|
||||
res.tweets.top = r.getCursor
|
||||
elif "bottom" in r{"entryId"}.getStr:
|
||||
res.tweets.bottom = r.getCursor
|
||||
if parsed.retweet.isSome:
|
||||
parsed.retweet = some parseLegacyTweet(tweet{"retweeted_status"})
|
||||
|
||||
proc parseTimeline*(js: JsonNode; after=""): Profile =
|
||||
result = Profile(tweets: Timeline(beginning: after.len == 0))
|
||||
let global = parseGlobalObjects(? js)
|
||||
result.content.add @[parsed]
|
||||
|
||||
let instructions = ? js{"timeline", "instructions"}
|
||||
if instructions.len == 0: return
|
||||
if result.content.len > 0:
|
||||
result.bottom = $(result.content[^1][0].id - 1)
|
||||
|
||||
result.parseInstructions(global, instructions)
|
||||
# proc finalizeTweet(global: GlobalObjects; id: string): Tweet =
|
||||
# let intId = if id.len > 0: parseBiggestInt(id) else: 0
|
||||
# result = global.tweets.getOrDefault(id, Tweet(id: intId))
|
||||
|
||||
var entries: JsonNode
|
||||
for i in instructions:
|
||||
if "addEntries" in i:
|
||||
entries = i{"addEntries", "entries"}
|
||||
# if result.quote.isSome:
|
||||
# let quote = get(result.quote).id
|
||||
# if $quote in global.tweets:
|
||||
# result.quote = some global.tweets[$quote]
|
||||
# else:
|
||||
# result.quote = some Tweet()
|
||||
|
||||
for e in ? entries:
|
||||
let entry = e{"entryId"}.getStr
|
||||
if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry:
|
||||
let tweet = finalizeTweet(global, e.getEntryId)
|
||||
if not tweet.available: continue
|
||||
result.tweets.content.add tweet
|
||||
elif "cursor-top" in entry:
|
||||
result.tweets.top = e.getCursor
|
||||
elif "cursor-bottom" in entry:
|
||||
result.tweets.bottom = e.getCursor
|
||||
elif entry.startsWith("sq-cursor"):
|
||||
with cursor, e{"content", "operation", "cursor"}:
|
||||
if cursor{"cursorType"}.getStr == "Bottom":
|
||||
result.tweets.bottom = cursor{"value"}.getStr
|
||||
else:
|
||||
result.tweets.top = cursor{"value"}.getStr
|
||||
# if result.retweet.isSome:
|
||||
# let rt = get(result.retweet).id
|
||||
# if $rt in global.tweets:
|
||||
# result.retweet = some finalizeTweet(global, $rt)
|
||||
# else:
|
||||
# result.retweet = some Tweet()
|
||||
|
||||
# proc parsePin(js: JsonNode; global: GlobalObjects): Tweet =
|
||||
# let pin = js{"pinEntry", "entry", "entryId"}.getStr
|
||||
# if pin.len == 0: return
|
||||
|
||||
# let id = pin.getId
|
||||
# if id notin global.tweets: return
|
||||
|
||||
# global.tweets[id].pinned = true
|
||||
# return finalizeTweet(global, id)
|
||||
|
||||
# proc parseGlobalObjects(js: JsonNode): GlobalObjects =
|
||||
# result = GlobalObjects()
|
||||
# let
|
||||
# tweets = ? js{"globalObjects", "tweets"}
|
||||
# users = ? js{"globalObjects", "users"}
|
||||
|
||||
# for k, v in users:
|
||||
# result.users[k] = parseUser(v, k)
|
||||
|
||||
# for k, v in tweets:
|
||||
# var tweet = parseTweet(v, v{"card"})
|
||||
# if tweet.user.id in result.users:
|
||||
# tweet.user = result.users[tweet.user.id]
|
||||
# result.tweets[k] = tweet
|
||||
|
||||
# proc parseInstructions(res: var Profile; global: GlobalObjects; js: JsonNode) =
|
||||
# if js.kind != JArray or js.len == 0:
|
||||
# return
|
||||
|
||||
# for i in js:
|
||||
# if res.tweets.beginning and i{"pinEntry"}.notNull:
|
||||
# with pin, parsePin(i, global):
|
||||
# res.pinned = some pin
|
||||
|
||||
# with r, i{"replaceEntry", "entry"}:
|
||||
# if "top" in r{"entryId"}.getStr:
|
||||
# res.tweets.top = r.getCursor
|
||||
# elif "bottom" in r{"entryId"}.getStr:
|
||||
# res.tweets.bottom = r.getCursor
|
||||
|
||||
# proc parseTimeline*(js: JsonNode; after=""): Profile =
|
||||
# result = Profile(tweets: Timeline(beginning: after.len == 0))
|
||||
# let global = parseGlobalObjects(? js)
|
||||
|
||||
# let instructions = ? js{"timeline", "instructions"}
|
||||
# if instructions.len == 0: return
|
||||
|
||||
# result.parseInstructions(global, instructions)
|
||||
|
||||
# var entries: JsonNode
|
||||
# for i in instructions:
|
||||
# if "addEntries" in i:
|
||||
# entries = i{"addEntries", "entries"}
|
||||
|
||||
# for e in ? entries:
|
||||
# let entry = e{"entryId"}.getStr
|
||||
# if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry:
|
||||
# let tweet = finalizeTweet(global, e.getEntryId)
|
||||
# if not tweet.available: continue
|
||||
# result.tweets.content.add tweet
|
||||
# elif "cursor-top" in entry:
|
||||
# result.tweets.top = e.getCursor
|
||||
# elif "cursor-bottom" in entry:
|
||||
# result.tweets.bottom = e.getCursor
|
||||
# elif entry.startsWith("sq-cursor"):
|
||||
# with cursor, e{"content", "operation", "cursor"}:
|
||||
# if cursor{"cursorType"}.getStr == "Bottom":
|
||||
# result.tweets.bottom = cursor{"value"}.getStr
|
||||
# else:
|
||||
# result.tweets.top = cursor{"value"}.getStr
|
||||
|
||||
proc parsePhotoRail*(js: JsonNode): PhotoRail =
|
||||
with error, js{"error"}:
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ proc fetchProfile*(after: string; query: Query; cfg: Config; skipRail=false;
|
|||
|
||||
result =
|
||||
case query.kind
|
||||
of posts: await getTimeline(userId, after)
|
||||
# of posts: await getTimeline(userId, after)
|
||||
of replies: await getGraphUserTweets(userId, TimelineKind.replies, after)
|
||||
of media: await getGraphUserTweets(userId, TimelineKind.media, after)
|
||||
of favorites: await getFavorites(userId, cfg, after)
|
||||
|
|
@ -63,10 +63,18 @@ proc fetchProfile*(after: string; query: Query; cfg: Config; skipRail=false;
|
|||
result.user = await user
|
||||
result.photoRail = await rail
|
||||
|
||||
result.tweets.query = query
|
||||
|
||||
if result.user.protected or result.user.suspended:
|
||||
return
|
||||
|
||||
result.tweets.query = query
|
||||
if not skipPinned and query.kind == posts and
|
||||
result.user.pinnedTweet > 0 and after.len == 0:
|
||||
let tweet = await getCachedTweet(result.user.pinnedTweet)
|
||||
if not tweet.isNil:
|
||||
tweet.pinned = true
|
||||
tweet.user = result.user
|
||||
result.pinned = some tweet
|
||||
|
||||
proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
|
||||
rss, after: string): Future[string] {.async.} =
|
||||
|
|
|
|||
|
|
@ -41,7 +41,8 @@ proc getPoolJson*(): JsonNode =
|
|||
let
|
||||
maxReqs =
|
||||
case api
|
||||
of Api.photoRail, Api.search: 180
|
||||
of Api.search: 100000
|
||||
of Api.photoRail: 180
|
||||
of Api.timeline: 187
|
||||
of Api.userTweets: 300
|
||||
of Api.userTweetsAndReplies, Api.userRestId,
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
from base import BaseTestCase, Timeline
|
||||
from parameterized import parameterized
|
||||
|
||||
normal = [['mobile_test'], ['mobile_test_2']]
|
||||
normal = [['jack'], ['elonmusk']]
|
||||
|
||||
after = [['mobile_test', 'HBaAgJPsqtGNhA0AAA%3D%3D'],
|
||||
['mobile_test_2', 'HBaAgJPsqtGNhA0AAA%3D%3D']]
|
||||
after = [['jack', '1681686036294803456'],
|
||||
['elonmusk', '1681686036294803456']]
|
||||
|
||||
no_more = [['mobile_test_8?cursor=HBaAwJCsk%2F6%2FtgQAAA%3D%3D']]
|
||||
no_more = [['mobile_test_8?cursor=1000']]
|
||||
|
||||
empty = [['emptyuser'], ['mobile_test_10']]
|
||||
|
||||
|
|
|
|||
|
|
@ -80,16 +80,16 @@ retweet = [
|
|||
|
||||
|
||||
class TweetTest(BaseTestCase):
|
||||
@parameterized.expand(timeline)
|
||||
def test_timeline(self, index, fullname, username, date, tid, text):
|
||||
self.open_nitter(username)
|
||||
tweet = get_timeline_tweet(index)
|
||||
self.assert_exact_text(fullname, tweet.fullname)
|
||||
self.assert_exact_text('@' + username, tweet.username)
|
||||
self.assert_exact_text(date, tweet.date)
|
||||
self.assert_text(text, tweet.text)
|
||||
permalink = self.find_element(tweet.date + ' a')
|
||||
self.assertIn(tid, permalink.get_attribute('href'))
|
||||
# @parameterized.expand(timeline)
|
||||
# def test_timeline(self, index, fullname, username, date, tid, text):
|
||||
# self.open_nitter(username)
|
||||
# tweet = get_timeline_tweet(index)
|
||||
# self.assert_exact_text(fullname, tweet.fullname)
|
||||
# self.assert_exact_text('@' + username, tweet.username)
|
||||
# self.assert_exact_text(date, tweet.date)
|
||||
# self.assert_text(text, tweet.text)
|
||||
# permalink = self.find_element(tweet.date + ' a')
|
||||
# self.assertIn(tid, permalink.get_attribute('href'))
|
||||
|
||||
@parameterized.expand(status)
|
||||
def test_status(self, tid, fullname, username, date, text):
|
||||
|
|
@ -123,14 +123,14 @@ class TweetTest(BaseTestCase):
|
|||
link = self.find_link_text(f'@{un}')
|
||||
self.assertIn(f'/{un}', link.get_property('href'))
|
||||
|
||||
@parameterized.expand(retweet)
|
||||
def test_retweet(self, index, url, retweet_by, fullname, username, text):
|
||||
self.open_nitter(url)
|
||||
tweet = get_timeline_tweet(index)
|
||||
self.assert_text(f'{retweet_by} retweeted', tweet.retweet)
|
||||
self.assert_text(text, tweet.text)
|
||||
self.assert_exact_text(fullname, tweet.fullname)
|
||||
self.assert_exact_text(username, tweet.username)
|
||||
# @parameterized.expand(retweet)
|
||||
# def test_retweet(self, index, url, retweet_by, fullname, username, text):
|
||||
# self.open_nitter(url)
|
||||
# tweet = get_timeline_tweet(index)
|
||||
# self.assert_text(f'{retweet_by} retweeted', tweet.retweet)
|
||||
# self.assert_text(text, tweet.text)
|
||||
# self.assert_exact_text(fullname, tweet.fullname)
|
||||
# self.assert_exact_text(username, tweet.username)
|
||||
|
||||
@parameterized.expand(invalid)
|
||||
def test_invalid_id(self, tweet):
|
||||
|
|
|
|||
Loading…
Reference in a new issue