From 50f821dbd8a7bbea75e9cdf2c4189e108b7b0541 Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 22 Jul 2023 03:03:45 +0200 Subject: [PATCH 1/2] Use search instead of old timeline endpoint --- src/api.nim | 28 +++--- src/apiutils.nim | 2 +- src/consts.nim | 63 ++++++------- src/parser.nim | 194 ++++++++++++++++++++-------------------- src/routes/timeline.nim | 12 ++- tests/test_timeline.py | 8 +- tests/test_tweet.py | 36 ++++---- 7 files changed, 178 insertions(+), 165 deletions(-) diff --git a/src/api.nim b/src/api.nim index e8d0830..c7dc0e0 100644 --- a/src/api.nim +++ b/src/api.nim @@ -33,12 +33,12 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi js = await fetch(url ? params, apiId) result = parseGraphTimeline(js, "user", after) -proc getTimeline*(id: string; after=""; replies=false): Future[Profile] {.async.} = - if id.len == 0: return - let - ps = genParams({"userId": id, "include_tweet_replies": $replies}, after) - url = oldUserTweets / (id & ".json") ? ps - result = parseTimeline(await fetch(url, Api.timeline), after) +# proc getTimeline*(id: string; after=""; replies=false): Future[Profile] {.async.} = +# if id.len == 0: return +# let +# ps = genParams({"userId": id, "include_tweet_replies": $replies}, after) +# url = oldUserTweets / (id & ".json") ? ps +# result = parseTimeline(await fetch(url, Api.timeline), after) proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} = if id.len == 0: return @@ -123,20 +123,22 @@ proc getGraphSearch*(query: Query; after=""): Future[Profile] {.async.} = result.tweets.query = query proc getTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} = - let q = genQueryParam(query) + var q = genQueryParam(query) + if q.len == 0 or q == emptyQuery: return Timeline(query: query, beginning: true) + if after.len > 0: + q &= " max_id:" & after + let url = tweetSearch ? genParams({ - "q": q, - "tweet_search_mode": "live", - "max_id": after + "q": q , + "modules": "status", + "result_type": "recent", }) - result = parseTweetSearch(await fetch(url, Api.search)) + result = parseTweetSearch(await fetch(url, Api.search), after) result.query = query - if after.len == 0: - result.beginning = true proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} = if query.text.len == 0: diff --git a/src/apiutils.nim b/src/apiutils.nim index dbc6cca..c0c01d4 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -16,7 +16,7 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor=""; for p in pars: result &= p if ext: - result &= ("ext", "mediaStats") + result &= ("ext", "mediaStats,isBlueVerified,isVerified,blue,blueVerified") result &= ("include_ext_alt_text", "1") result &= ("include_ext_media_availability", "1") if count.len > 0: diff --git a/src/consts.nim b/src/consts.nim index 7ba09fb..80a098f 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -9,9 +9,9 @@ const photoRail* = api / "1.1/statuses/media_timeline.json" userSearch* = api / "1.1/users/search.json" - tweetSearch* = api / "1.1/search/tweets.json" + tweetSearch* = api / "1.1/search/universal.json" - oldUserTweets* = api / "2/timeline/profile" + # oldUserTweets* = api / "2/timeline/profile" graphql = api / "graphql" graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery" @@ -28,27 +28,28 @@ const graphListTweets* = graphql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline" timelineParams* = { - "include_profile_interstitial_type": "0", - "include_blocking": "0", - "include_blocked_by": "0", - "include_followed_by": "0", - "include_want_retweets": "0", - "include_mute_edge": "0", - "include_can_dm": "0", - "include_can_media_tag": "1", - "include_ext_is_blue_verified": "1", - "skip_status": "1", - "cards_platform": "Web-12", - "include_cards": "1", - "include_composer_source": "0", - "include_reply_count": "1", + "cards_platform": "Web-13", "tweet_mode": "extended", - "include_entities": "1", - "include_user_entities": "1", - "include_ext_media_color": "0", + "ui_lang": "en-US", "send_error_codes": "1", "simple_quoted_tweet": "1", - "include_quote_count": "1" + "skip_status": "1", + "include_blocked_by": "0", + "include_blocking": "0", + "include_can_dm": "0", + "include_can_media_tag": "1", + "include_cards": "1", + "include_composer_source": "0", + "include_entities": "1", + "include_ext_is_blue_verified": "1", + "include_ext_media_color": "0", + "include_followed_by": "0", + "include_mute_edge": "0", + "include_profile_interstitial_type": "0", + "include_quote_count": "1", + "include_reply_count": "1", + "include_user_entities": "1", + "include_want_retweets": "0", }.toSeq gqlFeatures* = """{ @@ -100,17 +101,17 @@ const "includeHasBirdwatchNotes": false }""" - oldUserTweetsVariables* = """{ - "userId": "$1", $2 - "count": 20, - "includePromotedContent": false, - "withDownvotePerspective": false, - "withReactionsMetadata": false, - "withReactionsPerspective": false, - "withVoice": false, - "withV2Timeline": true -} -""" +# oldUserTweetsVariables* = """{ +# "userId": "$1", $2 +# "count": 20, +# "includePromotedContent": false, +# "withDownvotePerspective": false, +# "withReactionsMetadata": false, +# "withReactionsPerspective": false, +# "withVoice": false, +# "withV2Timeline": true +# } +# """ userTweetsVariables* = """{ "rest_id": "$1", $2 diff --git a/src/parser.nim b/src/parser.nim index 193d77f..991ca6e 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -1,5 +1,5 @@ # SPDX-License-Identifier: AGPL-3.0-only -import strutils, options, tables, times, math +import strutils, options, times, math import packedjson, packedjson/deserialiser import types, parserutils, utils import experimental/parser/unifiedcard @@ -295,110 +295,112 @@ proc parseLegacyTweet(js: JsonNode): Tweet = if result.quote.isSome: result.quote = some parseLegacyTweet(js{"quoted_status"}) -proc parseTweetSearch*(js: JsonNode): Timeline = - if js.kind == JNull or "statuses" notin js: - return Timeline(beginning: true) +proc parseTweetSearch*(js: JsonNode; after=""): Timeline = + result.beginning = after.len == 0 - for tweet in js{"statuses"}: - let parsed = parseLegacyTweet(tweet) - - if parsed.retweet.isSome: - parsed.retweet = some parseLegacyTweet(tweet{"retweeted_status"}) - - result.content.add @[parsed] - - let cursor = js{"search_metadata", "next_results"}.getStr - if cursor.len > 0 and "max_id" in cursor: - result.bottom = cursor[cursor.find("=") + 1 .. cursor.find("&q=")] - -proc finalizeTweet(global: GlobalObjects; id: string): Tweet = - let intId = if id.len > 0: parseBiggestInt(id) else: 0 - result = global.tweets.getOrDefault(id, Tweet(id: intId)) - - if result.quote.isSome: - let quote = get(result.quote).id - if $quote in global.tweets: - result.quote = some global.tweets[$quote] - else: - result.quote = some Tweet() - - if result.retweet.isSome: - let rt = get(result.retweet).id - if $rt in global.tweets: - result.retweet = some finalizeTweet(global, $rt) - else: - result.retweet = some Tweet() - -proc parsePin(js: JsonNode; global: GlobalObjects): Tweet = - let pin = js{"pinEntry", "entry", "entryId"}.getStr - if pin.len == 0: return - - let id = pin.getId - if id notin global.tweets: return - - global.tweets[id].pinned = true - return finalizeTweet(global, id) - -proc parseGlobalObjects(js: JsonNode): GlobalObjects = - result = GlobalObjects() - let - tweets = ? js{"globalObjects", "tweets"} - users = ? js{"globalObjects", "users"} - - for k, v in users: - result.users[k] = parseUser(v, k) - - for k, v in tweets: - var tweet = parseTweet(v, v{"card"}) - if tweet.user.id in result.users: - tweet.user = result.users[tweet.user.id] - result.tweets[k] = tweet - -proc parseInstructions(res: var Profile; global: GlobalObjects; js: JsonNode) = - if js.kind != JArray or js.len == 0: + if js.kind == JNull or "modules" notin js or js{"modules"}.len == 0: return - for i in js: - if res.tweets.beginning and i{"pinEntry"}.notNull: - with pin, parsePin(i, global): - res.pinned = some pin + for item in js{"modules"}: + with tweet, item{"status", "data"}: + let parsed = parseLegacyTweet(tweet) - with r, i{"replaceEntry", "entry"}: - if "top" in r{"entryId"}.getStr: - res.tweets.top = r.getCursor - elif "bottom" in r{"entryId"}.getStr: - res.tweets.bottom = r.getCursor + if parsed.retweet.isSome: + parsed.retweet = some parseLegacyTweet(tweet{"retweeted_status"}) -proc parseTimeline*(js: JsonNode; after=""): Profile = - result = Profile(tweets: Timeline(beginning: after.len == 0)) - let global = parseGlobalObjects(? js) + result.content.add @[parsed] - let instructions = ? js{"timeline", "instructions"} - if instructions.len == 0: return + if result.content.len > 0: + result.bottom = $(result.content[^1][0].id - 1) - result.parseInstructions(global, instructions) +# proc finalizeTweet(global: GlobalObjects; id: string): Tweet = +# let intId = if id.len > 0: parseBiggestInt(id) else: 0 +# result = global.tweets.getOrDefault(id, Tweet(id: intId)) - var entries: JsonNode - for i in instructions: - if "addEntries" in i: - entries = i{"addEntries", "entries"} +# if result.quote.isSome: +# let quote = get(result.quote).id +# if $quote in global.tweets: +# result.quote = some global.tweets[$quote] +# else: +# result.quote = some Tweet() - for e in ? entries: - let entry = e{"entryId"}.getStr - if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry: - let tweet = finalizeTweet(global, e.getEntryId) - if not tweet.available: continue - result.tweets.content.add tweet - elif "cursor-top" in entry: - result.tweets.top = e.getCursor - elif "cursor-bottom" in entry: - result.tweets.bottom = e.getCursor - elif entry.startsWith("sq-cursor"): - with cursor, e{"content", "operation", "cursor"}: - if cursor{"cursorType"}.getStr == "Bottom": - result.tweets.bottom = cursor{"value"}.getStr - else: - result.tweets.top = cursor{"value"}.getStr +# if result.retweet.isSome: +# let rt = get(result.retweet).id +# if $rt in global.tweets: +# result.retweet = some finalizeTweet(global, $rt) +# else: +# result.retweet = some Tweet() + +# proc parsePin(js: JsonNode; global: GlobalObjects): Tweet = +# let pin = js{"pinEntry", "entry", "entryId"}.getStr +# if pin.len == 0: return + +# let id = pin.getId +# if id notin global.tweets: return + +# global.tweets[id].pinned = true +# return finalizeTweet(global, id) + +# proc parseGlobalObjects(js: JsonNode): GlobalObjects = +# result = GlobalObjects() +# let +# tweets = ? js{"globalObjects", "tweets"} +# users = ? js{"globalObjects", "users"} + +# for k, v in users: +# result.users[k] = parseUser(v, k) + +# for k, v in tweets: +# var tweet = parseTweet(v, v{"card"}) +# if tweet.user.id in result.users: +# tweet.user = result.users[tweet.user.id] +# result.tweets[k] = tweet + +# proc parseInstructions(res: var Profile; global: GlobalObjects; js: JsonNode) = +# if js.kind != JArray or js.len == 0: +# return + +# for i in js: +# if res.tweets.beginning and i{"pinEntry"}.notNull: +# with pin, parsePin(i, global): +# res.pinned = some pin + +# with r, i{"replaceEntry", "entry"}: +# if "top" in r{"entryId"}.getStr: +# res.tweets.top = r.getCursor +# elif "bottom" in r{"entryId"}.getStr: +# res.tweets.bottom = r.getCursor + +# proc parseTimeline*(js: JsonNode; after=""): Profile = +# result = Profile(tweets: Timeline(beginning: after.len == 0)) +# let global = parseGlobalObjects(? js) + +# let instructions = ? js{"timeline", "instructions"} +# if instructions.len == 0: return + +# result.parseInstructions(global, instructions) + +# var entries: JsonNode +# for i in instructions: +# if "addEntries" in i: +# entries = i{"addEntries", "entries"} + +# for e in ? entries: +# let entry = e{"entryId"}.getStr +# if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry: +# let tweet = finalizeTweet(global, e.getEntryId) +# if not tweet.available: continue +# result.tweets.content.add tweet +# elif "cursor-top" in entry: +# result.tweets.top = e.getCursor +# elif "cursor-bottom" in entry: +# result.tweets.bottom = e.getCursor +# elif entry.startsWith("sq-cursor"): +# with cursor, e{"content", "operation", "cursor"}: +# if cursor{"cursorType"}.getStr == "Bottom": +# result.tweets.bottom = cursor{"value"}.getStr +# else: +# result.tweets.top = cursor{"value"}.getStr proc parsePhotoRail*(js: JsonNode): PhotoRail = with error, js{"error"}: diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index b574631..8b8a23c 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -53,7 +53,7 @@ proc fetchProfile*(after: string; query: Query; skipRail=false; result = case query.kind - of posts: await getTimeline(userId, after) + # of posts: await getTimeline(userId, after) of replies: await getGraphUserTweets(userId, TimelineKind.replies, after) of media: await getGraphUserTweets(userId, TimelineKind.media, after) else: Profile(tweets: await getTweetSearch(query, after)) @@ -61,10 +61,18 @@ proc fetchProfile*(after: string; query: Query; skipRail=false; result.user = await user result.photoRail = await rail + result.tweets.query = query + if result.user.protected or result.user.suspended: return - result.tweets.query = query + if not skipPinned and query.kind == posts and + result.user.pinnedTweet > 0 and after.len == 0: + let tweet = await getCachedTweet(result.user.pinnedTweet) + if not tweet.isNil: + tweet.pinned = true + tweet.user = result.user + result.pinned = some tweet proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs; rss, after: string): Future[string] {.async.} = diff --git a/tests/test_timeline.py b/tests/test_timeline.py index dd78396..a630ae1 100644 --- a/tests/test_timeline.py +++ b/tests/test_timeline.py @@ -1,12 +1,12 @@ from base import BaseTestCase, Timeline from parameterized import parameterized -normal = [['mobile_test'], ['mobile_test_2']] +normal = [['jack'], ['elonmusk']] -after = [['mobile_test', 'HBaAgJPsqtGNhA0AAA%3D%3D'], - ['mobile_test_2', 'HBaAgJPsqtGNhA0AAA%3D%3D']] +after = [['jack', '1681686036294803456'], + ['elonmusk', '1681686036294803456']] -no_more = [['mobile_test_8?cursor=HBaAwJCsk%2F6%2FtgQAAA%3D%3D']] +no_more = [['mobile_test_8?cursor=1000']] empty = [['emptyuser'], ['mobile_test_10']] diff --git a/tests/test_tweet.py b/tests/test_tweet.py index e4231a4..7a3c4ed 100644 --- a/tests/test_tweet.py +++ b/tests/test_tweet.py @@ -80,16 +80,16 @@ retweet = [ class TweetTest(BaseTestCase): - @parameterized.expand(timeline) - def test_timeline(self, index, fullname, username, date, tid, text): - self.open_nitter(username) - tweet = get_timeline_tweet(index) - self.assert_exact_text(fullname, tweet.fullname) - self.assert_exact_text('@' + username, tweet.username) - self.assert_exact_text(date, tweet.date) - self.assert_text(text, tweet.text) - permalink = self.find_element(tweet.date + ' a') - self.assertIn(tid, permalink.get_attribute('href')) + # @parameterized.expand(timeline) + # def test_timeline(self, index, fullname, username, date, tid, text): + # self.open_nitter(username) + # tweet = get_timeline_tweet(index) + # self.assert_exact_text(fullname, tweet.fullname) + # self.assert_exact_text('@' + username, tweet.username) + # self.assert_exact_text(date, tweet.date) + # self.assert_text(text, tweet.text) + # permalink = self.find_element(tweet.date + ' a') + # self.assertIn(tid, permalink.get_attribute('href')) @parameterized.expand(status) def test_status(self, tid, fullname, username, date, text): @@ -123,14 +123,14 @@ class TweetTest(BaseTestCase): link = self.find_link_text(f'@{un}') self.assertIn(f'/{un}', link.get_property('href')) - @parameterized.expand(retweet) - def test_retweet(self, index, url, retweet_by, fullname, username, text): - self.open_nitter(url) - tweet = get_timeline_tweet(index) - self.assert_text(f'{retweet_by} retweeted', tweet.retweet) - self.assert_text(text, tweet.text) - self.assert_exact_text(fullname, tweet.fullname) - self.assert_exact_text(username, tweet.username) + # @parameterized.expand(retweet) + # def test_retweet(self, index, url, retweet_by, fullname, username, text): + # self.open_nitter(url) + # tweet = get_timeline_tweet(index) + # self.assert_text(f'{retweet_by} retweeted', tweet.retweet) + # self.assert_text(text, tweet.text) + # self.assert_exact_text(fullname, tweet.fullname) + # self.assert_exact_text(username, tweet.username) @parameterized.expand(invalid) def test_invalid_id(self, tweet): From 72d8f35cd1ec1205824711a41dab4b8d7a6b298a Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 22 Jul 2023 04:06:04 +0200 Subject: [PATCH 2/2] Search isn't rate limited --- src/tokens.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tokens.nim b/src/tokens.nim index 8a25257..b69786e 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -41,7 +41,8 @@ proc getPoolJson*(): JsonNode = let maxReqs = case api - of Api.photoRail, Api.search: 180 + of Api.search: 100000 + of Api.photoRail: 180 of Api.timeline: 187 of Api.userTweets: 300 of Api.userTweetsAndReplies, Api.userRestId,