From ae9fa02bf5f8459e59f0f18120fee9f06404aef5 Mon Sep 17 00:00:00 2001 From: Zed Date: Fri, 25 Aug 2023 16:28:30 +0200 Subject: [PATCH] Switch to TweetDetail for tweets --- src/consts.nim | 10 +++++++--- src/parser.nim | 25 +++++++++++++++---------- src/tokens.nim | 3 ++- tests/test_card.py | 10 +++++----- tests/test_timeline.py | 2 +- tests/test_tweet_media.py | 2 +- 6 files changed, 31 insertions(+), 21 deletions(-) diff --git a/src/consts.nim b/src/consts.nim index 8bf6422..96cea47 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -17,7 +17,7 @@ const graphUserTweets* = graphql / "3JNH4e9dq1BifLxAa3UMWg/UserWithProfileTweetsQueryV2" graphUserTweetsAndReplies* = graphql / "8IS8MaO-2EN6GZZZb8jF0g/UserWithProfileTweetsAndRepliesQueryV2" graphUserMedia* = graphql / "PDfFf8hGeJvUCiTyWtw4wQ/MediaTimelineV2" - graphTweet* = graphql / "83h5UyHZ9wEKBVzALX8R_g/ConversationTimelineV2" + graphTweet* = graphql / "q94uRCEn65LZThakYcPT6g/TweetDetail" graphTweetResult* = graphql / "sITyJdhRPpvpEjg4waUmTA/TweetResultByIdQuery" graphSearchTimeline* = graphql / "gkjsKepM6gl_HmFWoWKfgg/SearchTimeline" graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId" @@ -89,8 +89,12 @@ const tweetVariables* = """{ "focalTweetId": "$1", $2 - "includeHasBirdwatchNotes": false -}""" + "includeHasBirdwatchNotes": false, + "includePromotedContent": false, + "withBirdwatchNotes": false, + "withVoice": false, + "withV2Timeline": true +}""".replace(" ", "").replace("\n", "") # oldUserTweetsVariables* = """{ # "userId": "$1", $2 diff --git a/src/parser.nim b/src/parser.nim index 03242c1..d5190a3 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -324,7 +324,7 @@ proc parseGraphTweet(js: JsonNode): Tweet = of "TweetWithVisibilityResults": return parseGraphTweet(js{"tweet"}) - var jsCard = copy(js{"tweet_card", "legacy"}) + var jsCard = copy(js{"card", "legacy"}) if jsCard.kind != JNull: var values = newJObject() for val in jsCard["binding_values"]: @@ -342,7 +342,6 @@ proc parseGraphTweet(js: JsonNode): Tweet = result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"})) proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] = - let thread = js{"content", "items"} for t in js{"content", "items"}: let entryId = t{"entryId"}.getStr if "cursor-showmore" in entryId: @@ -350,11 +349,16 @@ proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] = result.thread.cursor = cursor.getStr result.thread.hasMore = true elif "tweet" in entryId: - let tweet = parseGraphTweet(t{"item", "content", "tweetResult", "result"}) - result.thread.content.add tweet + let + isLegacy = t{"item"}.hasKey("itemContent") + (contentKey, resultKey) = if isLegacy: ("itemContent", "tweet_results") + else: ("content", "tweetResult") - if t{"item", "content", "tweetDisplayType"}.getStr == "SelfThread": - result.self = true + with content, t{"item", contentKey}: + result.thread.content.add parseGraphTweet(content{resultKey, "result"}) + + if content{"tweetDisplayType"}.getStr == "SelfThread": + result.self = true proc parseGraphTweetResult*(js: JsonNode): Tweet = with tweet, js{"data", "tweet_result", "result"}: @@ -363,14 +367,14 @@ proc parseGraphTweetResult*(js: JsonNode): Tweet = proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation = result = Conversation(replies: Result[Chain](beginning: true)) - let instructions = ? js{"data", "timeline_response", "instructions"} + let instructions = ? js{"data", "threaded_conversation_with_injections_v2", "instructions"} if instructions.len == 0: return for e in instructions[0]{"entries"}: let entryId = e{"entryId"}.getStr if entryId.startsWith("tweet"): - with tweetResult, e{"content", "content", "tweetResult", "result"}: + with tweetResult, e{"content", "itemContent", "tweet_results", "result"}: let tweet = parseGraphTweet(tweetResult) if not tweet.available: @@ -385,7 +389,7 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation = let tweet = Tweet( id: parseBiggestInt(id), available: false, - text: e{"content", "content", "tombstoneInfo", "richText"}.getTombstone + text: e{"content", "itemContent", "tombstoneInfo", "richText"}.getTombstone ) if id == tweetId: @@ -397,9 +401,10 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation = if self: result.after = thread else: + echo "adding thread: ", thread.content.len result.replies.content.add thread elif entryId.startsWith("cursor-bottom"): - result.replies.bottom = e{"content", "content", "value"}.getStr + result.replies.bottom = e{"content", "itemContent", "value"}.getStr proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile = result = Profile(tweets: Timeline(beginning: after.len == 0)) diff --git a/src/tokens.nim b/src/tokens.nim index a3ed78a..bb9696c 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -52,10 +52,11 @@ proc getPoolJson*(): JsonNode = maxReqs = case api of Api.search: 50 + of Api.tweetDetail: 150 of Api.photoRail: 180 of Api.userTweets, Api.userTweetsAndReplies, Api.userMedia, Api.userRestId, Api.userScreenName, - Api.tweetResult, Api.tweetDetail, + Api.tweetResult, Api.list, Api.listTweets, Api.listMembers, Api.listBySlug: 500 of Api.userSearch: 900 reqs = maxReqs - apiStatus.remaining diff --git a/tests/test_card.py b/tests/test_card.py index f84ddca..733bd40 100644 --- a/tests/test_card.py +++ b/tests/test_card.py @@ -13,11 +13,6 @@ card = [ 'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim', 'gist.github.com', True], - ['FluentAI/status/1116417904831029248', - 'Amazon’s Alexa isn’t just AI — thousands of humans are listening', - 'One of the only ways to improve Alexa is to have human beings check it for errors', - 'theverge.com', True], - ['nim_lang/status/1082989146040340480', 'Nim in 2018: A short recap', 'There were several big news in the Nim world in 2018 – two new major releases, partnership with Status, and much more. But let us go chronologically.', @@ -25,6 +20,11 @@ card = [ ] no_thumb = [ + ['FluentAI/status/1116417904831029248', + 'Amazon’s Alexa isn’t just AI — thousands of humans are listening', + 'One of the only ways to improve Alexa is to have human beings check it for errors', + 'theverge.com'], + ['Thom_Wolf/status/1122466524860702729', 'facebookresearch/fairseq', 'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.', diff --git a/tests/test_timeline.py b/tests/test_timeline.py index b56d6ad..919aa70 100644 --- a/tests/test_timeline.py +++ b/tests/test_timeline.py @@ -6,7 +6,7 @@ normal = [['jack'], ['elonmusk']] after = [['jack', '1681686036294803456'], ['elonmusk', '1681686036294803456']] -no_more = [['mobile_test_8?cursor=1000']] +no_more = [['mobile_test_8?cursor=DAABCgABF4YVAqN___kKAAICNn_4msIQAAgAAwAAAAIAAA']] empty = [['emptyuser'], ['mobile_test_10']] diff --git a/tests/test_tweet_media.py b/tests/test_tweet_media.py index 7a00983..f54cea7 100644 --- a/tests/test_tweet_media.py +++ b/tests/test_tweet_media.py @@ -14,7 +14,7 @@ poll = [ image = [ ['mobile_test/status/519364660823207936', 'BzUnaDFCUAAmrjs'], - ['mobile_test_2/status/324619691039543297', 'BIFH45vCUAAQecj'] + #['mobile_test_2/status/324619691039543297', 'BIFH45vCUAAQecj'] ] gif = [