diff --git a/README.md b/README.md index 4f8235d..d1b8fc9 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,10 @@ XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscW - Archiving tweets/profiles - Developer API +## New Features + +- Likes tab + ## Resources The wiki contains @@ -99,6 +103,12 @@ $ nimble md $ cp nitter.example.conf nitter.conf ``` +Edit `twitter_oauth.sh` with your Twitter account name and password. + +``` +$ ./twitter_oauth.sh | tee -a guest_accounts.jsonl +``` + Set your hostname, port, HMAC key, https (must be correct for cookies), and Redis info in `nitter.conf`. To run Redis, either run `redis-server --daemonize yes`, or `systemctl enable --now redis` (or diff --git a/nitter.example.conf b/nitter.example.conf index ffe7a3b..f0b4214 100644 --- a/nitter.example.conf +++ b/nitter.example.conf @@ -33,9 +33,6 @@ tokenCount = 10 # always at least `tokenCount` usable tokens. only increase this if you receive # major bursts all the time and don't have a rate limiting setup via e.g. nginx -#cookieHeader = "ct0=XXXXXXXXXXXXXXXXX; auth_token=XXXXXXXXXXXXXX" # authentication cookie of a logged in account, required for the likes tab and NSFW content -#xCsrfToken = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" # required for the likes tab and NSFW content - # Change default preferences here, see src/prefs_impl.nim for a complete list [Preferences] theme = "Nitter" diff --git a/screenshot.png b/screenshot.png index 650e326..0cb8815 100644 Binary files a/screenshot.png and b/screenshot.png differ diff --git a/src/api.nim b/src/api.nim index d0bfd8c..ef60812 100644 --- a/src/api.nim +++ b/src/api.nim @@ -71,10 +71,20 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} proc getFavorites*(id: string; cfg: Config; after=""): Future[Profile] {.async.} = if id.len == 0: return - let - ps = genParams({"userId": id}, after) - url = consts.favorites / (id & ".json") ? ps - result = parseTimeline(await fetch(url, Api.favorites), after) + var + variables = %*{ + "userId": id, + "includePromotedContent":false, + "withClientEventToken":false, + "withBirdwatchNotes":false, + "withVoice":true, + "withV2Timeline":false + } + if after.len > 0: + variables["cursor"] = % after + let + url = consts.favorites ? {"variables": $variables, "features": gqlFeatures} + result = parseGraphTimeline(await fetch(url, Api.favorites), after) proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} = if id.len == 0: return diff --git a/src/apiutils.nim b/src/apiutils.nim index 22e75c9..774dcb5 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -49,7 +49,7 @@ proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string = proc genHeaders*(url, oauthToken, oauthTokenSecret: string): HttpHeaders = let header = getOauthHeader(url, oauthToken, oauthTokenSecret) - + result = newHttpHeaders({ "connection": "keep-alive", "authorization": header, @@ -149,11 +149,6 @@ template retry(bod) = proc fetch*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[JsonNode] {.async.} = - if len(cfg.cookieHeader) != 0: - additional_headers.add("Cookie", cfg.cookieHeader) - if len(cfg.xCsrfToken) != 0: - additional_headers.add("x-csrf-token", cfg.xCsrfToken) - retry: var body: string fetchImpl(body, additional_headers): diff --git a/src/config.nim b/src/config.nim index fe4aba5..2c216a2 100644 --- a/src/config.nim +++ b/src/config.nim @@ -41,9 +41,7 @@ proc getConfig*(path: string): (Config, parseCfg.Config) = enableRss: cfg.get("Config", "enableRSS", true), enableDebug: cfg.get("Config", "enableDebug", false), proxy: cfg.get("Config", "proxy", ""), - proxyAuth: cfg.get("Config", "proxyAuth", ""), - cookieHeader: cfg.get("Config", "cookieHeader", ""), - xCsrfToken: cfg.get("Config", "xCsrfToken", "") + proxyAuth: cfg.get("Config", "proxyAuth", "") ) return (conf, cfg) diff --git a/src/consts.nim b/src/consts.nim index 7d79c06..0a24469 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -11,7 +11,6 @@ const photoRail* = api / "1.1/statuses/media_timeline.json" timelineApi = api / "2/timeline" - favorites* = timelineApi / "favorites" graphql = api / "graphql" graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery" @@ -30,6 +29,7 @@ const graphRetweeters* = graphql / "RCR9gqwYD1NEgi9FWzA50A/Retweeters" graphFollowers* = graphql / "EAqBhgcGr_qPOzhS4Q3scQ/Followers" graphFollowing* = graphql / "JPZiqKjET7_M1r5Tlr8pyA/Following" + favorites* = graphql / "eSSNbhECHHWWALkkQq-YTA/Likes" timelineParams* = { "include_can_media_tag": "1", @@ -50,6 +50,7 @@ const gqlFeatures* = """{ "android_graphql_skip_api_media_color_palette": false, "blue_business_profile_image_shape_enabled": false, + "c9s_tweet_anatomy_moderator_badge_enabled": false, "creator_subscriptions_subscription_count_enabled": false, "creator_subscriptions_tweet_preview_api_enabled": true, "freedom_of_speech_not_reach_fetch_enabled": false, @@ -71,6 +72,7 @@ const "responsive_web_twitter_article_tweet_consumption_enabled": false, "responsive_web_twitter_blue_verified_badge_is_enabled": true, "rweb_lists_timeline_redesign_enabled": true, + "rweb_video_timestamps_enabled": true, "spaces_2022_h2_clipping": true, "spaces_2022_h2_spaces_communities": true, "standardized_nudges_misinfo": false, diff --git a/src/parser.nim b/src/parser.nim index fac0bd6..95a1fbc 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -33,6 +33,7 @@ proc parseGraphUser(js: JsonNode): User = var user = js{"user_result", "result"} if user.isNull: user = ? js{"user_results", "result"} + result = parseUser(user{"legacy"}) if result.verifiedType == VerifiedType.none and user{"is_blue_verified"}.getBool(false): @@ -534,7 +535,8 @@ proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile = let instructions = if root == "list": ? js{"data", "list", "timeline_response", "timeline", "instructions"} - else: ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"} + elif root == "user": ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"} + else: ? js{"data", "user", "result", "timeline", "timeline", "instructions"} if instructions.len == 0: return @@ -554,6 +556,21 @@ proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile = result.tweets.content.add thread.content elif entryId.startsWith("cursor-bottom"): result.tweets.bottom = e{"content", "value"}.getStr + # TODO cleanup + if i{"type"}.getStr == "TimelineAddEntries": + for e in i{"entries"}: + let entryId = e{"entryId"}.getStr + if entryId.startsWith("tweet"): + with tweetResult, e{"content", "itemContent", "tweet_results", "result"}: + let tweet = parseGraphTweet(tweetResult, false) + if not tweet.available: + tweet.id = parseBiggestInt(entryId.getId()) + result.tweets.content.add tweet + elif "-conversation-" in entryId or entryId.startsWith("homeConversation"): + let (thread, self) = parseGraphThread(e) + result.tweets.content.add thread.content + elif entryId.startsWith("cursor-bottom"): + result.tweets.bottom = e{"content", "value"}.getStr if after.len == 0 and i{"__typename"}.getStr == "TimelinePinEntry": with tweetResult, i{"entry", "content", "content", "tweetResult", "result"}: let tweet = parseGraphTweet(tweetResult, false) diff --git a/src/types.nim b/src/types.nim index 5e58716..a99aed5 100644 --- a/src/types.nim +++ b/src/types.nim @@ -282,9 +282,7 @@ type redisConns*: int redisMaxConns*: int redisPassword*: string - - cookieHeader*: string - xCsrfToken*: string + redisDb*: int Rss* = object feed*, cursor*: string diff --git a/src/views/search.nim b/src/views/search.nim index 8e797b7..0e5e808 100644 --- a/src/views/search.nim +++ b/src/views/search.nim @@ -38,9 +38,8 @@ proc renderProfileTabs*(query: Query; username: string; cfg: Config): VNode = a(href=(link & "/with_replies")): text "Tweets & Replies" li(class=query.getTabClass(media)): a(href=(link & "/media")): text "Media" - if len(cfg.xCsrfToken) != 0 and len(cfg.cookieHeader) != 0: - li(class=query.getTabClass(favorites)): - a(href=(link & "/favorites")): text "Likes" + li(class=query.getTabClass(favorites)): + a(href=(link & "/favorites")): text "Likes" li(class=query.getTabClass(tweets)): a(href=(link & "/search")): text "Search" diff --git a/twitter_oauth.sh b/twitter_oauth.sh new file mode 100644 index 0000000..f7cfba0 --- /dev/null +++ b/twitter_oauth.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Grab oauth token for use with Nitter (requires Twitter account). +# results: {"oauth_token":"xxxxxxxxxx-xxxxxxxxx","oauth_token_secret":"xxxxxxxxxxxxxxxxxxxxx"} + +username="" +password="" + +if [[ -z "$username" || -z "$password" ]]; then + echo "needs username and password" + exit 1 +fi + +bearer_token='AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F' +guest_token=$(curl -s -XPOST https://api.twitter.com/1.1/guest/activate.json -H "Authorization: Bearer ${bearer_token}" | jq -r '.guest_token') +base_url='https://api.twitter.com/1.1/onboarding/task.json' +header=(-H "Authorization: Bearer ${bearer_token}" -H "User-Agent: TwitterAndroid/10.21.1" -H "Content-Type: application/json" -H "X-Guest-Token: ${guest_token}") + +# start flow +flow_1=$(curl -si -XPOST "${base_url}?flow_name=login" "${header[@]}") + +# get 'att', now needed in headers, and 'flow_token' from flow_1 +att=$(sed -En 's/^att: (.*)\r/\1/p' <<< "${flow_1}") +flow_token=$(sed -n '$p' <<< "${flow_1}" | jq -r .flow_token) + +# username +token_2=$(curl -s -XPOST "${base_url}" -H "att: ${att}" "${header[@]}" \ + -d '{"flow_token":"'"${flow_token}"'","subtask_inputs":[{"subtask_id":"LoginEnterUserIdentifierSSO","settings_list":{"setting_responses":[{"key":"user_identifier","response_data":{"text_data":{"result":"'"${username}"'"}}}],"link":"next_link"}}]}' | jq -r .flow_token) + +# password +token_3=$(curl -s -XPOST "${base_url}" -H "att: ${att}" "${header[@]}" \ + -d '{"flow_token":"'"${token_2}"'","subtask_inputs":[{"enter_password":{"password":"'"${password}"'","link":"next_link"},"subtask_id":"LoginEnterPassword"}]}' | jq -r .flow_token) + +# finally print oauth_token and secret +curl -s -XPOST "${base_url}" -H "att: ${att}" "${header[@]}" \ + -d '{"flow_token":"'"${token_3}"'","subtask_inputs":[{"check_logged_in_account":{"link":"AccountDuplicationCheck_false"},"subtask_id":"AccountDuplicationCheck"}]}' | \ + jq -c '.subtasks[0]|if(.open_account) then {oauth_token: .open_account.oauth_token, oauth_token_secret: .open_account.oauth_token_secret} else empty end' \ No newline at end of file