From f2443a0da2fc3532b18278d3db9dde66d34a7f4a Mon Sep 17 00:00:00 2001 From: taskylizard <75871323+taskylizard@users.noreply.github.com> Date: Sat, 24 Feb 2024 13:36:22 +0000 Subject: [PATCH 01/12] add my fork changes --- Dockerfile.arm64 | 25 ------ docker-compose.yml | 17 +++- src/auth.nim | 2 +- src/nitter.nim | 3 +- src/routes/twitter_api.nim | 171 +++++++++++++++++++++++++++++++++++++ src/views/general.nim | 2 - src/views/profile.nim | 2 +- 7 files changed, 188 insertions(+), 34 deletions(-) delete mode 100644 Dockerfile.arm64 create mode 100644 src/routes/twitter_api.nim diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 deleted file mode 100644 index 70024b2..0000000 --- a/Dockerfile.arm64 +++ /dev/null @@ -1,25 +0,0 @@ -FROM alpine:3.18 as nim -LABEL maintainer="setenforce@protonmail.com" - -RUN apk --no-cache add libsass-dev pcre gcc git libc-dev "nim=1.6.14-r0" "nimble=0.13.1-r2" - -WORKDIR /src/nitter - -COPY nitter.nimble . -RUN nimble install -y --depsOnly - -COPY . . -RUN nimble build -d:danger -d:lto -d:strip \ - && nimble scss \ - && nimble md - -FROM alpine:3.18 -WORKDIR /src/ -RUN apk --no-cache add pcre ca-certificates openssl1.1-compat -COPY --from=nim /src/nitter/nitter ./ -COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf -COPY --from=nim /src/nitter/public ./public -EXPOSE 8080 -RUN adduser -h /src/ -D -s /bin/sh nitter -USER nitter -CMD ./nitter diff --git a/docker-compose.yml b/docker-compose.yml index ec8ade5..9e2d996 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,14 +1,19 @@ version: "3" -services: - +networks: nitter: - image: zedeus/nitter:latest + +services: + nitter: + build: . container_name: nitter + hostname: nitter ports: - - "127.0.0.1:8080:8080" # Replace with "8080:8080" if you don't use a reverse proxy + - "8002:8080" # Replace with "8080:8080" if you don't use a reverse proxy volumes: - ./nitter.conf:/src/nitter.conf:Z,ro + - ./guest_accounts.json:/src/guest_accounts.json:Z,ro + - ./public/.twitterminator.txt:/src/public/.twitterminator.txt:Z,ro depends_on: - nitter-redis restart: unless-stopped @@ -23,6 +28,8 @@ services: - no-new-privileges:true cap_drop: - ALL + networks: + - nitter nitter-redis: image: redis:6-alpine @@ -42,6 +49,8 @@ services: - no-new-privileges:true cap_drop: - ALL + networks: + - nitter volumes: nitter-redis: diff --git a/src/auth.nim b/src/auth.nim index b288c50..de1b1d8 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -202,7 +202,7 @@ proc initAccountPool*(cfg: Config; path: string) = quit 1 let accountsPrePurge = accountPool.len - accountPool.keepItIf(not it.hasExpired) + #accountPool.keepItIf(not it.hasExpired) log "Successfully added ", accountPool.len, " valid accounts." if accountsPrePurge > accountPool.len: diff --git a/src/nitter.nim b/src/nitter.nim index dfc1dfd..744eff9 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -10,7 +10,7 @@ import types, config, prefs, formatters, redis_cache, http_pool, auth import views/[general, about] import routes/[ preferences, timeline, status, media, search, rss, list, debug, - unsupported, embed, resolver, router_utils] + twitter_api, unsupported, embed, resolver, router_utils] const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances" const issuesUrl = "https://github.com/zedeus/nitter/issues" @@ -53,6 +53,7 @@ createSearchRouter(cfg) createMediaRouter(cfg) createEmbedRouter(cfg) createRssRouter(cfg) +createTwitterApiRouter(cfg) createDebugRouter(cfg) settings: diff --git a/src/routes/twitter_api.nim b/src/routes/twitter_api.nim new file mode 100644 index 0000000..0b8eef3 --- /dev/null +++ b/src/routes/twitter_api.nim @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: AGPL-3.0-only + +import json, asyncdispatch, options, uri +import times +import jester +import router_utils +import ".."/[types, api, apiutils, query, consts] +import httpclient, strutils +import sequtils + +export api + +proc videoToJson*(t: Video): JsonNode = + result = newJObject() + result["durationMs"] = %t.durationMs + result["url"] = %t.url + result["thumb"] = %t.thumb + result["views"] = %t.views + result["available"] = %t.available + result["reason"] = %t.reason + result["title"] = %t.title + result["description"] = %t.description + # result["playbackType"] = %t.playbackType + # result["variants"] = %t.variants + # playbackType*: VideoType + # variants*: seq[VideoVariant] + +proc tweetToJson*(t: Tweet): JsonNode = + result = newJObject() + result["id"] = %t.id + result["threadId"] = %t.threadId + result["replyId"] = %t.replyId + result["user"] = %*{ "username": t.user.username } + result["text"] = %t.text + result["time"] = newJString(times.format(t.time, "yyyy-MM-dd'T'HH:mm:ss")) + result["reply"] = %t.reply + result["pinned"] = %t.pinned + result["hasThread"] = %t.hasThread + result["available"] = %t.available + result["tombstone"] = %t.tombstone + result["location"] = %t.location + result["source"] = %t.source + # result["stats"] = toJson(t.stats) # Define conversion for TweetStats type + # result["retweet"] = t.retweet.map(toJson) # Define conversion for Tweet type + # result["attribution"] = t.attribution.map(toJson) # Define conversion for User type + # result["mediaTags"] = toJson(t.mediaTags) # Define conversion for seq[User] + # result["quote"] = t.quote.map(toJson) # Define conversion for Tweet type + # result["card"] = t.card.map(toJson) # Define conversion for Card type + # result["poll"] = t.poll.map(toJson) # Define conversion for Poll type + # result["gif"] = t.gif.map(toJson) # Define conversion for Gif type + # result["video"] = videoToJson(t.video.get()) + result["photos"] = %t.photos + +proc getUserProfileJson*(username: string): Future[JsonNode] {.async.} = + let user: User = await getGraphUser(username) + let response: JsonNode = %*{ + "id": user.id, + "username": user.username + } + result = response + +proc getUserTweetsJson*(id: string): Future[JsonNode] {.async.} = + let tweetsGraph = await getGraphUserTweets(id, TimelineKind.tweets) + let repliesGraph = await getGraphUserTweets(id, TimelineKind.replies) + let mediaGraph = await getGraphUserTweets(id, TimelineKind.media) + + let tweetsContent = tweetsGraph.tweets.content[0] + let tweetsJson = tweetsContent.map(tweetToJson) + + let repliesContent = repliesGraph.tweets.content[0] + let repliesJson = repliesContent.map(tweetToJson) + + let mediaContent = mediaGraph.tweets.content[0] + let mediaJson = mediaContent.map(tweetToJson) + + let response: JsonNode = %*{ + "tweets": %tweetsJson, + "replies": %repliesJson, + "media": %mediaJson + } + + result = response + +proc searchTimeline*(query: Query; after=""): Future[string] {.async.} = + let q = genQueryParam(query) + var + variables = %*{ + "rawQuery": q, + "count": 20, + "product": "Latest", + "withDownvotePerspective": false, + "withReactionsMetadata": false, + "withReactionsPerspective": false + } + if after.len > 0: + variables["cursor"] = % after + let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures} + result = await fetchRaw(url, Api.search) + +proc getUserTweets*(id: string; after=""): Future[string] {.async.} = + if id.len == 0: return + let + cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + variables = userTweetsVariables % [id, cursor] + params = {"variables": variables, "features": gqlFeatures} + result = await fetchRaw(graphUserTweets ? params, Api.userTweets) + +proc getUserReplies*(id: string; after=""): Future[string] {.async.} = + if id.len == 0: return + let + cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + variables = userTweetsVariables % [id, cursor] + params = {"variables": variables, "features": gqlFeatures} + result = await fetchRaw(graphUserTweets ? params, Api.userTweetsAndReplies) + +proc getUserMedia*(id: string; after=""): Future[string] {.async.} = + if id.len == 0: return + let + cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + variables = userTweetsVariables % [id, cursor] + params = {"variables": variables, "features": gqlFeatures} + result = await fetchRaw(graphUserTweets ? params, Api.userMedia) + +proc getTweetById*(id: string; after=""): Future[string] {.async.} = + let + cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + variables = tweetVariables % [id, cursor] + params = {"variables": variables, "features": gqlFeatures} + result = await fetchRaw(graphTweet ? params, Api.tweetDetail) + +proc createTwitterApiRouter*(cfg: Config) = + router api: + get "/api/echo": + resp Http200, {"Content-Type": "text/html"}, "hello, world!" + + get "/api/user/@username": + let username = @"username" + let response = await getUserProfileJson(username) + respJson response + + # get "/api/user/@id/tweets": + # let id = @"id" + # let response = await getUserTweetsJson(id) + # respJson response + + get "/api/user/@username/timeline": + let username = @"username" + let query = Query(fromUser: @[username]) + let response = await searchTimeline(query) + resp Http200, { "Content-Type": "application/json" }, response + + get "/api/user/@id/tweets": + let id = @"id" + let after = getCursor() + let response = await getUserTweets(id, after) + resp Http200, { "Content-Type": "application/json" }, response + + get "/api/user/@id/replies": + let id = @"id" + let response = await getUserReplies(id) + resp Http200, { "Content-Type": "application/json" }, response + + get "/api/user/@id/media": + let id = @"id" + let response = await getUserMedia(id) + resp Http200, { "Content-Type": "application/json" }, response + + get "/api/tweet/@id": + let id = @"id" + let response = await getTweetById(id) + resp Http200, { "Content-Type": "application/json" }, response diff --git a/src/views/general.nim b/src/views/general.nim index 5ba40a3..35efb0b 100644 --- a/src/views/general.nim +++ b/src/views/general.nim @@ -32,8 +32,6 @@ proc renderNavbar(cfg: Config; req: Request; rss, canonical: string): VNode = if cfg.enableRss and rss.len > 0: icon "rss-feed", title="RSS Feed", href=rss icon "bird", title="Open in Twitter", href=canonical - a(href="https://liberapay.com/zedeus"): verbatim lp - icon "info", title="About", href="/about" icon "cog", title="Preferences", href=("/settings?referer=" & encodeUrl(path)) proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; diff --git a/src/views/profile.nim b/src/views/profile.nim index 2b2e410..8f67f5a 100644 --- a/src/views/profile.nim +++ b/src/views/profile.nim @@ -13,7 +13,7 @@ proc renderStat(num: int; class: string; text=""): VNode = text insertSep($num, ',') proc renderUserCard*(user: User; prefs: Prefs): VNode = - buildHtml(tdiv(class="profile-card")): + buildHtml(tdiv(class="profile-card", "data-profile-id" = $user.id)): tdiv(class="profile-card-info"): let url = getPicUrl(user.getUserPic()) From 0321285903e7d7222627aa8b36dd88587b43413f Mon Sep 17 00:00:00 2001 From: taskylizard <75871323+taskylizard@users.noreply.github.com> Date: Sat, 24 Feb 2024 13:46:06 +0000 Subject: [PATCH 02/12] feat: add catppuccin themes --- public/css/themes/catppuccin_frappe.css | 82 ++++++++++++++++++++++ public/css/themes/catppuccin_latte.css | 82 ++++++++++++++++++++++ public/css/themes/catppuccin_macchiato.css | 82 ++++++++++++++++++++++ public/css/themes/catppuccin_mocha.css | 82 ++++++++++++++++++++++ 4 files changed, 328 insertions(+) create mode 100644 public/css/themes/catppuccin_frappe.css create mode 100644 public/css/themes/catppuccin_latte.css create mode 100644 public/css/themes/catppuccin_macchiato.css create mode 100644 public/css/themes/catppuccin_mocha.css diff --git a/public/css/themes/catppuccin_frappe.css b/public/css/themes/catppuccin_frappe.css new file mode 100644 index 0000000..23d951e --- /dev/null +++ b/public/css/themes/catppuccin_frappe.css @@ -0,0 +1,82 @@ +body { + /* Catppuccin Frappe color palette */ + --rosewater: #f2d5cf; + --flamingo: #eebebe; + --pink: #f4b8e4; + --mauve: #ca9ee6; + --red: #e78284; + --maroon: #ea999c; + --peach: #ef9f76; + --yellow: #e5c890; + --green: #a6d189; + --teal: #81c8be; + --sky: #99d1db; + --sapphire: #85c1dc; + --blue: #8caaee; + --lavender: #babbf1; + --text: #c6d0f5; + --subtext1: #b5bfe2; + --subtext0: #a5adce; + --overlay2: #949cbb; + --overlay1: #838ba7; + --overlay0: #737994; + --surface2: #626880; + --surface1: #51576d; + --surface0: #414559; + --base: #303446; + --mantle: #292c3c; + --crust: #232634; + + /* Predefined colors */ + /* --bg_color: var(--mantle); */ + --bg_color: var(--base); + --fg_color: var(--text); + --fg_faded: var(--subtext1); + --fg_dark: var(--accent); + --fg_nav: var(--accent); + + /* --bg_panel: var(--crust); + --bg_elements: var(--base); + --bg_overlays: var(--base); */ + --bg_panel: var(--mantle); + --bg_elements: var(--crust); + --bg_overlays: var(--mantle); + --bg_hover: var(--crust); + + --grey: var(--subtext1); + --dark_grey: var(--overlay0); + --darker_grey: var(--base); + --darkest_grey: var(--mantle); + --border_grey: var(--base); + + --accent: var(--rosewater); + --accent_light: hsl(10, 56%, 96%); /* increase lightness by 5% */ + --accent_dark: hsl(10, 56%, 86%); /* decrease lightness by 5% */ + --accent_border: hsl(10, 56%, 86%); + + --play_button: var(--accent); + --play_button_hover: var(--accent); + + --more_replies_dots: hsl(10, 56%, 86%); + --error_red: var(--red); + + --verified_blue: var(--blue); + --icon_text: var(--text); + + --tab: var(--text); + --tab_selected: var(--accent); + + --profile_stat: var(--text); + + /* Catppuccin tweaks */ + /* background-color: var(--bg_color); + color: var(--fg_color); + line-height: 1.3; + margin: 0; */ + + /* Fix Poll Leader color */ + /* Text is illegible when using light accents otherwise */ + .poll-meter.leader { + color: var(--bg_color); + } +} diff --git a/public/css/themes/catppuccin_latte.css b/public/css/themes/catppuccin_latte.css new file mode 100644 index 0000000..be9101c --- /dev/null +++ b/public/css/themes/catppuccin_latte.css @@ -0,0 +1,82 @@ +body { + /* Catppuccin Latte color palette */ + --rosewater: #dc8a78; + --flamingo: #dd7878; + --pink: #ea76cb; + --mauve: #8839ef; + --red: #d20f39; + --maroon: #e64553; + --peach: #fe640b; + --yellow: #df8e1d; + --green: #40a02b; + --teal: #179299; + --sky: #04a5e5; + --sapphire: #209fb5; + --blue: #1e66f5; + --lavender: #7287fd; + --text: #4c4f69; + --subtext1: #5c5f77; + --subtext0: #6c6f85; + --overlay2: #7c7f93; + --overlay1: #8c8fa1; + --overlay0: #9ca0b0; + --surface2: #acb0be; + --surface1: #bcc0cc; + --surface0: #ccd0da; + --base: #eff1f5; + --mantle: #e6e9ef; + --crust: #dce0e8; + + /* Predefined colors */ + /* --bg_color: var(--mantle); */ + --bg_color: var(--base); + --fg_color: var(--text); + --fg_faded: var(--subtext1); + --fg_dark: var(--accent); + --fg_nav: var(--accent); + + /* --bg_panel: var(--crust); + --bg_elements: var(--base); + --bg_overlays: var(--base); */ + --bg_panel: var(--mantle); + --bg_elements: var(--crust); + --bg_overlays: var(--mantle); + --bg_hover: var(--crust); + + --grey: var(--subtext1); + --dark_grey: var(--overlay0); + --darker_grey: var(--base); + --darkest_grey: var(--mantle); + --border_grey: var(--base); + + --accent: var(--rosewater); + --accent_light: hsl(10, 56%, 96%); /* increase lightness by 5% */ + --accent_dark: hsl(10, 56%, 86%); /* decrease lightness by 5% */ + --accent_border: hsl(10, 56%, 86%); + + --play_button: var(--accent); + --play_button_hover: var(--accent); + + --more_replies_dots: hsl(10, 56%, 86%); + --error_red: var(--red); + + --verified_blue: var(--blue); + --icon_text: var(--text); + + --tab: var(--text); + --tab_selected: var(--accent); + + --profile_stat: var(--text); + + /* Catppuccin tweaks */ + /* background-color: var(--bg_color); + color: var(--fg_color); + line-height: 1.3; + margin: 0; */ + + /* Fix Poll Leader color */ + /* Text is illegible when using light accents otherwise */ + .poll-meter.leader { + color: var(--bg_color); + } +} diff --git a/public/css/themes/catppuccin_macchiato.css b/public/css/themes/catppuccin_macchiato.css new file mode 100644 index 0000000..c1cb1bb --- /dev/null +++ b/public/css/themes/catppuccin_macchiato.css @@ -0,0 +1,82 @@ +body { + /* Catppuccin Macchiato color palette */ + --rosewater: #f4dbd6; + --flamingo: #f0c6c6; + --pink: #f5bde6; + --mauve: #c6a0f6; + --red: #ed8796; + --maroon: #ee99a0; + --peach: #f5a97f; + --yellow: #eed49f; + --green: #a6da95; + --teal: #8bd5ca; + --sky: #91d7e3; + --sapphire: #7dc4e4; + --blue: #8aadf4; + --lavender: #b7bdf8; + --text: #cad3f5; + --subtext1: #b8c0e0; + --subtext0: #a5adcb; + --overlay2: #939ab7; + --overlay1: #8087a2; + --overlay0: #6e738d; + --surface2: #5b6078; + --surface1: #494d64; + --surface0: #363a4f; + --base: #24273a; + --mantle: #1e2030; + --crust: #181926; + + /* Predefined colors */ + /* --bg_color: var(--mantle); */ + --bg_color: var(--base); + --fg_color: var(--text); + --fg_faded: var(--subtext1); + --fg_dark: var(--accent); + --fg_nav: var(--accent); + + /* --bg_panel: var(--crust); + --bg_elements: var(--base); + --bg_overlays: var(--base); */ + --bg_panel: var(--mantle); + --bg_elements: var(--crust); + --bg_overlays: var(--mantle); + --bg_hover: var(--crust); + + --grey: var(--subtext1); + --dark_grey: var(--overlay0); + --darker_grey: var(--base); + --darkest_grey: var(--mantle); + --border_grey: var(--base); + + --accent: var(--rosewater); + --accent_light: hsl(10, 56%, 96%); /* increase lightness by 5% */ + --accent_dark: hsl(10, 56%, 86%); /* decrease lightness by 5% */ + --accent_border: hsl(10, 56%, 86%); + + --play_button: var(--accent); + --play_button_hover: var(--accent); + + --more_replies_dots: hsl(10, 56%, 86%); + --error_red: var(--red); + + --verified_blue: var(--blue); + --icon_text: var(--text); + + --tab: var(--text); + --tab_selected: var(--accent); + + --profile_stat: var(--text); + + /* Catppuccin tweaks */ + /* background-color: var(--bg_color); + color: var(--fg_color); + line-height: 1.3; + margin: 0; */ + + /* Fix Poll Leader color */ + /* Text is illegible when using light accents otherwise */ + .poll-meter.leader { + color: var(--bg_color); + } +} diff --git a/public/css/themes/catppuccin_mocha.css b/public/css/themes/catppuccin_mocha.css new file mode 100644 index 0000000..c2bfa87 --- /dev/null +++ b/public/css/themes/catppuccin_mocha.css @@ -0,0 +1,82 @@ +body { + /* Catppuccin Mocha color palette */ + --rosewater: #f5e0dc; + --flamingo: #f2cdcd; + --pink: #f5c2e7; + --mauve: #cba6f7; + --red: #f38ba8; + --maroon: #eba0ac; + --peach: #fab387; + --yellow: #f9e2af; + --green: #a6e3a1; + --teal: #94e2d5; + --sky: #89dceb; + --sapphire: #74c7ec; + --blue: #89b4fa; + --lavender: #b4befe; + --text: #cdd6f4; + --subtext1: #bac2de; + --subtext0: #a6adc8; + --overlay2: #9399b2; + --overlay1: #7f849c; + --overlay0: #6c7086; + --surface2: #585b70; + --surface1: #45475a; + --surface0: #313244; + --base: #1e1e2e; + --mantle: #181825; + --crust: #11111b; + + /* Predefined colors */ + /* --bg_color: var(--mantle); */ + --bg_color: var(--base); + --fg_color: var(--text); + --fg_faded: var(--subtext1); + --fg_dark: var(--accent); + --fg_nav: var(--accent); + + /* --bg_panel: var(--crust); + --bg_elements: var(--base); + --bg_overlays: var(--base); */ + --bg_panel: var(--mantle); + --bg_elements: var(--crust); + --bg_overlays: var(--mantle); + --bg_hover: var(--crust); + + --grey: var(--subtext1); + --dark_grey: var(--overlay0); + --darker_grey: var(--base); + --darkest_grey: var(--mantle); + --border_grey: var(--base); + + --accent: var(--rosewater); + --accent_light: hsl(10, 56%, 96%); /* increase lightness by 5% */ + --accent_dark: hsl(10, 56%, 86%); /* decrease lightness by 5% */ + --accent_border: hsl(10, 56%, 86%); + + --play_button: var(--accent); + --play_button_hover: var(--accent); + + --more_replies_dots: hsl(10, 56%, 86%); + --error_red: var(--red); + + --verified_blue: var(--blue); + --icon_text: var(--text); + + --tab: var(--text); + --tab_selected: var(--accent); + + --profile_stat: var(--text); + + /* Catppuccin tweaks */ + /* background-color: var(--bg_color); + color: var(--fg_color); + line-height: 1.3; + margin: 0; */ + + /* Fix Poll Leader color */ + /* Text is illegible when using light accents otherwise */ + .poll-meter.leader { + color: var(--bg_color); + } +} From 3855af14f14032e4b0f298b22befb6231afee4e1 Mon Sep 17 00:00:00 2001 From: taskylizard <75871323+taskylizard@users.noreply.github.com> Date: Sat, 24 Feb 2024 14:27:18 +0000 Subject: [PATCH 03/12] feat(themes): wip tokyonight thene --- docker-compose.yml | 1 - public/css/themes/tokyonight.css | 81 ++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 public/css/themes/tokyonight.css diff --git a/docker-compose.yml b/docker-compose.yml index 9e2d996..207914e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,6 @@ services: volumes: - ./nitter.conf:/src/nitter.conf:Z,ro - ./guest_accounts.json:/src/guest_accounts.json:Z,ro - - ./public/.twitterminator.txt:/src/public/.twitterminator.txt:Z,ro depends_on: - nitter-redis restart: unless-stopped diff --git a/public/css/themes/tokyonight.css b/public/css/themes/tokyonight.css new file mode 100644 index 0000000..c5df1cf --- /dev/null +++ b/public/css/themes/tokyonight.css @@ -0,0 +1,81 @@ +body { + /* Catppuccin Mocha color palette */ + --rosewater: #f5e0dc; + --flamingo: #f2cdcd; + --pink: #f5c2e7; + --mauve: #cba6f7; + --red: #f38ba8; + --maroon: #eba0ac; + --peach: #fab387; + --yellow: #f9e2af; + --green: #a6e3a1; + --teal: #94e2d5; + --sky: #89dceb; + --sapphire: #74c7ec; + --blue: #89b4fa; + --lavender: #b4befe; + --text: #cdd6f4; + --subtext1: #bac2de; + --subtext0: #a6adc8; + --overlay2: #9399b2; + --overlay1: #7f849c; + --overlay0: #6c7086; + --surface2: #585b70; + --surface1: #45475a; + --surface0: #313244; + --base: #1e1e2e; + + /* Predefined colors */ + --bg_color: #1a1b26; + --fg_color: #c0caf5; + --fg_faded: #a9b1d6; + --fg_dark: var(--accent); + --fg_nav: var(--accent); + + --bg_panel: #171722; + --bg_elements: #16161e; + --bg_overlays: #171722; + --bg_hover: #16161e; + + --grey: var(--subtext1); + --dark_grey: var(--overlay0); + --darker_grey: var(--base); + --darkest_grey: #171722; + --border_grey: var(--base); + + --accent: #7aa2f7; + --accent_light: #7aa2f7; + --accent_dark: #7aa2f7; + --accent_border: #7aa2f7; + + --play_button: var(--accent); + --play_button_hover: var(--accent); + + --more_replies_dots: #cba6f7; + --error_red: var(--red); + + --verified_blue: var(--blue); + --icon_text: var(--text); + + --tab: var(--text); + --tab_selected: var(--accent); + + --profile_stat: var(--text); + + /* Catppuccin tweaks */ + /* background-color: var(--bg_color); + color: var(--fg_color); + line-height: 1.3; + margin: 0; */ + + /* Fix Poll Leader color */ + /* Text is illegible when using light accents otherwise */ + .poll-meter.leader { + color: var(--bg_color); + } + + ::selection { + background-color: #515c7e40; + color: #a9b1d6; + } +} From 4584932e4f0d03e0e9b212ef2b8b38fad4a6ffcf Mon Sep 17 00:00:00 2001 From: taskylizard <75871323+taskylizard@users.noreply.github.com> Date: Sun, 19 May 2024 05:39:08 +0000 Subject: [PATCH 04/12] Pull from https://github.com/PrivacyDevel/nitter/pull/50 --- src/api.nim | 53 ++++++++++++ src/apiutils.nim | 29 +++++-- src/config.nim | 5 ++ src/consts.nim | 21 +++++ src/formatters.nim | 2 - src/nitter.nim | 7 +- src/parser.nim | 174 +++++++++++++++++++++++++++++++++++++- src/query.nim | 7 ++ src/routes/rss.nim | 5 +- src/routes/status.nim | 25 +++++- src/routes/timeline.nim | 63 ++++++++------ src/sass/tweet/_base.scss | 1 + src/types.nim | 9 +- src/views/general.nim | 4 +- src/views/profile.nim | 16 ++-- src/views/search.nim | 13 ++- src/views/tweet.nim | 21 +++-- 17 files changed, 388 insertions(+), 67 deletions(-) diff --git a/src/api.nim b/src/api.nim index d6a4564..ef60812 100644 --- a/src/api.nim +++ b/src/api.nim @@ -69,6 +69,23 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures} result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after) +proc getFavorites*(id: string; cfg: Config; after=""): Future[Profile] {.async.} = + if id.len == 0: return + var + variables = %*{ + "userId": id, + "includePromotedContent":false, + "withClientEventToken":false, + "withBirdwatchNotes":false, + "withVoice":true, + "withV2Timeline":false + } + if after.len > 0: + variables["cursor"] = % after + let + url = consts.favorites ? {"variables": $variables, "features": gqlFeatures} + result = parseGraphTimeline(await fetch(url, Api.favorites), after) + proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} = if id.len == 0: return let @@ -86,6 +103,42 @@ proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} = js = await fetch(graphTweet ? params, Api.tweetDetail) result = parseGraphConversation(js, id) +proc getGraphFavoriters*(id: string; after=""): Future[UsersTimeline] {.async.} = + if id.len == 0: return + let + cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + variables = reactorsVariables % [id, cursor] + params = {"variables": variables, "features": gqlFeatures} + js = await fetch(graphFavoriters ? params, Api.favoriters) + result = parseGraphFavoritersTimeline(js, id) + +proc getGraphRetweeters*(id: string; after=""): Future[UsersTimeline] {.async.} = + if id.len == 0: return + let + cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + variables = reactorsVariables % [id, cursor] + params = {"variables": variables, "features": gqlFeatures} + js = await fetch(graphRetweeters ? params, Api.retweeters) + result = parseGraphRetweetersTimeline(js, id) + +proc getGraphFollowing*(id: string; after=""): Future[UsersTimeline] {.async.} = + if id.len == 0: return + let + cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + variables = followVariables % [id, cursor] + params = {"variables": variables, "features": gqlFeatures} + js = await fetch(graphFollowing ? params, Api.following) + result = parseGraphFollowTimeline(js, id) + +proc getGraphFollowers*(id: string; after=""): Future[UsersTimeline] {.async.} = + if id.len == 0: return + let + cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + variables = followVariables % [id, cursor] + params = {"variables": variables, "features": gqlFeatures} + js = await fetch(graphFollowers ? params, Api.followers) + result = parseGraphFollowTimeline(js, id) + proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} = result = (await getGraphTweet(id, after)).replies result.beginning = after.len == 0 diff --git a/src/apiutils.nim b/src/apiutils.nim index 1ff05eb..774dcb5 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -3,6 +3,7 @@ import httpclient, asyncdispatch, options, strutils, uri, times, math, tables import jsony, packedjson, zippy, oauth1 import types, auth, consts, parserutils, http_pool import experimental/types/common +import config const rlRemaining = "x-rate-limit-remaining" @@ -48,7 +49,7 @@ proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string = proc genHeaders*(url, oauthToken, oauthTokenSecret: string): HttpHeaders = let header = getOauthHeader(url, oauthToken, oauthTokenSecret) - + result = newHttpHeaders({ "connection": "keep-alive", "authorization": header, @@ -61,7 +62,14 @@ proc genHeaders*(url, oauthToken, oauthTokenSecret: string): HttpHeaders = "DNT": "1" }) -template fetchImpl(result, fetchBody) {.dirty.} = +template updateAccount() = + if resp.headers.hasKey(rlRemaining): + let + remaining = parseInt(resp.headers[rlRemaining]) + reset = parseInt(resp.headers[rlReset]) + account.setRateLimit(api, remaining, reset) + +template fetchImpl(result, additional_headers, fetchBody) {.dirty.} = once: pool = HttpPool() @@ -72,13 +80,19 @@ template fetchImpl(result, fetchBody) {.dirty.} = try: var resp: AsyncResponse - pool.use(genHeaders($url, account.oauthToken, account.oauthSecret)): + var headers = genHeaders($url, account.oauthToken, account.oauthSecret) + for key, value in additional_headers.pairs(): + headers.add(key, value) + pool.use(headers): template getContent = resp = await c.get($url) result = await resp.body getContent() + if resp.status == $Http429: + raise rateLimitError() + if resp.status == $Http503: badClient = true raise newException(BadClientError, "Bad client") @@ -133,10 +147,11 @@ template retry(bod) = echo "[accounts] Rate limited, retrying ", api, " request..." bod -proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} = +proc fetch*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[JsonNode] {.async.} = + retry: var body: string - fetchImpl body: + fetchImpl(body, additional_headers): if body.startsWith('{') or body.startsWith('['): result = parseJson(body) else: @@ -149,9 +164,9 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} = invalidate(account) raise rateLimitError() -proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} = +proc fetchRaw*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[string] {.async.} = retry: - fetchImpl result: + fetchImpl(result, additional_headers): if not (result.startsWith('{') or result.startsWith('[')): echo resp.status, ": ", result, " --- url: ", url result.setLen(0) diff --git a/src/config.nim b/src/config.nim index 1b05ffe..2c216a2 100644 --- a/src/config.nim +++ b/src/config.nim @@ -1,6 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only import parsecfg except Config import types, strutils +from os import getEnv proc get*[T](config: parseCfg.Config; section, key: string; default: T): T = let val = config.getSectionValue(section, key) @@ -44,3 +45,7 @@ proc getConfig*(path: string): (Config, parseCfg.Config) = ) return (conf, cfg) + + +let configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf") +let (cfg*, fullCfg*) = getConfig(configPath) diff --git a/src/consts.nim b/src/consts.nim index e1c35e6..0a24469 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -10,6 +10,8 @@ const photoRail* = api / "1.1/statuses/media_timeline.json" + timelineApi = api / "2/timeline" + graphql = api / "graphql" graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery" graphUserById* = graphql / "oPppcargziU1uDQHAUmH-A/UserResultByIdQuery" @@ -23,6 +25,11 @@ const graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug" graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers" graphListTweets* = graphql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline" + graphFavoriters* = graphql / "mDc_nU8xGv0cLRWtTaIEug/Favoriters" + graphRetweeters* = graphql / "RCR9gqwYD1NEgi9FWzA50A/Retweeters" + graphFollowers* = graphql / "EAqBhgcGr_qPOzhS4Q3scQ/Followers" + graphFollowing* = graphql / "JPZiqKjET7_M1r5Tlr8pyA/Following" + favorites* = graphql / "eSSNbhECHHWWALkkQq-YTA/Likes" timelineParams* = { "include_can_media_tag": "1", @@ -43,6 +50,7 @@ const gqlFeatures* = """{ "android_graphql_skip_api_media_color_palette": false, "blue_business_profile_image_shape_enabled": false, + "c9s_tweet_anatomy_moderator_badge_enabled": false, "creator_subscriptions_subscription_count_enabled": false, "creator_subscriptions_tweet_preview_api_enabled": true, "freedom_of_speech_not_reach_fetch_enabled": false, @@ -64,6 +72,7 @@ const "responsive_web_twitter_article_tweet_consumption_enabled": false, "responsive_web_twitter_blue_verified_badge_is_enabled": true, "rweb_lists_timeline_redesign_enabled": true, + "rweb_video_timestamps_enabled": true, "spaces_2022_h2_clipping": true, "spaces_2022_h2_spaces_communities": true, "standardized_nudges_misinfo": false, @@ -114,3 +123,15 @@ const "rest_id": "$1", $2 "count": 20 }""" + + reactorsVariables* = """{ + "tweetId" : "$1", $2 + "count" : 20, + "includePromotedContent": false +}""" + + followVariables* = """{ + "userId" : "$1", $2 + "count" : 20, + "includePromotedContent": false +}""" diff --git a/src/formatters.nim b/src/formatters.nim index 8267f23..3630917 100644 --- a/src/formatters.nim +++ b/src/formatters.nim @@ -82,8 +82,6 @@ proc proxifyVideo*(manifest: string; proxy: bool): string = for line in manifest.splitLines: let url = if line.startsWith("#EXT-X-MAP:URI"): line[16 .. ^2] - elif line.startsWith("#EXT-X-MEDIA") and "URI=" in line: - line[line.find("URI=") + 5 .. -1 + line.find("\"", start= 5 + line.find("URI="))] else: line if url.startsWith('/'): let path = "https://video.twimg.com" & url diff --git a/src/nitter.nim b/src/nitter.nim index 744eff9..f976db2 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -1,5 +1,6 @@ # SPDX-License-Identifier: AGPL-3.0-only import asyncdispatch, strformat, logging +import config from net import Port from htmlgen import a from os import getEnv @@ -10,15 +11,12 @@ import types, config, prefs, formatters, redis_cache, http_pool, auth import views/[general, about] import routes/[ preferences, timeline, status, media, search, rss, list, debug, - twitter_api, unsupported, embed, resolver, router_utils] + unsupported, embed, resolver, router_utils] const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances" const issuesUrl = "https://github.com/zedeus/nitter/issues" let - configPath = getEnv("NITTER_CONF_FILE", "./nitter.conf") - (cfg, fullCfg) = getConfig(configPath) - accountsPath = getEnv("NITTER_ACCOUNTS_FILE", "./guest_accounts.json") initAccountPool(cfg, accountsPath) @@ -53,7 +51,6 @@ createSearchRouter(cfg) createMediaRouter(cfg) createEmbedRouter(cfg) createRssRouter(cfg) -createTwitterApiRouter(cfg) createDebugRouter(cfg) settings: diff --git a/src/parser.nim b/src/parser.nim index ec856a6..95a1fbc 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -3,6 +3,7 @@ import strutils, options, times, math import packedjson, packedjson/deserialiser import types, parserutils, utils import experimental/parser/unifiedcard +import std/tables proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet @@ -32,7 +33,8 @@ proc parseGraphUser(js: JsonNode): User = var user = js{"user_result", "result"} if user.isNull: user = ? js{"user_results", "result"} - result = parseUser(user{"legacy"}, user{"rest_id"}.getStr) + + result = parseUser(user{"legacy"}) if result.verifiedType == VerifiedType.none and user{"is_blue_verified"}.getBool(false): result.verifiedType = blue @@ -236,8 +238,11 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = # graphql with rt, js{"retweeted_status_result", "result"}: # needed due to weird edgecase where the actual tweet data isn't included - if "legacy" in rt: - result.retweet = some parseGraphTweet(rt) + var rt_tweet = rt + if "tweet" in rt: + rt_tweet = rt{"tweet"} + if "legacy" in rt_tweet: + result.retweet = some parseGraphTweet(rt_tweet) return if jsCard.kind != JNull: @@ -289,6 +294,121 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet = result.text.removeSuffix(" Learn more.") result.available = false +proc parseLegacyTweet(js: JsonNode): Tweet = + result = parseTweet(js, js{"card"}) + if not result.isNil and result.available: + result.user = parseUser(js{"user"}) + + if result.quote.isSome: + result.quote = some parseLegacyTweet(js{"quoted_status"}) + +proc parseTweetSearch*(js: JsonNode; after=""): Timeline = + result.beginning = after.len == 0 + + if js.kind == JNull or "modules" notin js or js{"modules"}.len == 0: + return + + for item in js{"modules"}: + with tweet, item{"status", "data"}: + let parsed = parseLegacyTweet(tweet) + + if parsed.retweet.isSome: + parsed.retweet = some parseLegacyTweet(tweet{"retweeted_status"}) + + result.content.add @[parsed] + + if result.content.len > 0: + result.bottom = $(result.content[^1][0].id - 1) + +proc finalizeTweet(global: GlobalObjects; id: string): Tweet = + let intId = if id.len > 0: parseBiggestInt(id) else: 0 + result = global.tweets.getOrDefault(id, Tweet(id: intId)) + + if result.quote.isSome: + let quote = get(result.quote).id + if $quote in global.tweets: + result.quote = some global.tweets[$quote] + else: + result.quote = some Tweet() + + if result.retweet.isSome: + let rt = get(result.retweet).id + if $rt in global.tweets: + result.retweet = some finalizeTweet(global, $rt) + else: + result.retweet = some Tweet() + +proc parsePin(js: JsonNode; global: GlobalObjects): Tweet = + let pin = js{"pinEntry", "entry", "entryId"}.getStr + if pin.len == 0: return + + let id = pin.getId + if id notin global.tweets: return + + global.tweets[id].pinned = true + return finalizeTweet(global, id) + +proc parseGlobalObjects(js: JsonNode): GlobalObjects = + result = GlobalObjects() + let + tweets = ? js{"globalObjects", "tweets"} + users = ? js{"globalObjects", "users"} + + for k, v in users: + result.users[k] = parseUser(v, k) + + for k, v in tweets: + var tweet = parseTweet(v, v{"card"}) + if tweet.user.id in result.users: + tweet.user = result.users[tweet.user.id] + result.tweets[k] = tweet + +proc parseInstructions(res: var Profile; global: GlobalObjects; js: JsonNode) = + if js.kind != JArray or js.len == 0: + return + + for i in js: + if res.tweets.beginning and i{"pinEntry"}.notNull: + with pin, parsePin(i, global): + res.pinned = some pin + + with r, i{"replaceEntry", "entry"}: + if "top" in r{"entryId"}.getStr: + res.tweets.top = r.getCursor + elif "bottom" in r{"entryId"}.getStr: + res.tweets.bottom = r.getCursor + +proc parseTimeline*(js: JsonNode; after=""): Profile = + result = Profile(tweets: Timeline(beginning: after.len == 0)) + let global = parseGlobalObjects(? js) + + let instructions = ? js{"timeline", "instructions"} + if instructions.len == 0: return + + result.parseInstructions(global, instructions) + + var entries: JsonNode + for i in instructions: + if "addEntries" in i: + entries = i{"addEntries", "entries"} + + for e in ? entries: + let entry = e{"entryId"}.getStr + if "tweet" in entry or entry.startsWith("sq-I-t") or "tombstone" in entry: + let tweet = finalizeTweet(global, e.getEntryId) + if not tweet.available: continue + result.tweets.content.add tweet + elif "cursor-top" in entry: + result.tweets.top = e.getCursor + elif "cursor-bottom" in entry: + result.tweets.bottom = e.getCursor + elif entry.startsWith("sq-cursor"): + with cursor, e{"content", "operation", "cursor"}: + if cursor{"cursorType"}.getStr == "Bottom": + result.tweets.bottom = cursor{"value"}.getStr + else: + result.tweets.top = cursor{"value"}.getStr + proc parsePhotoRail*(js: JsonNode): PhotoRail = with error, js{"error"}: if error.getStr == "Not authorized.": @@ -415,7 +535,8 @@ proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile = let instructions = if root == "list": ? js{"data", "list", "timeline_response", "timeline", "instructions"} - else: ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"} + elif root == "user": ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"} + else: ? js{"data", "user", "result", "timeline", "timeline", "instructions"} if instructions.len == 0: return @@ -435,6 +556,21 @@ proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile = result.tweets.content.add thread.content elif entryId.startsWith("cursor-bottom"): result.tweets.bottom = e{"content", "value"}.getStr + # TODO cleanup + if i{"type"}.getStr == "TimelineAddEntries": + for e in i{"entries"}: + let entryId = e{"entryId"}.getStr + if entryId.startsWith("tweet"): + with tweetResult, e{"content", "itemContent", "tweet_results", "result"}: + let tweet = parseGraphTweet(tweetResult, false) + if not tweet.available: + tweet.id = parseBiggestInt(entryId.getId()) + result.tweets.content.add tweet + elif "-conversation-" in entryId or entryId.startsWith("homeConversation"): + let (thread, self) = parseGraphThread(e) + result.tweets.content.add thread.content + elif entryId.startsWith("cursor-bottom"): + result.tweets.bottom = e{"content", "value"}.getStr if after.len == 0 and i{"__typename"}.getStr == "TimelinePinEntry": with tweetResult, i{"entry", "content", "content", "tweetResult", "result"}: let tweet = parseGraphTweet(tweetResult, false) @@ -445,6 +581,36 @@ proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile = tweet.id = parseBiggestInt(entryId) result.pinned = some tweet +proc parseGraphUsersTimeline(timeline: JsonNode; after=""): UsersTimeline = + result = UsersTimeline(beginning: after.len == 0) + + let instructions = ? timeline{"instructions"} + + if instructions.len == 0: + return + + for i in instructions: + if i{"type"}.getStr == "TimelineAddEntries": + for e in i{"entries"}: + let entryId = e{"entryId"}.getStr + if entryId.startsWith("user"): + with graphUser, e{"content", "itemContent"}: + let user = parseGraphUser(graphUser) + result.content.add user + elif entryId.startsWith("cursor-bottom"): + result.bottom = e{"content", "value"}.getStr + elif entryId.startsWith("cursor-top"): + result.top = e{"content", "value"}.getStr + +proc parseGraphFavoritersTimeline*(js: JsonNode; root: string; after=""): UsersTimeline = + return parseGraphUsersTimeline(js{"data", "favoriters_timeline", "timeline"}, after) + +proc parseGraphRetweetersTimeline*(js: JsonNode; root: string; after=""): UsersTimeline = + return parseGraphUsersTimeline(js{"data", "retweeters_timeline", "timeline"}, after) + +proc parseGraphFollowTimeline*(js: JsonNode; root: string; after=""): UsersTimeline = + return parseGraphUsersTimeline(js{"data", "user", "result", "timeline", "timeline"}, after) + proc parseGraphSearch*[T: User | Tweets](js: JsonNode; after=""): Result[T] = result = Result[T](beginning: after.len == 0) diff --git a/src/query.nim b/src/query.nim index 06e1da2..b5d79d9 100644 --- a/src/query.nim +++ b/src/query.nim @@ -40,6 +40,13 @@ proc getMediaQuery*(name: string): Query = sep: "OR" ) + +proc getFavoritesQuery*(name: string): Query = + Query( + kind: favorites, + fromUser: @[name] + ) + proc getReplyQuery*(name: string): Query = Query( kind: replies, diff --git a/src/routes/rss.nim b/src/routes/rss.nim index 447f4ad..0896536 100644 --- a/src/routes/rss.nim +++ b/src/routes/rss.nim @@ -23,7 +23,7 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async. names = getNames(name) if names.len == 1: - profile = await fetchProfile(after, query, skipRail=true, skipPinned=true) + profile = await fetchProfile(after, query, cfg, skipRail=true, skipPinned=true) else: var q = query q.fromUser = names @@ -102,7 +102,7 @@ proc createRssRouter*(cfg: Config) = get "/@name/@tab/rss": cond cfg.enableRss cond '.' notin @"name" - cond @"tab" in ["with_replies", "media", "search"] + cond @"tab" in ["with_replies", "media", "favorites", "search"] let name = @"name" tab = @"tab" @@ -110,6 +110,7 @@ proc createRssRouter*(cfg: Config) = case tab of "with_replies": getReplyQuery(name) of "media": getMediaQuery(name) + of "favorites": getFavoritesQuery(name) of "search": initQuery(params(request), name=name) else: Query(fromUser: @[name]) diff --git a/src/routes/status.nim b/src/routes/status.nim index 7e89220..036eca0 100644 --- a/src/routes/status.nim +++ b/src/routes/status.nim @@ -5,7 +5,7 @@ import jester, karax/vdom import router_utils import ".."/[types, formatters, api] -import ../views/[general, status] +import ../views/[general, status, search] export uri, sequtils, options, sugar export router_utils @@ -14,6 +14,29 @@ export status proc createStatusRouter*(cfg: Config) = router status: + get "/@name/status/@id/@reactors": + cond '.' notin @"name" + let id = @"id" + + if id.len > 19 or id.any(c => not c.isDigit): + resp Http404, showError("Invalid tweet ID", cfg) + + let prefs = cookiePrefs() + + # used for the infinite scroll feature + if @"scroll".len > 0: + let replies = await getReplies(id, getCursor()) + if replies.content.len == 0: + resp Http404, "" + resp $renderReplies(replies, prefs, getPath()) + + if @"reactors" == "favoriters": + resp renderMain(renderUserList(await getGraphFavoriters(id, getCursor()), prefs), + request, cfg, prefs) + elif @"reactors" == "retweeters": + resp renderMain(renderUserList(await getGraphRetweeters(id, getCursor()), prefs), + request, cfg, prefs) + get "/@name/status/@id/?": cond '.' notin @"name" let id = @"id" diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 3568ab7..b71e182 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -16,6 +16,7 @@ proc getQuery*(request: Request; tab, name: string): Query = case tab of "with_replies": getReplyQuery(name) of "media": getMediaQuery(name) + of "favorites": getFavoritesQuery(name) of "search": initQuery(params(request), name=name) else: Query(fromUser: @[name]) @@ -27,7 +28,7 @@ template skipIf[T](cond: bool; default; body: Future[T]): Future[T] = else: body -proc fetchProfile*(after: string; query: Query; skipRail=false; +proc fetchProfile*(after: string; query: Query; cfg: Config; skipRail=false; skipPinned=false): Future[Profile] {.async.} = let name = query.fromUser[0] @@ -56,6 +57,7 @@ proc fetchProfile*(after: string; query: Query; skipRail=false; of posts: await getGraphUserTweets(userId, TimelineKind.tweets, after) of replies: await getGraphUserTweets(userId, TimelineKind.replies, after) of media: await getGraphUserTweets(userId, TimelineKind.media, after) + of favorites: await getFavorites(userId, cfg, after) else: Profile(tweets: await getGraphTweetSearch(query, after)) result.user = await user @@ -71,7 +73,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs; html = renderTweetSearch(timeline, prefs, getPath()) return renderMain(html, request, cfg, prefs, "Multi", rss=rss) - var profile = await fetchProfile(after, query, skipPinned=prefs.hidePins) + var profile = await fetchProfile(after, query, cfg, skipPinned=prefs.hidePins) template u: untyped = profile.user if u.suspended: @@ -79,7 +81,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs; if profile.user.id.len == 0: return - let pHtml = renderProfile(profile, prefs, getPath()) + let pHtml = renderProfile(profile, cfg, prefs, getPath()) result = renderMain(pHtml, request, cfg, prefs, pageTitle(u), pageDesc(u), rss=rss, images = @[u.getUserPic("_400x400")], banner=u.banner) @@ -109,35 +111,42 @@ proc createTimelineRouter*(cfg: Config) = get "/@name/?@tab?/?": cond '.' notin @"name" cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"] - cond @"tab" in ["with_replies", "media", "search", ""] + cond @"tab" in ["with_replies", "media", "search", "favorites", "following", "followers", ""] let prefs = cookiePrefs() after = getCursor() names = getNames(@"name") + tab = @"tab" - var query = request.getQuery(@"tab", @"name") - if names.len != 1: - query.fromUser = names - - # used for the infinite scroll feature - if @"scroll".len > 0: - if query.fromUser.len != 1: - var timeline = await getGraphTweetSearch(query, after) - if timeline.content.len == 0: resp Http404 - timeline.beginning = true - resp $renderTweetSearch(timeline, prefs, getPath()) + case tab: + of "followers": + resp renderMain(renderUserList(await getGraphFollowers(await getUserId(@"name"), getCursor()), prefs), request, cfg, prefs) + of "following": + resp renderMain(renderUserList(await getGraphFollowing(await getUserId(@"name"), getCursor()), prefs), request, cfg, prefs) else: - var profile = await fetchProfile(after, query, skipRail=true) - if profile.tweets.content.len == 0: resp Http404 - profile.tweets.beginning = true - resp $renderTimelineTweets(profile.tweets, prefs, getPath()) + var query = request.getQuery(@"tab", @"name") + if names.len != 1: + query.fromUser = names - let rss = - if @"tab".len == 0: - "/$1/rss" % @"name" - elif @"tab" == "search": - "/$1/search/rss?$2" % [@"name", genQueryUrl(query)] - else: - "/$1/$2/rss" % [@"name", @"tab"] + # used for the infinite scroll feature + if @"scroll".len > 0: + if query.fromUser.len != 1: + var timeline = await getGraphTweetSearch(query, after) + if timeline.content.len == 0: resp Http404 + timeline.beginning = true + resp $renderTweetSearch(timeline, prefs, getPath()) + else: + var profile = await fetchProfile(after, query, cfg, skipRail=true) + if profile.tweets.content.len == 0: resp Http404 + profile.tweets.beginning = true + resp $renderTimelineTweets(profile.tweets, prefs, getPath()) - respTimeline(await showTimeline(request, query, cfg, prefs, rss, after)) + let rss = + if @"tab".len == 0: + "/$1/rss" % @"name" + elif @"tab" == "search": + "/$1/search/rss?$2" % [@"name", genQueryUrl(query)] + else: + "/$1/$2/rss" % [@"name", @"tab"] + + respTimeline(await showTimeline(request, query, cfg, prefs, rss, after)) diff --git a/src/sass/tweet/_base.scss b/src/sass/tweet/_base.scss index 69f51c0..3431a7b 100644 --- a/src/sass/tweet/_base.scss +++ b/src/sass/tweet/_base.scss @@ -207,6 +207,7 @@ padding-top: 5px; min-width: 1em; margin-right: 0.8em; + pointer-events: all; } .show-thread { diff --git a/src/types.nim b/src/types.nim index ddbebdf..a99aed5 100644 --- a/src/types.nim +++ b/src/types.nim @@ -23,9 +23,14 @@ type listTweets userRestId userScreenName + favorites userTweets userTweetsAndReplies userMedia + favoriters + retweeters + following + followers RateLimit* = object remaining*: int @@ -111,7 +116,7 @@ type variants*: seq[VideoVariant] QueryKind* = enum - posts, replies, media, users, tweets, userList + posts, replies, media, users, tweets, userList, favorites Query* = object kind*: QueryKind @@ -231,6 +236,7 @@ type replies*: Result[Chain] Timeline* = Result[Tweets] + UsersTimeline* = Result[User] Profile* = object user*: User @@ -276,6 +282,7 @@ type redisConns*: int redisMaxConns*: int redisPassword*: string + redisDb*: int Rss* = object feed*, cursor*: string diff --git a/src/views/general.nim b/src/views/general.nim index 35efb0b..87d30f2 100644 --- a/src/views/general.nim +++ b/src/views/general.nim @@ -32,6 +32,8 @@ proc renderNavbar(cfg: Config; req: Request; rss, canonical: string): VNode = if cfg.enableRss and rss.len > 0: icon "rss-feed", title="RSS Feed", href=rss icon "bird", title="Open in Twitter", href=canonical + a(href="https://liberapay.com/zedeus"): verbatim lp + icon "info", title="About", href="/about" icon "cog", title="Preferences", href=("/settings?referer=" & encodeUrl(path)) proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; @@ -71,7 +73,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; link(rel="alternate", type="application/rss+xml", href=rss, title="RSS feed") if prefs.hlsPlayback: - script(src="/js/hls.min.js", `defer`="") + script(src="/js/hls.light.min.js", `defer`="") script(src="/js/hlsPlayback.js", `defer`="") if prefs.infiniteScroll: diff --git a/src/views/profile.nim b/src/views/profile.nim index 8f67f5a..2ec79f7 100644 --- a/src/views/profile.nim +++ b/src/views/profile.nim @@ -13,7 +13,7 @@ proc renderStat(num: int; class: string; text=""): VNode = text insertSep($num, ',') proc renderUserCard*(user: User; prefs: Prefs): VNode = - buildHtml(tdiv(class="profile-card", "data-profile-id" = $user.id)): + buildHtml(tdiv(class="profile-card")): tdiv(class="profile-card-info"): let url = getPicUrl(user.getUserPic()) @@ -58,10 +58,14 @@ proc renderUserCard*(user: User; prefs: Prefs): VNode = tdiv(class="profile-card-extra-links"): ul(class="profile-statlist"): - renderStat(user.tweets, "posts", text="Tweets") - renderStat(user.following, "following") - renderStat(user.followers, "followers") - renderStat(user.likes, "likes") + a(href="/" & user.username): + renderStat(user.tweets, "posts", text="Tweets") + a(href="/" & user.username & "/following"): + renderStat(user.following, "following") + a(href="/" & user.username & "/followers"): + renderStat(user.followers, "followers") + a(href="/" & user.username & "/favorites"): + renderStat(user.likes, "likes") proc renderPhotoRail(profile: Profile): VNode = let count = insertSep($profile.user.media, ',') @@ -99,7 +103,7 @@ proc renderProtected(username: string): VNode = h2: text "This account's tweets are protected." p: text &"Only confirmed followers have access to @{username}'s tweets." -proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode = +proc renderProfile*(profile: var Profile; cfg: Config; prefs: Prefs; path: string): VNode = profile.tweets.query.fromUser = @[profile.user.username] buildHtml(tdiv(class="profile-tabs")): diff --git a/src/views/search.nim b/src/views/search.nim index 401e6da..0e5e808 100644 --- a/src/views/search.nim +++ b/src/views/search.nim @@ -3,7 +3,7 @@ import strutils, strformat, sequtils, unicode, tables, options import karax/[karaxdsl, vdom] import renderutils, timeline -import ".."/[types, query] +import ".."/[types, query, config] const toggles = { "nativeretweets": "Retweets", @@ -29,7 +29,7 @@ proc renderSearch*(): VNode = placeholder="Enter username...", dir="auto") button(`type`="submit"): icon "search" -proc renderProfileTabs*(query: Query; username: string): VNode = +proc renderProfileTabs*(query: Query; username: string; cfg: Config): VNode = let link = "/" & username buildHtml(ul(class="tab")): li(class=query.getTabClass(posts)): @@ -38,6 +38,8 @@ proc renderProfileTabs*(query: Query; username: string): VNode = a(href=(link & "/with_replies")): text "Tweets & Replies" li(class=query.getTabClass(media)): a(href=(link & "/media")): text "Media" + li(class=query.getTabClass(favorites)): + a(href=(link & "/favorites")): text "Likes" li(class=query.getTabClass(tweets)): a(href=(link & "/search")): text "Search" @@ -97,7 +99,7 @@ proc renderTweetSearch*(results: Timeline; prefs: Prefs; path: string; text query.fromUser.join(" | ") if query.fromUser.len > 0: - renderProfileTabs(query, query.fromUser.join(",")) + renderProfileTabs(query, query.fromUser.join(","), cfg) if query.fromUser.len == 0 or query.kind == tweets: tdiv(class="timeline-header"): @@ -118,3 +120,8 @@ proc renderUserSearch*(results: Result[User]; prefs: Prefs): VNode = renderSearchTabs(results.query) renderTimelineUsers(results, prefs) + +proc renderUserList*(results: Result[User]; prefs: Prefs): VNode = + buildHtml(tdiv(class="timeline-container")): + tdiv(class="timeline-header") + renderTimelineUsers(results, prefs) diff --git a/src/views/tweet.nim b/src/views/tweet.nim index 2fe4ac9..13b4a24 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -180,14 +180,19 @@ func formatStat(stat: int): string = if stat > 0: insertSep($stat, ',') else: "" -proc renderStats(stats: TweetStats; views: string): VNode = +proc renderStats(stats: TweetStats; views: string; tweet: Tweet): VNode = buildHtml(tdiv(class="tweet-stats")): - span(class="tweet-stat"): icon "comment", formatStat(stats.replies) - span(class="tweet-stat"): icon "retweet", formatStat(stats.retweets) - span(class="tweet-stat"): icon "quote", formatStat(stats.quotes) - span(class="tweet-stat"): icon "heart", formatStat(stats.likes) - if views.len > 0: - span(class="tweet-stat"): icon "play", insertSep(views, ',') + a(href=getLink(tweet)): + span(class="tweet-stat"): icon "comment", formatStat(stats.replies) + a(href=getLink(tweet, false) & "/retweeters"): + span(class="tweet-stat"): icon "retweet", formatStat(stats.retweets) + a(href="/search?q=quoted_tweet_id:" & $tweet.id): + span(class="tweet-stat"): icon "quote", formatStat(stats.quotes) + a(href=getLink(tweet, false) & "/favoriters"): + span(class="tweet-stat"): icon "heart", formatStat(stats.likes) + a(href=getLink(tweet)): + if views.len > 0: + span(class="tweet-stat"): icon "play", insertSep(views, ',') proc renderReply(tweet: Tweet): VNode = buildHtml(tdiv(class="replying-to")): @@ -345,7 +350,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0; renderMediaTags(tweet.mediaTags) if not prefs.hideTweetStats: - renderStats(tweet.stats, views) + renderStats(tweet.stats, views, tweet) if showThread: a(class="show-thread", href=("/i/status/" & $tweet.threadId)): From 372b58d1fe912e73b84b75a95dcbec2ef4456b65 Mon Sep 17 00:00:00 2001 From: taskylizard <75871323+taskylizard@users.noreply.github.com> Date: Sun, 19 May 2024 06:32:16 +0000 Subject: [PATCH 05/12] feat: cache proxy --- .gitignore | 1 + docker-compose.yml | 16 + proxy/.npmrc | 1 + proxy/Dockerfile | 21 ++ proxy/package.json | 27 ++ proxy/pnpm-lock.yaml | 755 +++++++++++++++++++++++++++++++++++++++++++ proxy/src/index.ts | 89 +++++ proxy/src/proxy.ts | 209 ++++++++++++ proxy/src/types.d.ts | 39 +++ proxy/tsconfig.json | 18 ++ 10 files changed, 1176 insertions(+) create mode 100644 proxy/.npmrc create mode 100644 proxy/Dockerfile create mode 100644 proxy/package.json create mode 100644 proxy/pnpm-lock.yaml create mode 100644 proxy/src/index.ts create mode 100644 proxy/src/proxy.ts create mode 100644 proxy/src/types.d.ts create mode 100644 proxy/tsconfig.json diff --git a/.gitignore b/.gitignore index ea520dc..55031d6 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ nitter nitter.conf guest_accounts.json* dump.rdb +proxy/node_modules diff --git a/docker-compose.yml b/docker-compose.yml index 207914e..2c4625e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,22 @@ networks: nitter: services: + # proxy: + # hostname: nitter-proxy + # container_name: nitter-proxy + # build: + # context: ./proxy + # dockerfile: Dockerfile + # environment: + # HOST: "0.0.0.0" + # PORT: "8080" + # NITTER_BASE_URL: "http://nitter:8080" + # CONCURRENCY: "1" + # ports: + # - "8002:8080" + # networks: + # - nitter + nitter: build: . container_name: nitter diff --git a/proxy/.npmrc b/proxy/.npmrc new file mode 100644 index 0000000..3bd3b7d --- /dev/null +++ b/proxy/.npmrc @@ -0,0 +1 @@ +shell-emulator=true diff --git a/proxy/Dockerfile b/proxy/Dockerfile new file mode 100644 index 0000000..000ead8 --- /dev/null +++ b/proxy/Dockerfile @@ -0,0 +1,21 @@ +# this is our first build stage, it will not persist in the final image +FROM node:20.11.0-buster-slim as intermediate + +# installation required packages +RUN apt-get update && apt-get install -y --no-install-recommends ssh git python python3 build-essential +RUN npm install -g pnpm@9.1.1 +RUN mkdir -p /opt +WORKDIR /opt + +COPY tsconfig.json /opt +COPY package.json pnpm-lock.yaml ./ +RUN pnpm install --frozen-lockfile + +COPY ./src /opt/src + +RUN pnpm run build + +# copy just the package form the previous image +FROM node:20.11.0-buster-slim +COPY --from=intermediate /opt /opt +ENTRYPOINT ["node", "/opt/build/index.js"] diff --git a/proxy/package.json b/proxy/package.json new file mode 100644 index 0000000..0092699 --- /dev/null +++ b/proxy/package.json @@ -0,0 +1,27 @@ +{ + "name": "nitter-proxy", + "version": "1.6.7", + "scripts": { + "clean": "rm -rf build", + "prebuild": "npm run clean", + "build": "tsc --build" + }, + "author": "", + "license": "ISC", + "dependencies": { + "@fastify/rate-limit": "^9.1.0", + "axios": "^1.6.7", + "axios-retry-after": "^2.0.0", + "fastify": "^4.21.0", + "fastq": "^1.17.1", + "lru-cache": "^10.2.0", + "lru-ttl-cache": "^2.4.8", + "pino": "^8.14.2", + "pino-pretty": "^10.2.0", + "typescript": "^5.3.3" + }, + "devDependencies": { + "@types/node": "^20.12.12", + "dotenv": "^16.4.4" + } +} diff --git a/proxy/pnpm-lock.yaml b/proxy/pnpm-lock.yaml new file mode 100644 index 0000000..2d53adc --- /dev/null +++ b/proxy/pnpm-lock.yaml @@ -0,0 +1,755 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@fastify/rate-limit': + specifier: ^9.1.0 + version: 9.1.0 + axios: + specifier: ^1.6.7 + version: 1.6.7 + axios-retry-after: + specifier: ^2.0.0 + version: 2.0.0(axios@1.6.7) + fastify: + specifier: ^4.21.0 + version: 4.26.1 + fastq: + specifier: ^1.17.1 + version: 1.17.1 + lru-cache: + specifier: ^10.2.0 + version: 10.2.0 + lru-ttl-cache: + specifier: ^2.4.8 + version: 2.4.8 + pino: + specifier: ^8.14.2 + version: 8.18.0 + pino-pretty: + specifier: ^10.2.0 + version: 10.3.1 + typescript: + specifier: ^5.3.3 + version: 5.3.3 + devDependencies: + '@types/node': + specifier: ^20.12.12 + version: 20.12.12 + dotenv: + specifier: ^16.4.4 + version: 16.4.4 + +packages: + + '@fastify/ajv-compiler@3.5.0': + resolution: {integrity: sha512-ebbEtlI7dxXF5ziNdr05mOY8NnDiPB1XvAlLHctRt/Rc+C3LCOVW5imUVX+mhvUhnNzmPBHewUkOFgGlCxgdAA==} + + '@fastify/error@3.4.1': + resolution: {integrity: sha512-wWSvph+29GR783IhmvdwWnN4bUxTD01Vm5Xad4i7i1VuAOItLvbPAb69sb0IQ2N57yprvhNIwAP5B6xfKTmjmQ==} + + '@fastify/fast-json-stringify-compiler@4.3.0': + resolution: {integrity: sha512-aZAXGYo6m22Fk1zZzEUKBvut/CIIQe/BapEORnxiD5Qr0kPHqqI69NtEMCme74h+at72sPhbkb4ZrLd1W3KRLA==} + + '@fastify/merge-json-schemas@0.1.1': + resolution: {integrity: sha512-fERDVz7topgNjtXsJTTW1JKLy0rhuLRcquYqNR9rF7OcVpCa2OVW49ZPDIhaRRCaUuvVxI+N416xUoF76HNSXA==} + + '@fastify/rate-limit@9.1.0': + resolution: {integrity: sha512-h5dZWCkuZXN0PxwqaFQLxeln8/LNwQwH9popywmDCFdKfgpi4b/HoMH1lluy6P+30CG9yzzpSpwTCIPNB9T1JA==} + + '@lukeed/ms@2.0.2': + resolution: {integrity: sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==} + engines: {node: '>=8'} + + '@types/node@20.12.12': + resolution: {integrity: sha512-eWLDGF/FOSPtAvEqeRAQ4C8LSA7M1I7i0ky1I8U7kD1J5ITyW3AsRhQrKVoWf5pFKZ2kILsEGJhsI9r93PYnOw==} + + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + + abstract-logging@2.0.1: + resolution: {integrity: sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==} + + ajv-formats@2.1.1: + resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} + peerDependencies: + ajv: ^8.0.0 + peerDependenciesMeta: + ajv: + optional: true + + ajv@8.12.0: + resolution: {integrity: sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==} + + archy@1.0.0: + resolution: {integrity: sha512-Xg+9RwCg/0p32teKdGMPTPnVXKD0w3DfHnFTficozsAgsvq2XenPJq/MYpzzQ/v8zrOyJn6Ds39VA4JIDwFfqw==} + + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + + atomic-sleep@1.0.0: + resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} + engines: {node: '>=8.0.0'} + + avvio@8.3.0: + resolution: {integrity: sha512-VBVH0jubFr9LdFASy/vNtm5giTrnbVquWBhT0fyizuNK2rQ7e7ONU2plZQWUNqtE1EmxFEb+kbSkFRkstiaS9Q==} + + axios-retry-after@2.0.0: + resolution: {integrity: sha512-tSB1DEF1bSwXmRNyPcopFsiHAF+PWVq5w2mAK7J0bTltn8x2UnfoSJzTVXPySt/WdrbQL4ES5AXtG9i016+CaA==} + engines: {node: '>= 14'} + peerDependencies: + axios: ^1.0.0 + + axios@1.6.7: + resolution: {integrity: sha512-/hDJGff6/c7u0hDkvkGxR/oy6CbCs8ziCsC7SqmhjfozqiJGc8Z11wrv9z9lYfY4K8l+H9TpjcMDX0xOZmx+RA==} + + base64-js@1.5.1: + resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + + buffer@6.0.3: + resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + + bytes@3.1.2: + resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} + engines: {node: '>= 0.8'} + + colorette@2.0.20: + resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} + + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + + cookie@0.5.0: + resolution: {integrity: sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==} + engines: {node: '>= 0.6'} + + dateformat@4.6.3: + resolution: {integrity: sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==} + + debug@4.3.4: + resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + + dotenv@16.4.4: + resolution: {integrity: sha512-XvPXc8XAQThSjAbY6cQ/9PcBXmFoWuw1sQ3b8HqUCR6ziGXjkTi//kB9SWa2UwqlgdAIuRqAa/9hVljzPehbYg==} + engines: {node: '>=12'} + + end-of-stream@1.4.4: + resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} + + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + + events@3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} + engines: {node: '>=0.8.x'} + + fast-content-type-parse@1.1.0: + resolution: {integrity: sha512-fBHHqSTFLVnR61C+gltJuE5GkVQMV0S2nqUO8TJ+5Z3qAKG8vAx4FKai1s5jq/inV1+sREynIWSuQ6HgoSXpDQ==} + + fast-copy@3.0.1: + resolution: {integrity: sha512-Knr7NOtK3HWRYGtHoJrjkaWepqT8thIVGAwt0p0aUs1zqkAzXZV4vo9fFNwyb5fcqK1GKYFYxldQdIDVKhUAfA==} + + fast-decode-uri-component@1.0.1: + resolution: {integrity: sha512-WKgKWg5eUxvRZGwW8FvfbaH7AXSh2cL+3j5fMGzUMCxWBJ3dV3a7Wz8y2f/uQ0e3B6WmodD3oS54jTQ9HVTIIg==} + + fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + + fast-json-stringify@5.12.0: + resolution: {integrity: sha512-7Nnm9UPa7SfHRbHVA1kJQrGXCRzB7LMlAAqHXQFkEQqueJm1V8owm0FsE/2Do55/4CcdhwiLQERaKomOnKQkyA==} + + fast-querystring@1.1.2: + resolution: {integrity: sha512-g6KuKWmFXc0fID8WWH0jit4g0AGBoJhCkJMb1RmbsSEUNvQ+ZC8D6CUZ+GtF8nMzSPXnhiePyyqqipzNNEnHjg==} + + fast-redact@3.3.0: + resolution: {integrity: sha512-6T5V1QK1u4oF+ATxs1lWUmlEk6P2T9HqJG3e2DnHOdVgZy2rFJBoEnrIedcTXlkAHU/zKC+7KETJ+KGGKwxgMQ==} + engines: {node: '>=6'} + + fast-safe-stringify@2.1.1: + resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==} + + fast-uri@2.3.0: + resolution: {integrity: sha512-eel5UKGn369gGEWOqBShmFJWfq/xSJvsgDzgLYC845GneayWvXBf0lJCBn5qTABfewy1ZDPoaR5OZCP+kssfuw==} + + fastify-plugin@4.5.1: + resolution: {integrity: sha512-stRHYGeuqpEZTL1Ef0Ovr2ltazUT9g844X5z/zEBFLG8RYlpDiOCIG+ATvYEp+/zmc7sN29mcIMp8gvYplYPIQ==} + + fastify@4.26.1: + resolution: {integrity: sha512-tznA/G55dsxzM5XChBfcvVSloG2ejeeotfPPJSFaWmHyCDVGMpvf3nRNbsCb/JTBF9RmQFBfuujWt3Nphjesng==} + + fastq@1.17.1: + resolution: {integrity: sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==} + + find-my-way@8.1.0: + resolution: {integrity: sha512-41QwjCGcVTODUmLLqTMeoHeiozbMXYMAE1CKFiDyi9zVZ2Vjh0yz3MF0WQZoIb+cmzP/XlbFjlF2NtJmvZHznA==} + engines: {node: '>=14'} + + follow-redirects@1.15.5: + resolution: {integrity: sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + + form-data@4.0.0: + resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} + engines: {node: '>= 6'} + + forwarded@0.2.0: + resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} + engines: {node: '>= 0.6'} + + help-me@5.0.0: + resolution: {integrity: sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==} + + ieee754@1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + + ipaddr.js@1.9.1: + resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} + engines: {node: '>= 0.10'} + + joycon@3.1.1: + resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} + engines: {node: '>=10'} + + json-schema-ref-resolver@1.0.1: + resolution: {integrity: sha512-EJAj1pgHc1hxF6vo2Z3s69fMjO1INq6eGHXZ8Z6wCQeldCuwxGK9Sxf4/cScGn3FZubCVUehfWtcDM/PLteCQw==} + + json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + + light-my-request@5.11.0: + resolution: {integrity: sha512-qkFCeloXCOMpmEdZ/MV91P8AT4fjwFXWaAFz3lUeStM8RcoM1ks4J/F8r1b3r6y/H4u3ACEJ1T+Gv5bopj7oDA==} + + lru-cache@10.2.0: + resolution: {integrity: sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q==} + engines: {node: 14 || >=16.14} + + lru-cache@6.0.0: + resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==} + engines: {node: '>=10'} + + lru-ttl-cache@2.4.8: + resolution: {integrity: sha512-24fyw4EHvHKSGXc6DMpcA5Iet9UnPxrXzcXA9bvkTs/9kQL6m9SllRyWAtZdxzUabKWdm38vOHavXwdDx2Zl8g==} + + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + + ms@2.1.2: + resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + on-exit-leak-free@2.1.2: + resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} + engines: {node: '>=14.0.0'} + + once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + + pino-abstract-transport@1.1.0: + resolution: {integrity: sha512-lsleG3/2a/JIWUtf9Q5gUNErBqwIu1tUKTT3dUzaf5DySw9ra1wcqKjJjLX1VTY64Wk1eEOYsVGSaGfCK85ekA==} + + pino-pretty@10.3.1: + resolution: {integrity: sha512-az8JbIYeN/1iLj2t0jR9DV48/LQ3RC6hZPpapKPkb84Q+yTidMCpgWxIT3N0flnBDilyBQ1luWNpOeJptjdp/g==} + hasBin: true + + pino-std-serializers@6.2.2: + resolution: {integrity: sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA==} + + pino@8.18.0: + resolution: {integrity: sha512-Mz/gKiRyuXu4HnpHgi1YWdHQCoWMufapzooisvFn78zl4dZciAxS+YeRkUxXl1ee/SzU80YCz1zpECCh4oC6Aw==} + hasBin: true + + process-warning@2.3.2: + resolution: {integrity: sha512-n9wh8tvBe5sFmsqlg+XQhaQLumwpqoAUruLwjCopgTmUBjJ/fjtBsJzKleCaIGBOMXYEhp1YfKl4d7rJ5ZKJGA==} + + process-warning@3.0.0: + resolution: {integrity: sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ==} + + process@0.11.10: + resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} + engines: {node: '>= 0.6.0'} + + proxy-addr@2.0.7: + resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} + engines: {node: '>= 0.10'} + + proxy-from-env@1.1.0: + resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + + pump@3.0.0: + resolution: {integrity: sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==} + + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + + quick-format-unescaped@4.0.4: + resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} + + readable-stream@4.5.2: + resolution: {integrity: sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + real-require@0.2.0: + resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==} + engines: {node: '>= 12.13.0'} + + require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + + ret@0.2.2: + resolution: {integrity: sha512-M0b3YWQs7R3Z917WRQy1HHA7Ba7D8hvZg6UE5mLykJxQVE2ju0IXbGlaHPPlkY+WN7wFP+wUMXmBFA0aV6vYGQ==} + engines: {node: '>=4'} + + reusify@1.0.4: + resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} + engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + + rfdc@1.3.1: + resolution: {integrity: sha512-r5a3l5HzYlIC68TpmYKlxWjmOP6wiPJ1vWv2HeLhNsRZMrCkxeqxiHlQ21oXmQ4F3SiryXBHhAD7JZqvOJjFmg==} + + safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + + safe-regex2@2.0.0: + resolution: {integrity: sha512-PaUSFsUaNNuKwkBijoAPHAK6/eM6VirvyPWlZ7BAQy4D+hCvh4B6lIG+nPdhbFfIbP+gTGBcrdsOaUs0F+ZBOQ==} + + safe-stable-stringify@2.4.3: + resolution: {integrity: sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==} + engines: {node: '>=10'} + + secure-json-parse@2.7.0: + resolution: {integrity: sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==} + + semver@7.6.0: + resolution: {integrity: sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==} + engines: {node: '>=10'} + hasBin: true + + set-cookie-parser@2.6.0: + resolution: {integrity: sha512-RVnVQxTXuerk653XfuliOxBP81Sf0+qfQE73LIYKcyMYHG94AuH0kgrQpRDuTZnSmjpysHmzxJXKNfa6PjFhyQ==} + + sonic-boom@3.8.0: + resolution: {integrity: sha512-ybz6OYOUjoQQCQ/i4LU8kaToD8ACtYP+Cj5qd2AO36bwbdewxWJ3ArmJ2cr6AvxlL2o0PqnCcPGUgkILbfkaCA==} + + split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + + string_decoder@1.3.0: + resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + + strip-json-comments@3.1.1: + resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} + engines: {node: '>=8'} + + thread-stream@2.4.1: + resolution: {integrity: sha512-d/Ex2iWd1whipbT681JmTINKw0ZwOUBZm7+Gjs64DHuX34mmw8vJL2bFAaNacaW72zYiTJxSHi5abUuOi5nsfg==} + + toad-cache@3.7.0: + resolution: {integrity: sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw==} + engines: {node: '>=12'} + + typescript@5.3.3: + resolution: {integrity: sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + + uri-js@4.4.1: + resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + + wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + + yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + +snapshots: + + '@fastify/ajv-compiler@3.5.0': + dependencies: + ajv: 8.12.0 + ajv-formats: 2.1.1(ajv@8.12.0) + fast-uri: 2.3.0 + + '@fastify/error@3.4.1': {} + + '@fastify/fast-json-stringify-compiler@4.3.0': + dependencies: + fast-json-stringify: 5.12.0 + + '@fastify/merge-json-schemas@0.1.1': + dependencies: + fast-deep-equal: 3.1.3 + + '@fastify/rate-limit@9.1.0': + dependencies: + '@lukeed/ms': 2.0.2 + fastify-plugin: 4.5.1 + toad-cache: 3.7.0 + + '@lukeed/ms@2.0.2': {} + + '@types/node@20.12.12': + dependencies: + undici-types: 5.26.5 + + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + + abstract-logging@2.0.1: {} + + ajv-formats@2.1.1(ajv@8.12.0): + optionalDependencies: + ajv: 8.12.0 + + ajv@8.12.0: + dependencies: + fast-deep-equal: 3.1.3 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + uri-js: 4.4.1 + + archy@1.0.0: {} + + asynckit@0.4.0: {} + + atomic-sleep@1.0.0: {} + + avvio@8.3.0: + dependencies: + '@fastify/error': 3.4.1 + archy: 1.0.0 + debug: 4.3.4 + fastq: 1.17.1 + transitivePeerDependencies: + - supports-color + + axios-retry-after@2.0.0(axios@1.6.7): + dependencies: + axios: 1.6.7 + + axios@1.6.7: + dependencies: + follow-redirects: 1.15.5 + form-data: 4.0.0 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + + base64-js@1.5.1: {} + + buffer@6.0.3: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + + bytes@3.1.2: {} + + colorette@2.0.20: {} + + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + + cookie@0.5.0: {} + + dateformat@4.6.3: {} + + debug@4.3.4: + dependencies: + ms: 2.1.2 + + delayed-stream@1.0.0: {} + + dotenv@16.4.4: {} + + end-of-stream@1.4.4: + dependencies: + once: 1.4.0 + + event-target-shim@5.0.1: {} + + events@3.3.0: {} + + fast-content-type-parse@1.1.0: {} + + fast-copy@3.0.1: {} + + fast-decode-uri-component@1.0.1: {} + + fast-deep-equal@3.1.3: {} + + fast-json-stringify@5.12.0: + dependencies: + '@fastify/merge-json-schemas': 0.1.1 + ajv: 8.12.0 + ajv-formats: 2.1.1(ajv@8.12.0) + fast-deep-equal: 3.1.3 + fast-uri: 2.3.0 + json-schema-ref-resolver: 1.0.1 + rfdc: 1.3.1 + + fast-querystring@1.1.2: + dependencies: + fast-decode-uri-component: 1.0.1 + + fast-redact@3.3.0: {} + + fast-safe-stringify@2.1.1: {} + + fast-uri@2.3.0: {} + + fastify-plugin@4.5.1: {} + + fastify@4.26.1: + dependencies: + '@fastify/ajv-compiler': 3.5.0 + '@fastify/error': 3.4.1 + '@fastify/fast-json-stringify-compiler': 4.3.0 + abstract-logging: 2.0.1 + avvio: 8.3.0 + fast-content-type-parse: 1.1.0 + fast-json-stringify: 5.12.0 + find-my-way: 8.1.0 + light-my-request: 5.11.0 + pino: 8.18.0 + process-warning: 3.0.0 + proxy-addr: 2.0.7 + rfdc: 1.3.1 + secure-json-parse: 2.7.0 + semver: 7.6.0 + toad-cache: 3.7.0 + transitivePeerDependencies: + - supports-color + + fastq@1.17.1: + dependencies: + reusify: 1.0.4 + + find-my-way@8.1.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-querystring: 1.1.2 + safe-regex2: 2.0.0 + + follow-redirects@1.15.5: {} + + form-data@4.0.0: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + mime-types: 2.1.35 + + forwarded@0.2.0: {} + + help-me@5.0.0: {} + + ieee754@1.2.1: {} + + ipaddr.js@1.9.1: {} + + joycon@3.1.1: {} + + json-schema-ref-resolver@1.0.1: + dependencies: + fast-deep-equal: 3.1.3 + + json-schema-traverse@1.0.0: {} + + light-my-request@5.11.0: + dependencies: + cookie: 0.5.0 + process-warning: 2.3.2 + set-cookie-parser: 2.6.0 + + lru-cache@10.2.0: {} + + lru-cache@6.0.0: + dependencies: + yallist: 4.0.0 + + lru-ttl-cache@2.4.8: + dependencies: + bytes: 3.1.2 + ms: 2.1.3 + + mime-db@1.52.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + + minimist@1.2.8: {} + + ms@2.1.2: {} + + ms@2.1.3: {} + + on-exit-leak-free@2.1.2: {} + + once@1.4.0: + dependencies: + wrappy: 1.0.2 + + pino-abstract-transport@1.1.0: + dependencies: + readable-stream: 4.5.2 + split2: 4.2.0 + + pino-pretty@10.3.1: + dependencies: + colorette: 2.0.20 + dateformat: 4.6.3 + fast-copy: 3.0.1 + fast-safe-stringify: 2.1.1 + help-me: 5.0.0 + joycon: 3.1.1 + minimist: 1.2.8 + on-exit-leak-free: 2.1.2 + pino-abstract-transport: 1.1.0 + pump: 3.0.0 + readable-stream: 4.5.2 + secure-json-parse: 2.7.0 + sonic-boom: 3.8.0 + strip-json-comments: 3.1.1 + + pino-std-serializers@6.2.2: {} + + pino@8.18.0: + dependencies: + atomic-sleep: 1.0.0 + fast-redact: 3.3.0 + on-exit-leak-free: 2.1.2 + pino-abstract-transport: 1.1.0 + pino-std-serializers: 6.2.2 + process-warning: 3.0.0 + quick-format-unescaped: 4.0.4 + real-require: 0.2.0 + safe-stable-stringify: 2.4.3 + sonic-boom: 3.8.0 + thread-stream: 2.4.1 + + process-warning@2.3.2: {} + + process-warning@3.0.0: {} + + process@0.11.10: {} + + proxy-addr@2.0.7: + dependencies: + forwarded: 0.2.0 + ipaddr.js: 1.9.1 + + proxy-from-env@1.1.0: {} + + pump@3.0.0: + dependencies: + end-of-stream: 1.4.4 + once: 1.4.0 + + punycode@2.3.1: {} + + quick-format-unescaped@4.0.4: {} + + readable-stream@4.5.2: + dependencies: + abort-controller: 3.0.0 + buffer: 6.0.3 + events: 3.3.0 + process: 0.11.10 + string_decoder: 1.3.0 + + real-require@0.2.0: {} + + require-from-string@2.0.2: {} + + ret@0.2.2: {} + + reusify@1.0.4: {} + + rfdc@1.3.1: {} + + safe-buffer@5.2.1: {} + + safe-regex2@2.0.0: + dependencies: + ret: 0.2.2 + + safe-stable-stringify@2.4.3: {} + + secure-json-parse@2.7.0: {} + + semver@7.6.0: + dependencies: + lru-cache: 6.0.0 + + set-cookie-parser@2.6.0: {} + + sonic-boom@3.8.0: + dependencies: + atomic-sleep: 1.0.0 + + split2@4.2.0: {} + + string_decoder@1.3.0: + dependencies: + safe-buffer: 5.2.1 + + strip-json-comments@3.1.1: {} + + thread-stream@2.4.1: + dependencies: + real-require: 0.2.0 + + toad-cache@3.7.0: {} + + typescript@5.3.3: {} + + undici-types@5.26.5: {} + + uri-js@4.4.1: + dependencies: + punycode: 2.3.1 + + wrappy@1.0.2: {} + + yallist@4.0.0: {} diff --git a/proxy/src/index.ts b/proxy/src/index.ts new file mode 100644 index 0000000..020c5ef --- /dev/null +++ b/proxy/src/index.ts @@ -0,0 +1,89 @@ + +import fastify, { + FastifyInstance, + FastifyListenOptions, + FastifyReply, + FastifyRequest, +} from "fastify" +import { PinoLoggerOptions } from "fastify/types/logger" +import { Proxy } from "./proxy" +import { Logger } from "pino" +import 'dotenv/config' + +const host = process.env.HOST +const port = parseInt(process.env.PORT ?? "8080", 10) +const baseUrl = process.env.NITTER_BASE_URL +const concurrency = parseInt(process.env.CONCURRENCY ?? "1", 10) +const retryAfterMillis = process.env.RETRY_AFTER_MILLIS ? parseInt(process.env.RETRY_AFTER_MILLIS, 10) : null +const maxCacheSize = parseInt(process.env.MAX_CACHE_SIZE ?? "100000", 10) +const logLevel = process.env.LOG_LEVEL ?? "debug" + +const server = fastify({ + logger: { + name: "app", + level: logLevel, + ...( logLevel == "trace" ? { transport: { target: 'pino-pretty' } } : {}) + } as PinoLoggerOptions +}) + +const log = server.log as Logger +const proxy = new Proxy(log, baseUrl, concurrency, retryAfterMillis, maxCacheSize) + +async function main() { + + server.register((fastify: FastifyInstance, opts, done) => { + + fastify.get(`/user/:username`, {}, + async (request: FastifyRequest, reply: FastifyReply) => { + log.debug({ + headers: request.headers, + reqId: request.id, + params: request.params }, 'incoming request /user/:username') + const { username } = request.params as any + const { status, data } = await proxy.getUser(username, { reqId: request.id }) + reply.status(status).send(data) + }); + + fastify.get(`/user/:userId/tweets`, {}, + async (request: FastifyRequest, reply: FastifyReply) => { + const { userId } = request.params as any + const { cursor } = request.query as any + const { status, data } = await proxy.getUserTweets(userId, cursor, { reqId: request.id }) + reply.status(status).send(data) + }); + + fastify.get(`/tweet/:id`, {}, + async (request: FastifyRequest, reply: FastifyReply) => { + const { id } = request.params as any + const { status, data } = await proxy.getTweetById(id, { reqId: request.id }) + reply.status(status).send(data) + }); + + done() + + }, { prefix: '/api' }) + + server.setNotFoundHandler((request: FastifyRequest, reply: FastifyReply) => { + reply.status(404) + .send({ message: `Method not found` }) + }) + + server.setErrorHandler((err: Error, request: FastifyRequest, reply: FastifyReply) => { + const { log } = request + log.error(err) + // Send error response + reply.status(500).send({ message: `Internal server error` }) + }) + + // await server.register(import('@fastify/rate-limit'), { + // max: 100, + // timeWindow: '1 minute' + // }) + + await server.listen({ port, host } as FastifyListenOptions); +} + +main().catch(err => { + log.fatal(err) + process.exit(1) +}) diff --git a/proxy/src/proxy.ts b/proxy/src/proxy.ts new file mode 100644 index 0000000..872f481 --- /dev/null +++ b/proxy/src/proxy.ts @@ -0,0 +1,209 @@ +// noinspection TypeScriptUnresolvedReference + +import axios from "axios" +import { AxiosInstance, AxiosRequestConfig } from "axios" +import fastq from "fastq" +import { Logger } from "pino" +import retry from "axios-retry-after" +import { LRUCache } from 'lru-cache' + +const GET_USER_POSITIVE_TTL_MS = process.env.GET_USER_POSITIVE_TTL + ? parseInt(process.env.GET_USER_POSITIVE_TTL, 10) * 1000 + : 30 * 24 * 3600 * 1000 +const GET_USER_NEGATIVE_TTL_MS = process.env.GET_USER_NEGATIVE_TTL + ? parseInt(process.env.GET_USER_NEGATIVE_TTL, 10) * 1000 + : 3600 * 1000 +const GET_TWEETS_POSITIVE_TTL_MS = process.env.GET_TWEETS_POSITIVE_TTL + ? parseInt(process.env.GET_TWEETS_POSITIVE_TTL, 10) * 1000 + : 60 * 1000 +const GET_TWEETS_NEGATIVE_TTL_MS = process.env.GET_TWEETS_NEGATIVE_TTL + ? parseInt(process.env.GET_TWEETS_NEGATIVE_TTL, 10) * 1000 + : 60 * 1000 +const GET_TWEET_POSITIVE_TTL_MS = process.env.GET_TWEET_POSITIVE_TTL + ? parseInt(process.env.GET_TWEET_POSITIVE_TTL, 10) * 1000 + : 60 * 1000 +const GET_TWEET_NEGATIVE_TTL_MS = process.env.GET_TWEET_NEGATIVE_TTL + ? parseInt(process.env.GET_TWEET_NEGATIVE_TTL, 10) * 1000 + : 60 * 1000 + +export interface Job { + reqId: string + url: string + params?: Record +} + +export interface JobResponse { + status: number, + data: any +} + +export class Proxy { + + private readonly cache: LRUCache + private readonly client: AxiosInstance + private readonly queue: fastq.queueAsPromised + private counter: { requests: number } + private timeWindowMillis = 15 * 60 * 1000 + private maxRequestsPerAccount = 15 * 60 + + constructor( + private log: Logger, + private baseUrl: string, + private concurrency: number, + retryAfterMillis: number, + maxCacheSize: number + ) { + this.cache = new LRUCache({ max: maxCacheSize }) + this.queue = fastq.promise(this, this.sendRequest, concurrency) + this.client = axios.create() + this.counter = { + requests: 0 + } + + setInterval(() => { + this.counter.requests = 0 + }, this.timeWindowMillis) + + if ( retryAfterMillis ) { + this.client.interceptors.response.use(null, retry(this.client, { + // Determine when we should attempt to retry + isRetryable (error) { + log.debug({ status: error.response?.status, headers: error.response?.headers }, 'checking retryable') + return ( + error.response && error.response.status === 429 + // Use X-Retry-After rather than Retry-After, and cap retry delay at 60 seconds + // && error.response.headers['x-retry-after'] && error.response.headers['x-retry-after'] <= 60 + ) + }, + // Customize the wait behavior + wait (error) { + log.debug({ status: error.response?.status, headers: error.response?.headers }, 'waiting for retry') + return new Promise( + // Use X-Retry-After rather than Retry-After + // resolve => setTimeout(resolve, error.response.headers['x-retry-after']) + resolve => setTimeout(resolve, retryAfterMillis) + ) + } + })) + } + } + + async getUser(username: string, options?: { reqId?: string }) { + const key = `usernames:${username}` + + if ( this.cache.has(key)) { + return this.cache.get(key) + } + + const result = await this.queue.push({ + url: `/api/user/${ username }`, + reqId: options?.reqId + }) + + if ( result.status === 200 ) { + this.cache.set(key, result, { ttl: GET_USER_POSITIVE_TTL_MS }) + } + if ( result.status === 404 ) { + this.cache.set(key, result, { ttl: GET_USER_NEGATIVE_TTL_MS }) + } + + return result + } + + async getUserTweets(userId: string, cursor?: string, options?: { reqId?: string }) { + const key = `users:${userId}:tweets:${cursor ?? 'last'}` + + if ( this.cache.has(key) ) { + return this.cache.get(key) + } + + const result = await this.queue.push({ + url: `/api/user/${ userId }/tweets`, + params: { cursor }, + reqId: options?.reqId + }) + + if ( result.status === 200 ) { + this.cache.set(key, result, { ttl: GET_TWEETS_POSITIVE_TTL_MS }) + } + if ( result.status === 404 ) { + this.cache.set(key, result, { ttl: GET_TWEETS_NEGATIVE_TTL_MS }) + } + + return result + } + + async getTweetById(tweetId: string, options?: { reqId?: string }) { + const key = `tweets:${tweetId}` + + if ( this.cache.has(key) ) { + return this.cache.get(key) + } + + const result = await this.queue.push({ + url: `/api/tweet/${ tweetId }`, + reqId: options?.reqId + }) + + if ( result.status === 200 ) { + this.cache.set(key, result, { ttl: GET_TWEET_POSITIVE_TTL_MS }) + } + if ( result.status === 404 ) { + this.cache.set(key, result, { ttl: GET_TWEET_NEGATIVE_TTL_MS }) + } + + return result + } + + private async sendRequest(job: Job): Promise { + + const { reqId, url, params } = job + + if ( this.counter.requests > this.concurrency * this.maxRequestsPerAccount ) { + return { + status: 429 + } + } + + let config = { + url, + method: "get", + baseURL: this.baseUrl, + params, + } as AxiosRequestConfig + + this.log.trace({ config, reqId: reqId }, 'sending request to nitter') + + try { + const response = await this.client.request(config) + + this.log.trace({ + status: response.status, + data: response.data, + reqId: reqId + }, 'nitter response') + + return { + status: response.status, + data: response.data, + } as JobResponse + + } catch(err) { + + this.log.warn({ err, reqId }, 'nitter error') + + if ( err.name === "AxiosError" ) { + + this.counter.requests = Number.MAX_SAFE_INTEGER + + return { + status: 429 + } as JobResponse + } + + return { + status: 500 + } + } + } +} diff --git a/proxy/src/types.d.ts b/proxy/src/types.d.ts new file mode 100644 index 0000000..a0c18fa --- /dev/null +++ b/proxy/src/types.d.ts @@ -0,0 +1,39 @@ +declare module 'axios-retry-after' { + + import { AxiosError, AxiosInstance } from "axios"; + + /** + * Function to enhance Axios instance with retry-after functionality. + * @param axios Axios instance to be enhanced. + * @param options Configuration options for retry behavior. + */ + export default function( + axios: AxiosInstance, + options?: AxiosRetryAfterOptions + ): (error: AxiosError) => Promise; + + /** + * Configuration options for axios-retry-after. + */ + export interface AxiosRetryAfterOptions { + /** + * Function to determine if an error response is retryable. + * @param error The Axios error to evaluate. + */ + isRetryable?: (error: AxiosError) => boolean; + + /** + * Function to wait for a specified amount of time. + * @param error The Axios error that contains retry-after header. + */ + wait?: (error: AxiosError) => Promise; + + /** + * Function to retry the original request. + * @param axios The Axios instance used for retrying the request. + * @param error The Axios error to retry. + */ + retry?: (axios: AxiosInstance, error: AxiosError) => Promise; + } +} + diff --git a/proxy/tsconfig.json b/proxy/tsconfig.json new file mode 100644 index 0000000..12c5931 --- /dev/null +++ b/proxy/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "lib": ["esnext"], + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "module": "commonjs", + "target": "esnext", + "allowJs": true, + "sourceMap": true, + "outDir": "./build" + }, + "include": ["src/**/*"], + "exclude": [ + "node_modules" + ] +} From af1d873de0ee79f6571b590cdea48f479a594964 Mon Sep 17 00:00:00 2001 From: taskylizard <75871323+taskylizard@users.noreply.github.com> Date: Sun, 19 May 2024 06:37:13 +0000 Subject: [PATCH 06/12] take into account authorizationError for skipping account https://github.com/unixfox/nitter-fork/commit/351337aade25275b7b74125575f9b40df7b64887 --- src/apiutils.nim | 4 ++-- src/types.nim | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/apiutils.nim b/src/apiutils.nim index 774dcb5..2574485 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -109,7 +109,7 @@ template fetchImpl(result, additional_headers, fetchBody) {.dirty.} = if result.startsWith("{\"errors"): let errors = result.fromJson(Errors) - if errors in {expiredToken, badToken}: + if errors in {expiredToken, badToken, authorizationError}: echo "fetch error: ", errors invalidate(account) raise rateLimitError() @@ -159,7 +159,7 @@ proc fetch*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders result = newJNull() let error = result.getError - if error in {expiredToken, badToken}: + if error in {expiredToken, badToken, authorizationError}: echo "fetchBody error: ", error invalidate(account) raise rateLimitError() diff --git a/src/types.nim b/src/types.nim index a99aed5..7b4923c 100644 --- a/src/types.nim +++ b/src/types.nim @@ -62,9 +62,10 @@ type tweetNotAuthorized = 179 forbidden = 200 badToken = 239 + authorizationError = 326 noCsrf = 353 tweetUnavailable = 421 - tweetCensored = 422 + tweetCensored = 426 VerifiedType* = enum none = "None" From a451755d2bf36b6923b1f3a9ce6fb9fa7b265931 Mon Sep 17 00:00:00 2001 From: taskylizard <75871323+taskylizard@users.noreply.github.com> Date: Sun, 19 May 2024 07:05:01 +0000 Subject: [PATCH 07/12] format --- proxy/.prettierignore | 2 + proxy/.prettierrc.yaml | 6 + proxy/package.json | 4 +- proxy/pnpm-lock.yaml | 10 ++ proxy/src/index.ts | 161 ++++++++++-------- proxy/src/proxy.ts | 361 +++++++++++++++++++++-------------------- proxy/src/types.d.ts | 58 ++++--- proxy/tsconfig.json | 4 +- 8 files changed, 333 insertions(+), 273 deletions(-) create mode 100644 proxy/.prettierignore create mode 100644 proxy/.prettierrc.yaml diff --git a/proxy/.prettierignore b/proxy/.prettierignore new file mode 100644 index 0000000..18f1a04 --- /dev/null +++ b/proxy/.prettierignore @@ -0,0 +1,2 @@ +**/*.md +pnpm-lock.yaml diff --git a/proxy/.prettierrc.yaml b/proxy/.prettierrc.yaml new file mode 100644 index 0000000..88e0a15 --- /dev/null +++ b/proxy/.prettierrc.yaml @@ -0,0 +1,6 @@ +proseWrap: always +semi: false +singleQuote: true +printWidth: 80 +trailingComma: none +htmlWhitespaceSensitivity: ignore diff --git a/proxy/package.json b/proxy/package.json index 0092699..0aa594b 100644 --- a/proxy/package.json +++ b/proxy/package.json @@ -4,6 +4,7 @@ "scripts": { "clean": "rm -rf build", "prebuild": "npm run clean", + "format": "prettier -w --cache --check .", "build": "tsc --build" }, "author": "", @@ -22,6 +23,7 @@ }, "devDependencies": { "@types/node": "^20.12.12", - "dotenv": "^16.4.4" + "dotenv": "^16.4.4", + "prettier": "^3.2.5" } } diff --git a/proxy/pnpm-lock.yaml b/proxy/pnpm-lock.yaml index 2d53adc..1876821 100644 --- a/proxy/pnpm-lock.yaml +++ b/proxy/pnpm-lock.yaml @@ -45,6 +45,9 @@ importers: dotenv: specifier: ^16.4.4 version: 16.4.4 + prettier: + specifier: ^3.2.5 + version: 3.2.5 packages: @@ -292,6 +295,11 @@ packages: resolution: {integrity: sha512-Mz/gKiRyuXu4HnpHgi1YWdHQCoWMufapzooisvFn78zl4dZciAxS+YeRkUxXl1ee/SzU80YCz1zpECCh4oC6Aw==} hasBin: true + prettier@3.2.5: + resolution: {integrity: sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A==} + engines: {node: '>=14'} + hasBin: true + process-warning@2.3.2: resolution: {integrity: sha512-n9wh8tvBe5sFmsqlg+XQhaQLumwpqoAUruLwjCopgTmUBjJ/fjtBsJzKleCaIGBOMXYEhp1YfKl4d7rJ5ZKJGA==} @@ -668,6 +676,8 @@ snapshots: sonic-boom: 3.8.0 thread-stream: 2.4.1 + prettier@3.2.5: {} + process-warning@2.3.2: {} process-warning@3.0.0: {} diff --git a/proxy/src/index.ts b/proxy/src/index.ts index 020c5ef..b7c121d 100644 --- a/proxy/src/index.ts +++ b/proxy/src/index.ts @@ -1,89 +1,116 @@ - import fastify, { - FastifyInstance, - FastifyListenOptions, - FastifyReply, - FastifyRequest, -} from "fastify" -import { PinoLoggerOptions } from "fastify/types/logger" -import { Proxy } from "./proxy" -import { Logger } from "pino" + FastifyInstance, + FastifyListenOptions, + FastifyReply, + FastifyRequest +} from 'fastify' +import { PinoLoggerOptions } from 'fastify/types/logger' +import { Proxy } from './proxy' +import { Logger } from 'pino' import 'dotenv/config' const host = process.env.HOST -const port = parseInt(process.env.PORT ?? "8080", 10) +const port = parseInt(process.env.PORT ?? '8080', 10) const baseUrl = process.env.NITTER_BASE_URL -const concurrency = parseInt(process.env.CONCURRENCY ?? "1", 10) -const retryAfterMillis = process.env.RETRY_AFTER_MILLIS ? parseInt(process.env.RETRY_AFTER_MILLIS, 10) : null -const maxCacheSize = parseInt(process.env.MAX_CACHE_SIZE ?? "100000", 10) -const logLevel = process.env.LOG_LEVEL ?? "debug" +const concurrency = parseInt(process.env.CONCURRENCY ?? '1', 10) +const retryAfterMillis = process.env.RETRY_AFTER_MILLIS + ? parseInt(process.env.RETRY_AFTER_MILLIS, 10) + : null +const maxCacheSize = parseInt(process.env.MAX_CACHE_SIZE ?? '100000', 10) +const logLevel = process.env.LOG_LEVEL ?? 'debug' const server = fastify({ - logger: { - name: "app", - level: logLevel, - ...( logLevel == "trace" ? { transport: { target: 'pino-pretty' } } : {}) - } as PinoLoggerOptions + logger: { + name: 'app', + level: logLevel, + ...(logLevel == 'trace' ? { transport: { target: 'pino-pretty' } } : {}) + } as PinoLoggerOptions }) const log = server.log as Logger -const proxy = new Proxy(log, baseUrl, concurrency, retryAfterMillis, maxCacheSize) +const proxy = new Proxy( + log, + baseUrl, + concurrency, + retryAfterMillis, + maxCacheSize +) async function main() { + server.register( + (fastify: FastifyInstance, opts, done) => { + fastify.get( + `/user/:username`, + {}, + async (request: FastifyRequest, reply: FastifyReply) => { + log.debug( + { + headers: request.headers, + reqId: request.id, + params: request.params + }, + 'incoming request /user/:username' + ) + const { username } = request.params as any + const { status, data } = await proxy.getUser(username, { + reqId: request.id + }) + reply.status(status).send(data) + } + ) - server.register((fastify: FastifyInstance, opts, done) => { + fastify.get( + `/user/:userId/tweets`, + {}, + async (request: FastifyRequest, reply: FastifyReply) => { + const { userId } = request.params as any + const { cursor } = request.query as any + const { status, data } = await proxy.getUserTweets(userId, cursor, { + reqId: request.id + }) + reply.status(status).send(data) + } + ) - fastify.get(`/user/:username`, {}, - async (request: FastifyRequest, reply: FastifyReply) => { - log.debug({ - headers: request.headers, - reqId: request.id, - params: request.params }, 'incoming request /user/:username') - const { username } = request.params as any - const { status, data } = await proxy.getUser(username, { reqId: request.id }) - reply.status(status).send(data) - }); + fastify.get( + `/tweet/:id`, + {}, + async (request: FastifyRequest, reply: FastifyReply) => { + const { id } = request.params as any + const { status, data } = await proxy.getTweetById(id, { + reqId: request.id + }) + reply.status(status).send(data) + } + ) - fastify.get(`/user/:userId/tweets`, {}, - async (request: FastifyRequest, reply: FastifyReply) => { - const { userId } = request.params as any - const { cursor } = request.query as any - const { status, data } = await proxy.getUserTweets(userId, cursor, { reqId: request.id }) - reply.status(status).send(data) - }); + done() + }, + { prefix: '/api' } + ) - fastify.get(`/tweet/:id`, {}, - async (request: FastifyRequest, reply: FastifyReply) => { - const { id } = request.params as any - const { status, data } = await proxy.getTweetById(id, { reqId: request.id }) - reply.status(status).send(data) - }); + server.setNotFoundHandler((request: FastifyRequest, reply: FastifyReply) => { + reply.status(404).send({ message: `Method not found` }) + }) - done() + server.setErrorHandler( + (err: Error, request: FastifyRequest, reply: FastifyReply) => { + const { log } = request + log.error(err) + // Send error response + reply.status(500).send({ message: `Internal server error` }) + } + ) - }, { prefix: '/api' }) + // await server.register(import('@fastify/rate-limit'), { + // max: 100, + // timeWindow: '1 minute' + // }) - server.setNotFoundHandler((request: FastifyRequest, reply: FastifyReply) => { - reply.status(404) - .send({ message: `Method not found` }) - }) - - server.setErrorHandler((err: Error, request: FastifyRequest, reply: FastifyReply) => { - const { log } = request - log.error(err) - // Send error response - reply.status(500).send({ message: `Internal server error` }) - }) - - // await server.register(import('@fastify/rate-limit'), { - // max: 100, - // timeWindow: '1 minute' - // }) - - await server.listen({ port, host } as FastifyListenOptions); + await server.listen({ port, host } as FastifyListenOptions) } -main().catch(err => { - log.fatal(err) - process.exit(1) +main().catch((err) => { + log.fatal(err) + process.exit(1) }) diff --git a/proxy/src/proxy.ts b/proxy/src/proxy.ts index 872f481..a9fb9e5 100644 --- a/proxy/src/proxy.ts +++ b/proxy/src/proxy.ts @@ -1,209 +1,226 @@ // noinspection TypeScriptUnresolvedReference -import axios from "axios" -import { AxiosInstance, AxiosRequestConfig } from "axios" -import fastq from "fastq" -import { Logger } from "pino" -import retry from "axios-retry-after" +import axios from 'axios' +import { AxiosInstance, AxiosRequestConfig } from 'axios' +import fastq from 'fastq' +import { Logger } from 'pino' +import retry from 'axios-retry-after' import { LRUCache } from 'lru-cache' const GET_USER_POSITIVE_TTL_MS = process.env.GET_USER_POSITIVE_TTL - ? parseInt(process.env.GET_USER_POSITIVE_TTL, 10) * 1000 - : 30 * 24 * 3600 * 1000 + ? parseInt(process.env.GET_USER_POSITIVE_TTL, 10) * 1000 + : 30 * 24 * 3600 * 1000 const GET_USER_NEGATIVE_TTL_MS = process.env.GET_USER_NEGATIVE_TTL - ? parseInt(process.env.GET_USER_NEGATIVE_TTL, 10) * 1000 - : 3600 * 1000 + ? parseInt(process.env.GET_USER_NEGATIVE_TTL, 10) * 1000 + : 3600 * 1000 const GET_TWEETS_POSITIVE_TTL_MS = process.env.GET_TWEETS_POSITIVE_TTL - ? parseInt(process.env.GET_TWEETS_POSITIVE_TTL, 10) * 1000 - : 60 * 1000 + ? parseInt(process.env.GET_TWEETS_POSITIVE_TTL, 10) * 1000 + : 60 * 1000 const GET_TWEETS_NEGATIVE_TTL_MS = process.env.GET_TWEETS_NEGATIVE_TTL - ? parseInt(process.env.GET_TWEETS_NEGATIVE_TTL, 10) * 1000 - : 60 * 1000 + ? parseInt(process.env.GET_TWEETS_NEGATIVE_TTL, 10) * 1000 + : 60 * 1000 const GET_TWEET_POSITIVE_TTL_MS = process.env.GET_TWEET_POSITIVE_TTL - ? parseInt(process.env.GET_TWEET_POSITIVE_TTL, 10) * 1000 - : 60 * 1000 + ? parseInt(process.env.GET_TWEET_POSITIVE_TTL, 10) * 1000 + : 60 * 1000 const GET_TWEET_NEGATIVE_TTL_MS = process.env.GET_TWEET_NEGATIVE_TTL - ? parseInt(process.env.GET_TWEET_NEGATIVE_TTL, 10) * 1000 - : 60 * 1000 + ? parseInt(process.env.GET_TWEET_NEGATIVE_TTL, 10) * 1000 + : 60 * 1000 export interface Job { - reqId: string - url: string - params?: Record + reqId: string + url: string + params?: Record } export interface JobResponse { - status: number, - data: any + status: number + data: any } export class Proxy { + private readonly cache: LRUCache + private readonly client: AxiosInstance + private readonly queue: fastq.queueAsPromised + private counter: { requests: number } + private timeWindowMillis = 15 * 60 * 1000 + private maxRequestsPerAccount = 15 * 60 - private readonly cache: LRUCache - private readonly client: AxiosInstance - private readonly queue: fastq.queueAsPromised - private counter: { requests: number } - private timeWindowMillis = 15 * 60 * 1000 - private maxRequestsPerAccount = 15 * 60 - - constructor( - private log: Logger, - private baseUrl: string, - private concurrency: number, - retryAfterMillis: number, - maxCacheSize: number - ) { - this.cache = new LRUCache({ max: maxCacheSize }) - this.queue = fastq.promise(this, this.sendRequest, concurrency) - this.client = axios.create() - this.counter = { - requests: 0 - } - - setInterval(() => { - this.counter.requests = 0 - }, this.timeWindowMillis) - - if ( retryAfterMillis ) { - this.client.interceptors.response.use(null, retry(this.client, { - // Determine when we should attempt to retry - isRetryable (error) { - log.debug({ status: error.response?.status, headers: error.response?.headers }, 'checking retryable') - return ( - error.response && error.response.status === 429 - // Use X-Retry-After rather than Retry-After, and cap retry delay at 60 seconds - // && error.response.headers['x-retry-after'] && error.response.headers['x-retry-after'] <= 60 - ) - }, - // Customize the wait behavior - wait (error) { - log.debug({ status: error.response?.status, headers: error.response?.headers }, 'waiting for retry') - return new Promise( - // Use X-Retry-After rather than Retry-After - // resolve => setTimeout(resolve, error.response.headers['x-retry-after']) - resolve => setTimeout(resolve, retryAfterMillis) - ) - } - })) - } + constructor( + private log: Logger, + private baseUrl: string, + private concurrency: number, + retryAfterMillis: number, + maxCacheSize: number + ) { + this.cache = new LRUCache({ max: maxCacheSize }) + this.queue = fastq.promise(this, this.sendRequest, concurrency) + this.client = axios.create() + this.counter = { + requests: 0 } - async getUser(username: string, options?: { reqId?: string }) { - const key = `usernames:${username}` + setInterval(() => { + this.counter.requests = 0 + }, this.timeWindowMillis) - if ( this.cache.has(key)) { - return this.cache.get(key) - } - - const result = await this.queue.push({ - url: `/api/user/${ username }`, - reqId: options?.reqId + if (retryAfterMillis) { + this.client.interceptors.response.use( + null, + retry(this.client, { + // Determine when we should attempt to retry + isRetryable(error) { + log.debug( + { + status: error.response?.status, + headers: error.response?.headers + }, + 'checking retryable' + ) + return ( + error.response && error.response.status === 429 + // Use X-Retry-After rather than Retry-After, and cap retry delay at 60 seconds + // && error.response.headers['x-retry-after'] && error.response.headers['x-retry-after'] <= 60 + ) + }, + // Customize the wait behavior + wait(error) { + log.debug( + { + status: error.response?.status, + headers: error.response?.headers + }, + 'waiting for retry' + ) + return new Promise( + // Use X-Retry-After rather than Retry-After + // resolve => setTimeout(resolve, error.response.headers['x-retry-after']) + (resolve) => setTimeout(resolve, retryAfterMillis) + ) + } }) + ) + } + } - if ( result.status === 200 ) { - this.cache.set(key, result, { ttl: GET_USER_POSITIVE_TTL_MS }) - } - if ( result.status === 404 ) { - this.cache.set(key, result, { ttl: GET_USER_NEGATIVE_TTL_MS }) - } + async getUser(username: string, options?: { reqId?: string }) { + const key = `usernames:${username}` - return result + if (this.cache.has(key)) { + return this.cache.get(key) } - async getUserTweets(userId: string, cursor?: string, options?: { reqId?: string }) { - const key = `users:${userId}:tweets:${cursor ?? 'last'}` + const result = await this.queue.push({ + url: `/api/user/${username}`, + reqId: options?.reqId + }) - if ( this.cache.has(key) ) { - return this.cache.get(key) - } - - const result = await this.queue.push({ - url: `/api/user/${ userId }/tweets`, - params: { cursor }, - reqId: options?.reqId - }) - - if ( result.status === 200 ) { - this.cache.set(key, result, { ttl: GET_TWEETS_POSITIVE_TTL_MS }) - } - if ( result.status === 404 ) { - this.cache.set(key, result, { ttl: GET_TWEETS_NEGATIVE_TTL_MS }) - } - - return result + if (result.status === 200) { + this.cache.set(key, result, { ttl: GET_USER_POSITIVE_TTL_MS }) + } + if (result.status === 404) { + this.cache.set(key, result, { ttl: GET_USER_NEGATIVE_TTL_MS }) } - async getTweetById(tweetId: string, options?: { reqId?: string }) { - const key = `tweets:${tweetId}` + return result + } - if ( this.cache.has(key) ) { - return this.cache.get(key) - } + async getUserTweets( + userId: string, + cursor?: string, + options?: { reqId?: string } + ) { + const key = `users:${userId}:tweets:${cursor ?? 'last'}` - const result = await this.queue.push({ - url: `/api/tweet/${ tweetId }`, - reqId: options?.reqId - }) - - if ( result.status === 200 ) { - this.cache.set(key, result, { ttl: GET_TWEET_POSITIVE_TTL_MS }) - } - if ( result.status === 404 ) { - this.cache.set(key, result, { ttl: GET_TWEET_NEGATIVE_TTL_MS }) - } - - return result + if (this.cache.has(key)) { + return this.cache.get(key) } - private async sendRequest(job: Job): Promise { + const result = await this.queue.push({ + url: `/api/user/${userId}/tweets`, + params: { cursor }, + reqId: options?.reqId + }) - const { reqId, url, params } = job - - if ( this.counter.requests > this.concurrency * this.maxRequestsPerAccount ) { - return { - status: 429 - } - } - - let config = { - url, - method: "get", - baseURL: this.baseUrl, - params, - } as AxiosRequestConfig - - this.log.trace({ config, reqId: reqId }, 'sending request to nitter') - - try { - const response = await this.client.request(config) - - this.log.trace({ - status: response.status, - data: response.data, - reqId: reqId - }, 'nitter response') - - return { - status: response.status, - data: response.data, - } as JobResponse - - } catch(err) { - - this.log.warn({ err, reqId }, 'nitter error') - - if ( err.name === "AxiosError" ) { - - this.counter.requests = Number.MAX_SAFE_INTEGER - - return { - status: 429 - } as JobResponse - } - - return { - status: 500 - } - } + if (result.status === 200) { + this.cache.set(key, result, { ttl: GET_TWEETS_POSITIVE_TTL_MS }) } + if (result.status === 404) { + this.cache.set(key, result, { ttl: GET_TWEETS_NEGATIVE_TTL_MS }) + } + + return result + } + + async getTweetById(tweetId: string, options?: { reqId?: string }) { + const key = `tweets:${tweetId}` + + if (this.cache.has(key)) { + return this.cache.get(key) + } + + const result = await this.queue.push({ + url: `/api/tweet/${tweetId}`, + reqId: options?.reqId + }) + + if (result.status === 200) { + this.cache.set(key, result, { ttl: GET_TWEET_POSITIVE_TTL_MS }) + } + if (result.status === 404) { + this.cache.set(key, result, { ttl: GET_TWEET_NEGATIVE_TTL_MS }) + } + + return result + } + + private async sendRequest(job: Job): Promise { + const { reqId, url, params } = job + + if (this.counter.requests > this.concurrency * this.maxRequestsPerAccount) { + return { + status: 429 + } + } + + let config = { + url, + method: 'get', + baseURL: this.baseUrl, + params + } as AxiosRequestConfig + + this.log.trace({ config, reqId: reqId }, 'sending request to nitter') + + try { + const response = await this.client.request(config) + + this.log.trace( + { + status: response.status, + data: response.data, + reqId: reqId + }, + 'nitter response' + ) + + return { + status: response.status, + data: response.data + } as JobResponse + } catch (err) { + this.log.warn({ err, reqId }, 'nitter error') + + if (err.name === 'AxiosError') { + this.counter.requests = Number.MAX_SAFE_INTEGER + + return { + status: 429 + } as JobResponse + } + + return { + status: 500 + } + } + } } diff --git a/proxy/src/types.d.ts b/proxy/src/types.d.ts index a0c18fa..cd9cf13 100644 --- a/proxy/src/types.d.ts +++ b/proxy/src/types.d.ts @@ -1,39 +1,37 @@ declare module 'axios-retry-after' { + import { AxiosError, AxiosInstance } from 'axios' - import { AxiosError, AxiosInstance } from "axios"; + /** + * Function to enhance Axios instance with retry-after functionality. + * @param axios Axios instance to be enhanced. + * @param options Configuration options for retry behavior. + */ + export default function ( + axios: AxiosInstance, + options?: AxiosRetryAfterOptions + ): (error: AxiosError) => Promise + + /** + * Configuration options for axios-retry-after. + */ + export interface AxiosRetryAfterOptions { + /** + * Function to determine if an error response is retryable. + * @param error The Axios error to evaluate. + */ + isRetryable?: (error: AxiosError) => boolean /** - * Function to enhance Axios instance with retry-after functionality. - * @param axios Axios instance to be enhanced. - * @param options Configuration options for retry behavior. + * Function to wait for a specified amount of time. + * @param error The Axios error that contains retry-after header. */ - export default function( - axios: AxiosInstance, - options?: AxiosRetryAfterOptions - ): (error: AxiosError) => Promise; + wait?: (error: AxiosError) => Promise /** - * Configuration options for axios-retry-after. + * Function to retry the original request. + * @param axios The Axios instance used for retrying the request. + * @param error The Axios error to retry. */ - export interface AxiosRetryAfterOptions { - /** - * Function to determine if an error response is retryable. - * @param error The Axios error to evaluate. - */ - isRetryable?: (error: AxiosError) => boolean; - - /** - * Function to wait for a specified amount of time. - * @param error The Axios error that contains retry-after header. - */ - wait?: (error: AxiosError) => Promise; - - /** - * Function to retry the original request. - * @param axios The Axios instance used for retrying the request. - * @param error The Axios error to retry. - */ - retry?: (axios: AxiosInstance, error: AxiosError) => Promise; - } + retry?: (axios: AxiosInstance, error: AxiosError) => Promise + } } - diff --git a/proxy/tsconfig.json b/proxy/tsconfig.json index 12c5931..ee6e859 100644 --- a/proxy/tsconfig.json +++ b/proxy/tsconfig.json @@ -12,7 +12,5 @@ "outDir": "./build" }, "include": ["src/**/*"], - "exclude": [ - "node_modules" - ] + "exclude": ["node_modules"] } From c53b8d4d8a2e9e37722eaee57369c082387550ba Mon Sep 17 00:00:00 2001 From: taskylizard <75871323+taskylizard@users.noreply.github.com> Date: Mon, 20 May 2024 13:01:40 +0000 Subject: [PATCH 08/12] feat(prefs): imgur & reddit link replacers --- src/formatters.nim | 9 +++++++++ src/prefs_impl.nim | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/src/formatters.nim b/src/formatters.nim index 3630917..ed17c80 100644 --- a/src/formatters.nim +++ b/src/formatters.nim @@ -21,6 +21,9 @@ let # Images aren't supported due to errors from Teddit when the image # wasn't first displayed via a post on the Teddit instance. + imgurRegex = re"((i|i.stack)\.)?imgur\.(com|io)" + mediumRegex = re"([a-zA-Z0-9_.-]+\.)?medium\.com" + wwwRegex = re"https?://(www[0-9]?\.)?" m3u8Regex = re"""url="(.+.m3u8)"""" userPicRegex = re"_(normal|bigger|mini|200x200|400x400)(\.[A-z]+)$" @@ -69,6 +72,12 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string = if prefs.replaceReddit in result and "/gallery/" in result: result = result.replace("/gallery/", "/comments/") + if prefs.replaceImgur.len > 0 and "imgur" in result: + result = result.replace(imgurRegex, prefs.replaceImgur) + + if prefs.replaceMedium.len > 0 and "medium.com" in result: + result = result.replace(mediumRegex, prefs.replaceMedium) + if absolute.len > 0 and "href" in result: result = result.replace("href=\"/", &"href=\"{absolute}/") diff --git a/src/prefs_impl.nim b/src/prefs_impl.nim index 8e2ac8f..053647d 100644 --- a/src/prefs_impl.nim +++ b/src/prefs_impl.nim @@ -107,6 +107,14 @@ genPrefs: "Reddit -> Teddit/Libreddit" placeholder: "Teddit hostname" + replaceImgur(input, ""): + "Imgur -> Rimgo" + placeholder: "Rimgo hostname" + + replaceMedium(input, ""): + "Medium -> Scribe" + placeholder: "Scribe hostname" + iterator allPrefs*(): Pref = for k, v in prefList: for pref in v: From 769cb16a6b7ecb303895ab50555929af8ef04d80 Mon Sep 17 00:00:00 2001 From: taskylizard <75871323+taskylizard@users.noreply.github.com> Date: Mon, 20 May 2024 15:03:29 +0000 Subject: [PATCH 09/12] feat: Timeline feed for followed accounts --- src/nitter.nim | 9 +- src/prefs_impl.nim | 5 + src/routes/follow.nim | 42 +++++++++ src/routes/home.nim | 49 ++++++++++ src/sass/profile/card.scss | 181 +++++++++++++++++++------------------ src/views/general.nim | 2 +- src/views/home.nim | 32 +++++++ src/views/profile.nim | 16 +++- src/views/renderutils.nim | 4 + src/views/timeline.nim | 11 ++- 10 files changed, 253 insertions(+), 98 deletions(-) create mode 100644 src/routes/follow.nim create mode 100644 src/routes/home.nim create mode 100644 src/views/home.nim diff --git a/src/nitter.nim b/src/nitter.nim index f976db2..62a6026 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -11,7 +11,7 @@ import types, config, prefs, formatters, redis_cache, http_pool, auth import views/[general, about] import routes/[ preferences, timeline, status, media, search, rss, list, debug, - unsupported, embed, resolver, router_utils] + unsupported, embed, resolver, router_utils, home,follow] const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances" const issuesUrl = "https://github.com/zedeus/nitter/issues" @@ -60,9 +60,6 @@ settings: reusePort = true routes: - get "/": - resp renderMain(renderSearch(), request, cfg, themePrefs()) - get "/about": resp renderMain(renderAbout(), request, cfg, themePrefs()) @@ -94,7 +91,9 @@ routes: const link = a("another instance", href = instancesUrl) resp Http429, showError( &"Instance has been rate limited.
Use {link} or try again later.", cfg) - + + extend home, "" + extend follow, "" extend rss, "" extend status, "" extend search, "" diff --git a/src/prefs_impl.nim b/src/prefs_impl.nim index 053647d..88c4e1e 100644 --- a/src/prefs_impl.nim +++ b/src/prefs_impl.nim @@ -50,6 +50,11 @@ macro genPrefs*(prefDsl: untyped) = const `name`*: PrefList = toOrderedTable(`table`) genPrefs: + Timeline: + following(input, ""): + "A comma-separated list of users to follow." + placeholder: "taskylizard,vercel,nodejs" + Display: theme(select, "Nitter"): "Theme" diff --git a/src/routes/follow.nim b/src/routes/follow.nim new file mode 100644 index 0000000..1e5450d --- /dev/null +++ b/src/routes/follow.nim @@ -0,0 +1,42 @@ +import jester, asyncdispatch, strutils, sequtils +import router_utils +import ../types + +export follow + +proc addUserToFollowing*(following, toAdd: string): string = + var updated = following.split(",") + if updated == @[""]: + return toAdd + elif toAdd in updated: + return following + else: + updated = concat(updated, @[toAdd]) + result = updated.join(",") + +proc removeUserFromFollowing*(following, remove: string): string = + var updated = following.split(",") + if updated == @[""]: + return "" + else: + updated = filter(updated, proc(x: string): bool = x != remove) + result = updated.join(",") + +proc createFollowRouter*(cfg: Config) = + router follow: + post "/follow/@name": + let + following = cookiePrefs().following + toAdd = @"name" + updated = addUserToFollowing(following, toAdd) + setCookie("following", updated, daysForward(if isEmptyOrWhitespace(updated): -10 else: 360), + httpOnly=true, secure=cfg.useHttps, path="/") + redirect(refPath()) + post "/unfollow/@name": + let + following = cookiePrefs().following + remove = @"name" + updated = removeUserFromFollowing(following, remove) + setCookie("following", updated, daysForward(360), + httpOnly=true, secure=cfg.useHttps, path="/") + redirect(refPath()) diff --git a/src/routes/home.nim b/src/routes/home.nim new file mode 100644 index 0000000..dcdbb5e --- /dev/null +++ b/src/routes/home.nim @@ -0,0 +1,49 @@ +import jester +import asyncdispatch, strutils, options, router_utils, timeline +import ".."/[prefs, types, utils, redis_cache] +import ../views/[general, home, search] + +export home + +proc showHome*(request: Request; query: Query; cfg: Config; prefs: Prefs; + after: string): Future[string] {.async.} = + let + timeline = await getGraphTweetSearch(query, after) + html = renderHome(timeline, prefs, getPath()) + return renderMain(html, request, cfg, prefs) + +proc createHomeRouter*(cfg: Config) = + router home: + get "/": + let + prefs = cookiePrefs() + after = getCursor() + names = getNames(prefs.following) + + var query = request.getQuery("", prefs.following) + query.fromUser = names + + if @"scroll".len > 0: + var timeline = await getGraphTweetSearch(query, after) + if timeline.content.len == 0: resp Http404 + timeline.beginning = true + resp $renderHome(timeline, prefs, getPath()) + + if names.len == 0: + resp renderMain(renderSearch(), request, cfg, themePrefs()) + resp (await showHome(request, query, cfg, prefs, after)) + get "/following": + let + prefs = cookiePrefs() + names = getNames(prefs.following) + var + profs: seq[User] + query = request.getQuery("", prefs.following) + query.fromUser = names + query.kind = userList + + for name in names: + let prof = await getCachedUser(name) + profs &= @[prof] + + resp renderMain(renderFollowing(query, profs, prefs), request, cfg, prefs) diff --git a/src/sass/profile/card.scss b/src/sass/profile/card.scss index 46a9679..98790a3 100644 --- a/src/sass/profile/card.scss +++ b/src/sass/profile/card.scss @@ -1,130 +1,139 @@ -@import '_variables'; -@import '_mixins'; +@import "_variables"; +@import "_mixins"; .profile-card { - flex-wrap: wrap; - background: var(--bg_panel); - padding: 12px; - display: flex; + flex-wrap: wrap; + background: var(--bg_panel); + padding: 12px; + display: flex; } .profile-card-info { - @include breakable; - width: 100%; + @include breakable; + width: 100%; } -.profile-card-tabs-name { - @include breakable; - max-width: 100%; +.profile-card-tabs-name-and-follow { + @include breakable; + width: 100%; + display: flex; + flex-wrap: wrap; + justify-content: space-between; +} + +.profile-card-follow-button { + float: none; } .profile-card-username { - @include breakable; - color: var(--fg_color); - font-size: 14px; - display: block; + @include breakable; + color: var(--fg_color); + font-size: 14px; + display: block; } .profile-card-fullname { - @include breakable; - color: var(--fg_color); - font-size: 16px; - font-weight: bold; - text-shadow: none; - max-width: 100%; + @include breakable; + color: var(--fg_color); + font-size: 16px; + font-weight: bold; + text-shadow: none; + max-width: 100%; } .profile-card-avatar { - display: inline-block; - position: relative; + display: inline-block; + position: relative; + width: 100%; + margin-right: 4px; + margin-bottom: 6px; + + &:after { + content: ""; + display: block; + margin-top: 100%; + } + + img { + box-sizing: border-box; + position: absolute; width: 100%; - margin-right: 4px; - margin-bottom: 6px; - - &:after { - content: ''; - display: block; - margin-top: 100%; - } - - img { - box-sizing: border-box; - position: absolute; - width: 100%; - height: 100%; - border: 4px solid var(--darker_grey); - background: var(--bg_panel); - } + height: 100%; + border: 4px solid var(--darker_grey); + background: var(--bg_panel); + } } .profile-card-extra { - display: contents; - flex: 100%; - margin-top: 7px; + display: contents; + flex: 100%; + margin-top: 7px; - .profile-bio { - @include breakable; - width: 100%; - margin: 4px -6px 6px 0; - white-space: pre-wrap; + .profile-bio { + @include breakable; + width: 100%; + margin: 4px -6px 6px 0; + white-space: pre-wrap; - p { - margin: 0; - } + p { + margin: 0; } + } - .profile-joindate, .profile-location, .profile-website { - color: var(--fg_faded); - margin: 1px 0; - width: 100%; - } + .profile-joindate, + .profile-location, + .profile-website { + color: var(--fg_faded); + margin: 1px 0; + width: 100%; + } } .profile-card-extra-links { - margin-top: 8px; - font-size: 14px; - width: 100%; + margin-top: 8px; + font-size: 14px; + width: 100%; } .profile-statlist { - display: flex; - flex-wrap: wrap; - padding: 0; - width: 100%; - justify-content: space-between; + display: flex; + flex-wrap: wrap; + padding: 0; + width: 100%; + justify-content: space-between; - li { - display: table-cell; - text-align: center; - } + li { + display: table-cell; + text-align: center; + } } .profile-stat-header { - font-weight: bold; - color: var(--profile_stat); + font-weight: bold; + color: var(--profile_stat); } .profile-stat-num { - display: block; - color: var(--profile_stat); + display: block; + color: var(--profile_stat); } -@media(max-width: 700px) { - .profile-card-info { - display: flex; - } +@media (max-width: 700px) { + .profile-card-info { + display: flex; + } - .profile-card-tabs-name { - flex-shrink: 100; - } + .profile-card-tabs-name { + flex-shrink: 100; + } - .profile-card-avatar { - width: 80px; - height: 80px; + .profile-card-avatar { + width: 98px; + height: auto; - img { - border-width: 2px; - width: unset; - } + img { + border-width: 2px; + width: unset; } + } } diff --git a/src/views/general.nim b/src/views/general.nim index 87d30f2..3e32f01 100644 --- a/src/views/general.nim +++ b/src/views/general.nim @@ -52,7 +52,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; let opensearchUrl = getUrlPrefix(cfg) & "/opensearch" buildHtml(head): - link(rel="stylesheet", type="text/css", href="/css/style.css?v=19") + link(rel="stylesheet", type="text/css", href="/css/style.css?v=20") link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2") if theme.len > 0: diff --git a/src/views/home.nim b/src/views/home.nim new file mode 100644 index 0000000..57acd2c --- /dev/null +++ b/src/views/home.nim @@ -0,0 +1,32 @@ +import karax/[karaxdsl, vdom] +import search, timeline, renderutils +import ../types + +proc renderFollowingUsers*(results: seq[User]; prefs: Prefs): VNode = + buildHtml(tdiv(class="timeline")): + for user in results: + renderUser(user, prefs) + +proc renderHomeTabs*(query: Query): VNode = + buildHtml(ul(class="tab")): + li(class=query.getTabClass(posts)): + a(href="/"): text "Tweets" + li(class=query.getTabClass(userList)): + a(href=("/following")): text "Following" + +proc renderHome*(results: Timeline; prefs: Prefs; path: string): VNode = + let query = results.query + buildHtml(tdiv(class="timeline-container")): + if query.fromUser.len > 0: + renderHomeTabs(query) + + if query.fromUser.len == 0 or query.kind == tweets: + tdiv(class="timeline-header"): + renderSearchPanel(query) + + renderTimelineTweets(results, prefs, path) + +proc renderFollowing*(query: Query; following: seq[User]; prefs: Prefs): VNode = + buildHtml(tdiv(class="timeline-container")): + renderHomeTabs(query) + renderFollowingUsers(following, prefs) diff --git a/src/views/profile.nim b/src/views/profile.nim index 2ec79f7..f35e7cc 100644 --- a/src/views/profile.nim +++ b/src/views/profile.nim @@ -12,7 +12,7 @@ proc renderStat(num: int; class: string; text=""): VNode = span(class="profile-stat-num"): text insertSep($num, ',') -proc renderUserCard*(user: User; prefs: Prefs): VNode = +proc renderUserCard*(user: User; prefs: Prefs; path: string): VNode = buildHtml(tdiv(class="profile-card")): tdiv(class="profile-card-info"): let @@ -24,9 +24,15 @@ proc renderUserCard*(user: User; prefs: Prefs): VNode = a(class="profile-card-avatar", href=url, target="_blank"): genImg(user.getUserPic(size)) - tdiv(class="profile-card-tabs-name"): - linkUser(user, class="profile-card-fullname") - linkUser(user, class="profile-card-username") + tdiv(class="profile-card-tabs-name-and-follow"): + tdiv(): + linkUser(user, class="profile-card-fullname") + linkUser(user, class="profile-card-username") + let following = isFollowing(user.username, prefs.following) + if not following: + buttonReferer "/follow/" & user.username, "Follow", path, "profile-card-follow-button" + else: + buttonReferer "/unfollow/" & user.username, "Unfollow", path, "profile-card-follow-button" tdiv(class="profile-card-extra"): if user.bio.len > 0: @@ -113,7 +119,7 @@ proc renderProfile*(profile: var Profile; cfg: Config; prefs: Prefs; path: strin let sticky = if prefs.stickyProfile: " sticky" else: "" tdiv(class=("profile-tab" & sticky)): - renderUserCard(profile.user, prefs) + renderUserCard(profile.user, prefs, path) if profile.photoRail.len > 0: renderPhotoRail(profile) diff --git a/src/views/renderutils.nim b/src/views/renderutils.nim index f298fad..86a6806 100644 --- a/src/views/renderutils.nim +++ b/src/views/renderutils.nim @@ -100,3 +100,7 @@ proc getTabClass*(query: Query; tab: QueryKind): string = proc getAvatarClass*(prefs: Prefs): string = if prefs.squareAvatars: "avatar" else: "avatar round" + +proc isFollowing*(name, following: string): bool = + let following = following.split(",") + return name in following diff --git a/src/views/timeline.nim b/src/views/timeline.nim index abeb6d3..305129b 100644 --- a/src/views/timeline.nim +++ b/src/views/timeline.nim @@ -55,7 +55,16 @@ proc renderThread(thread: Tweets; prefs: Prefs; path: string): VNode = renderTweet(tweet, prefs, path, class=(header & "thread"), index=i, last=(i == thread.high), showThread=show) -proc renderUser(user: User; prefs: Prefs): VNode = +proc threadFilter(tweets: openArray[Tweet]; threads: openArray[int64]; it: Tweet): seq[Tweet] = + result = @[it] + if it.retweet.isSome or it.replyId in threads: return + for t in tweets: + if t.id == result[0].replyId: + result.insert t + elif t.replyId == result[0].id: + result.add t + +proc renderUser*(user: User; prefs: Prefs): VNode = buildHtml(tdiv(class="timeline-item")): a(class="tweet-link", href=("/" & user.username)) tdiv(class="tweet-body profile-result"): From a57b66a1f37d50ce3b9e481ac1caeeef1fabdafd Mon Sep 17 00:00:00 2001 From: taskylizard <75871323+taskylizard@users.noreply.github.com> Date: Tue, 21 May 2024 17:42:38 +0000 Subject: [PATCH 10/12] feat(profile): add `data-profile-id` on profile cards --- src/views/profile.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/views/profile.nim b/src/views/profile.nim index f35e7cc..b89bf43 100644 --- a/src/views/profile.nim +++ b/src/views/profile.nim @@ -13,7 +13,7 @@ proc renderStat(num: int; class: string; text=""): VNode = text insertSep($num, ',') proc renderUserCard*(user: User; prefs: Prefs; path: string): VNode = - buildHtml(tdiv(class="profile-card")): + buildHtml(tdiv(class="profile-card", "data-profile-id" = $user.id)): tdiv(class="profile-card-info"): let url = getPicUrl(user.getUserPic()) From 5895d2e276d7204f250adc206a6ffe1be694c4f0 Mon Sep 17 00:00:00 2001 From: taskylizard <75871323+taskylizard@users.noreply.github.com> Date: Mon, 10 Jun 2024 21:01:42 +0000 Subject: [PATCH 11/12] feat: image zooming --- public/css/baguetteBox.min.css | 6 ++++++ public/js/baguetteBox.min.js | 7 +++++++ public/js/zoom.js | 3 +++ src/views/general.nim | 3 +++ 4 files changed, 19 insertions(+) create mode 100644 public/css/baguetteBox.min.css create mode 100644 public/js/baguetteBox.min.js create mode 100644 public/js/zoom.js diff --git a/public/css/baguetteBox.min.css b/public/css/baguetteBox.min.css new file mode 100644 index 0000000..f1c5ca9 --- /dev/null +++ b/public/css/baguetteBox.min.css @@ -0,0 +1,6 @@ +/*! + * baguetteBox.js + * @author feimosi + * @version 1.11.1 + * @url https://github.com/feimosi/baguetteBox.js + */#baguetteBox-overlay{display:none;opacity:0;position:fixed;overflow:hidden;top:0;left:0;width:100%;height:100%;z-index:1000000;background-color:#222;background-color:rgba(0,0,0,.8);-webkit-transition:opacity .5s ease;transition:opacity .5s ease}#baguetteBox-overlay.visible{opacity:1}#baguetteBox-overlay .full-image{display:inline-block;position:relative;width:100%;height:100%;text-align:center}#baguetteBox-overlay .full-image figure{display:inline;margin:0;height:100%}#baguetteBox-overlay .full-image img{display:inline-block;width:auto;height:auto;max-height:100%;max-width:100%;vertical-align:middle;-webkit-box-shadow:0 0 8px rgba(0,0,0,.6);-moz-box-shadow:0 0 8px rgba(0,0,0,.6);box-shadow:0 0 8px rgba(0,0,0,.6)}#baguetteBox-overlay .full-image figcaption{display:block;position:absolute;bottom:0;width:100%;text-align:center;line-height:1.8;white-space:normal;color:#ccc;background-color:#000;background-color:rgba(0,0,0,.6);font-family:sans-serif}#baguetteBox-overlay .full-image:before{content:"";display:inline-block;height:50%;width:1px;margin-right:-1px}#baguetteBox-slider{position:absolute;left:0;top:0;height:100%;width:100%;white-space:nowrap;-webkit-transition:left .4s ease,-webkit-transform .4s ease;transition:left .4s ease,-webkit-transform .4s ease;transition:left .4s ease,transform .4s ease;transition:left .4s ease,transform .4s ease,-webkit-transform .4s ease,-moz-transform .4s ease}#baguetteBox-slider.bounce-from-right{-webkit-animation:bounceFromRight .4s ease-out;animation:bounceFromRight .4s ease-out}#baguetteBox-slider.bounce-from-left{-webkit-animation:bounceFromLeft .4s ease-out;animation:bounceFromLeft .4s ease-out}@-webkit-keyframes bounceFromRight{0%,100%{margin-left:0}50%{margin-left:-30px}}@keyframes bounceFromRight{0%,100%{margin-left:0}50%{margin-left:-30px}}@-webkit-keyframes bounceFromLeft{0%,100%{margin-left:0}50%{margin-left:30px}}@keyframes bounceFromLeft{0%,100%{margin-left:0}50%{margin-left:30px}}.baguetteBox-button#next-button,.baguetteBox-button#previous-button{top:50%;top:calc(50% - 30px);width:44px;height:60px}.baguetteBox-button{position:absolute;cursor:pointer;outline:0;padding:0;margin:0;border:0;-moz-border-radius:15%;border-radius:15%;background-color:#323232;background-color:rgba(50,50,50,.5);color:#ddd;font:1.6em sans-serif;-webkit-transition:background-color .4s ease;transition:background-color .4s ease}.baguetteBox-button:focus,.baguetteBox-button:hover{background-color:rgba(50,50,50,.9)}.baguetteBox-button#next-button{right:2%}.baguetteBox-button#previous-button{left:2%}.baguetteBox-button#close-button{top:20px;right:2%;right:calc(2% + 6px);width:30px;height:30px}.baguetteBox-button svg{position:absolute;left:0;top:0}.baguetteBox-spinner{width:40px;height:40px;display:inline-block;position:absolute;top:50%;left:50%;margin-top:-20px;margin-left:-20px}.baguetteBox-double-bounce1,.baguetteBox-double-bounce2{width:100%;height:100%;-moz-border-radius:50%;border-radius:50%;background-color:#fff;opacity:.6;position:absolute;top:0;left:0;-webkit-animation:bounce 2s infinite ease-in-out;animation:bounce 2s infinite ease-in-out}.baguetteBox-double-bounce2{-webkit-animation-delay:-1s;animation-delay:-1s}@-webkit-keyframes bounce{0%,100%{-webkit-transform:scale(0);transform:scale(0)}50%{-webkit-transform:scale(1);transform:scale(1)}}@keyframes bounce{0%,100%{-webkit-transform:scale(0);-moz-transform:scale(0);transform:scale(0)}50%{-webkit-transform:scale(1);-moz-transform:scale(1);transform:scale(1)}} \ No newline at end of file diff --git a/public/js/baguetteBox.min.js b/public/js/baguetteBox.min.js new file mode 100644 index 0000000..a6c7b04 --- /dev/null +++ b/public/js/baguetteBox.min.js @@ -0,0 +1,7 @@ +/*! + * baguetteBox.js + * @author feimosi + * @version 1.11.1 + * @url https://github.com/feimosi/baguetteBox.js + */ +!function(e,t){"use strict";"function"==typeof define&&define.amd?define(t):"object"==typeof exports?module.exports=t():e.baguetteBox=t()}(this,function(){"use strict";var r,l,u,c,d,f='',g='',p='',b={},v={captions:!0,buttons:"auto",fullScreen:!1,noScrollbars:!1,bodyClass:"baguetteBox-open",titleTag:!1,async:!1,preload:2,animation:"slideIn",afterShow:null,afterHide:null,onChange:null,overlayBackgroundColor:"rgba(0,0,0,.8)"},m={},h=[],o=0,n=!1,i={},a=!1,y=/.+\.(gif|jpe?g|png|webp)/i,w={},k=[],s=null,x=function(e){-1!==e.target.id.indexOf("baguette-img")&&j()},E=function(e){e.stopPropagation?e.stopPropagation():e.cancelBubble=!0,D()},C=function(e){e.stopPropagation?e.stopPropagation():e.cancelBubble=!0,X()},B=function(e){e.stopPropagation?e.stopPropagation():e.cancelBubble=!0,j()},T=function(e){i.count++,1