diff --git a/README.md b/README.md index ddef620..2856ff9 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ is on implementing missing features. ## Todo (roughly in this order) - Search (images/videos, hashtags, etc.) -- Hiding retweets, showing replies, etc. +- Custom timeline filter - Media carousel below profile - Media-only/gallery view - Nitter link previews diff --git a/public/style.css b/public/style.css index 5461abb..8bc0369 100644 --- a/public/style.css +++ b/public/style.css @@ -497,6 +497,39 @@ video { word-wrap: break-word; } +.tab { + align-items: center; + display: flex; + flex-wrap: wrap; + list-style: none; + margin: 0 0 5px 0; + background-color: #161616; + padding: 0; +} + +.tab .tab-item { + margin-top: 0; +} + +.tab-item { + flex: 1 1 0; + text-align: center; +} + +.tab .tab-item a.active, .tab .tab-item.active a { + border-bottom-color: #ff6c60; + color: #ff6c60; +} + +.tab .tab-item a { + border-bottom: .1rem solid transparent; + color: inherit; + display: block; + padding: 8px 0; + text-decoration: none; + font-weight: bold; +} + .conversation { max-width: 580px; margin: 0 auto; diff --git a/screenshot.png b/screenshot.png index ad20375..6b29dce 100644 Binary files a/screenshot.png and b/screenshot.png differ diff --git a/src/api.nim b/src/api.nim index 876f683..fbd4332 100644 --- a/src/api.nim +++ b/src/api.nim @@ -1,21 +1,20 @@ import httpclient, asyncdispatch, htmlparser, times -import sequtils, strutils, strformat, json, xmltree, uri -import regex +import sequtils, strutils, json, xmltree, uri -import ./types, ./parser, ./parserutils, ./formatters +import types, parser, parserutils, formatters, search const agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" lang = "en-US,en;q=0.9" auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw" cardAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3" + jsonAccept = "application/json, text/javascript, */*; q=0.01" base = parseUri("https://twitter.com/") apiBase = parseUri("https://api.twitter.com/1.1/") - timelineParams = "?include_available_features=1&include_entities=1&include_new_items_bar=false&reset_error_state=false" - showUrl = "i/profiles/show/$1" & timelineParams - timelineUrl = showUrl % "$1/timeline/tweets" + timelineUrl = "i/profiles/show/$1/timeline/tweets" + timelineSearchUrl = "i/search/timeline" profilePopupUrl = "i/profiles/popup" profileIntentUrl = "intent/user" tweetUrl = "status" @@ -70,7 +69,7 @@ proc getGuestToken(force=false): Future[string] {.async.} = tokenUses = 0 let headers = newHttpHeaders({ - "Accept": "application/json, text/javascript, */*; q=0.01", + "Accept": jsonAccept, "Referer": $base, "User-Agent": agent, "Authorization": auth @@ -89,7 +88,7 @@ proc getVideo*(tweet: Tweet; token: string) {.async.} = if tweet.video.isNone(): return let headers = newHttpHeaders({ - "Accept": "application/json, text/javascript, */*; q=0.01", + "Accept": jsonAccept, "Referer": $(base / getLink(tweet)), "User-Agent": agent, "Authorization": auth, @@ -196,45 +195,9 @@ proc getProfile*(username: string): Future[Profile] {.async.} = result = parsePopupProfile(html) -proc getTimeline*(username: string; after=""): Future[Timeline] {.async.} = +proc getTweet*(username, id: string): Future[Conversation] {.async.} = let headers = newHttpHeaders({ - "Accept": "application/json, text/javascript, */*; q=0.01", - "Referer": $(base / username), - "User-Agent": agent, - "X-Twitter-Active-User": "yes", - "X-Requested-With": "XMLHttpRequest", - "Accept-Language": lang - }) - - var url = timelineUrl % username - let cleanAfter = after.replace(re"[^\d]*(\d+)[^\d]*", "$1") - if cleanAfter.len > 0: - url &= "&max_position=" & cleanAfter - - let json = await fetchJson(base / url, headers) - if json == nil: return Timeline() - - result = Timeline( - hasMore: json["has_more_items"].to(bool), - maxId: json.getOrDefault("max_position").getStr(""), - minId: json.getOrDefault("min_position").getStr(""), - ) - - if json["new_latent_count"].to(int) == 0: return - if not json.hasKey("items_html"): return - - let - html = parseHtml(json["items_html"].to(string)) - thread = parseThread(html) - vidsFut = getVideos(thread) - pollFut = getPolls(thread) - - await all(vidsFut, pollFut) - result.tweets = thread.tweets - -proc getTweet*(username: string; id: string): Future[Conversation] {.async.} = - let headers = newHttpHeaders({ - "Accept": "application/json, text/javascript, */*; q=0.01", + "Accept": jsonAccept, "Referer": $base, "User-Agent": agent, "X-Twitter-Active-User": "yes", @@ -255,3 +218,72 @@ proc getTweet*(username: string; id: string): Future[Conversation] {.async.} = let vidsFut = getConversationVideos(result) let pollFut = getConversationPolls(result) await all(vidsFut, pollFut) + +proc finishTimeline(json: JsonNode; query: Option[Query]): Future[Timeline] {.async.} = + if json == nil: return Timeline() + + result = Timeline( + hasMore: json["has_more_items"].to(bool), + maxId: json.getOrDefault("max_position").getStr(""), + minId: json.getOrDefault("min_position").getStr("").cleanPos(), + ) + + if json["new_latent_count"].to(int) == 0: return + if not json.hasKey("items_html"): return + + let + html = parseHtml(json["items_html"].to(string)) + thread = parseThread(html) + vidsFut = getVideos(thread) + pollFut = getPolls(thread) + + await all(vidsFut, pollFut) + result.tweets = thread.tweets + result.query = query + +proc getTimeline*(username, after: string): Future[Timeline] {.async.} = + let headers = newHttpHeaders({ + "Accept": jsonAccept, + "Referer": $(base / username), + "User-Agent": agent, + "X-Twitter-Active-User": "yes", + "X-Requested-With": "XMLHttpRequest", + "Accept-Language": lang + }) + + var params = toSeq({ + "include_available_features": "1", + "include_entities": "1", + "include_new_items_bar": "false", + "reset_error_state": "false" + }) + + if after.len > 0: + params.add {"max_position": after} + + let json = await fetchJson(base / (timelineUrl % username) ? params, headers) + result = await finishTimeline(json, none(Query)) + +proc getTimelineSearch*(username, after: string; query: Query): Future[Timeline] {.async.} = + let headers = newHttpHeaders({ + "Accept": jsonAccept, + "Referer": $(base / ("search?f=tweets&q=from%3A$1&src=typd" % username)), + "User-Agent": agent, + "X-Requested-With": "XMLHttpRequest", + "Authority": "twitter.com", + "Accept-Language": lang + }) + + let params = { + "f": "tweets", + "vertical": "default", + "q": genQueryParam(query), + "src": "typd", + "include_available_features": "1", + "include_entities": "1", + "max_position": if after.len > 0: genPos(after) else: "0", + "reset_error_state": "false" + } + + let json = await fetchJson(base / timelineSearchUrl ? params, headers) + result = await finishTimeline(json, some(query)) diff --git a/src/formatters.nim b/src/formatters.nim index 5630e75..83b2835 100644 --- a/src/formatters.nim +++ b/src/formatters.nim @@ -1,7 +1,7 @@ import strutils, strformat, htmlgen, xmltree, times import regex -import ./types, ./utils +import types, utils from unicode import Rune, `$` diff --git a/src/nitter.nim b/src/nitter.nim index 9f10a00..11e5d11 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -1,26 +1,36 @@ import asyncdispatch, asyncfile, httpclient, strutils, strformat, uri, os -import jester +import jester, regex -import api, utils, types, cache, formatters +import api, utils, types, cache, formatters, search include views/"user.nimf" include views/"general.nimf" const cacheDir {.strdefine.} = "/tmp/nitter" -proc showTimeline(name: string; num=""): Future[string] {.async.} = +proc showTimeline(name, after: string; query: Option[Query]): Future[string] {.async.} = let username = name.strip(chars={'/'}) profileFut = getCachedProfile(username) - tweetsFut = getTimeline(username, after=num) + + var timelineFut: Future[Timeline] + if query.isNone: + timelineFut = getTimeline(username, after) + else: + timelineFut = getTimelineSearch(username, after, get(query)) let profile = await profileFut if profile.username.len == 0: return "" - let profileHtml = renderProfile(profile, await tweetsFut, num.len == 0) + let profileHtml = renderProfile(profile, await timelineFut, after.len == 0) return renderMain(profileHtml, title=pageTitle(profile)) +template respTimeline(timeline: typed) = + if timeline.len == 0: + resp Http404, showError("User \"" & @"name" & "\" not found") + resp timeline + routes: get "/": resp renderMain(renderSearchPanel(), title=pageTitle("Search")) @@ -28,17 +38,24 @@ routes: post "/search": if @"query".len == 0: resp Http404, showError("Please enter a username.") - redirect("/" & @"query") get "/@name/?": cond '.' notin @"name" + respTimeline(await showTimeline(@"name", @"after", none(Query))) - let timeline = await showTimeline(@"name", @"after") - if timeline.len == 0: - resp Http404, showError("User \"" & @"name" & "\" not found") + get "/@name/search/?": + cond '.' notin @"name" + let query = initQuery(@"filter", @"sep", @"name") + respTimeline(await showTimeline(@"name", @"after", some(query))) - resp timeline + get "/@name/replies": + cond '.' notin @"name" + respTimeline(await showTimeline(@"name", @"after", some(getReplyQuery(@"name")))) + + get "/@name/media": + cond '.' notin @"name" + respTimeline(await showTimeline(@"name", @"after", some(getMediaQuery(@"name")))) get "/@name/status/@id": cond '.' notin @"name" diff --git a/src/parser.nim b/src/parser.nim index 17f514e..cd9274a 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -1,6 +1,6 @@ import xmltree, sequtils, strtabs, strutils, strformat, json -import ./types, ./parserutils, ./formatters +import types, parserutils, formatters proc parsePopupProfile*(node: XmlNode): Profile = let profile = node.select(".profile-card") diff --git a/src/parserutils.nim b/src/parserutils.nim index 315c4f0..5d808f9 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -1,7 +1,7 @@ import xmltree, htmlparser, strtabs, strformat, times import regex -import ./types, ./formatters, ./api +import types, formatters, api from q import nil diff --git a/src/search.nim b/src/search.nim new file mode 100644 index 0000000..a4cd04a --- /dev/null +++ b/src/search.nim @@ -0,0 +1,80 @@ +import asyncdispatch, strutils, strformat, uri, tables + +import types + +const + separators = @["AND", "OR"] + validFilters = @[ + "media", "images", "videos", "native_video", "twimg", + "links", "quote", "replies", "mentions", + "news", "verified", "safe" + ] + +# Experimental, this might break in the future +# Till then, it results in shorter urls +const + posPrefix = "thGAVUV0VFVBa" + posSuffix = "EjUAFQAlAFUAFQAA" + +proc initQuery*(filter, separator: string; name=""): Query = + var sep = separator.strip().toUpper() + Query( + filter: filter.split(",").filterIt(it in validFilters), + sep: if sep in separators: sep else: "AND", + fromUser: name, + queryType: custom + ) + +proc getMediaQuery*(name: string): Query = + Query( + filter: @["twimg", "native_video"], + sep: "OR", + fromUser: name, + queryType: media + ) + +proc getReplyQuery*(name: string): Query = + Query(fromUser: name, queryType: replies) + +proc genQueryParam*(query: Query): string = + var filters: seq[string] + var param: string + + if query.fromUser.len > 0: + param = &"from:{query.fromUser} " + + for f in query.filter: + filters.add "filter:" & f + for e in query.exclude: + filters.add "-filter:" & e + + return strip(param & filters.join(&" {query.sep} ")) + +proc genQueryUrl*(query: Query): string = + result = &"/{query.queryType}?" + if query.queryType != custom: return + + var params: seq[string] + if query.filter.len > 0: + params &= "filter=" & query.filter.join(",") + if query.exclude.len > 0: + params &= "not=" & query.exclude.join(",") + if query.sep.len > 0: + params &= "sep=" & query.sep + if params.len > 0: + result &= params.join("&") & "&" + +proc cleanPos*(pos: string): string = + pos.multiReplace((posPrefix, ""), (posSuffix, "")) + +proc genPos*(pos: string): string = + posPrefix & pos & posSuffix + +proc tabClass*(timeline: Timeline; tab: string): string = + result = '"' & "tab-item" + if timeline.query.isNone: + if tab == "tweets": + result &= " active" + elif $timeline.query.get().queryType == tab: + result &= " active" + result &= '"' diff --git a/src/types.nim b/src/types.nim index 82b745b..575172e 100644 --- a/src/types.nim +++ b/src/types.nim @@ -31,6 +31,16 @@ db("cache.db", "", "", ""): .}: Time type + QueryType* = enum + replies, media, custom = "search" + + Query* = object + filter*: seq[string] + exclude*: seq[string] + sep*: string + fromUser*: string + queryType*: QueryType + VideoType* = enum vmap, m3u8, mp4 @@ -106,6 +116,7 @@ type minId*: string maxId*: string hasMore*: bool + query*: Option[Query] proc contains*(thread: Thread; tweet: Tweet): bool = thread.tweets.anyIt(it.id == tweet.id) diff --git a/src/views/general.nimf b/src/views/general.nimf index 31b325c..3f3fee4 100644 --- a/src/views/general.nimf +++ b/src/views/general.nimf @@ -45,5 +45,5 @@ #end proc # #proc showError*(error: string): string = -${renderMain(renderError(error), title="Error | Nitter")} +#renderMain(renderError(error), title="Error | Nitter") #end proc diff --git a/src/views/user.nimf b/src/views/user.nimf index e7ce059..d1027be 100644 --- a/src/views/user.nimf +++ b/src/views/user.nimf @@ -1,6 +1,6 @@ #? stdtmpl(subsChar = '$', metaChar = '#') #import xmltree, strutils, uri -#import ../types, ../formatters, ../utils +#import ../types, ../formatters, ../utils, ../search #include "tweet.nimf" # #proc renderProfileCard*(profile: Profile): string = @@ -52,10 +52,13 @@ # #proc renderTimeline*(timeline: Timeline; profile: Profile; beginning: bool): string = #var retweets: seq[string] +#var query = "?" +#if timeline.query.isSome: query = genQueryUrl(get(timeline.query)) +#end if
#if not beginning:
- Load newest tweets + Load newest tweets
#end if # @@ -66,9 +69,9 @@ ${renderTweet(tweet, "timeline-tweet")} #end for # - #if timeline.hasMore: + #if timeline.hasMore or timeline.query.isSome and timeline.tweets.len > 0:
- Load older tweets + Load older tweets
#elif timeline.tweets.len > 0:
@@ -96,6 +99,13 @@ ${renderProfileCard(profile)}
+ #let link = "/" & profile.username + ${renderTimeline(timeline, profile, beginning)}