Merge remote-tracking branch 'upstream/master'

This commit is contained in:
Kadin Buckton 2019-10-10 14:05:27 -04:00
commit ebb1e319cd
9 changed files with 33 additions and 27 deletions

View file

@ -23,8 +23,8 @@ proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.
let last = result.content[^1] let last = result.content[^1]
result.minId = result.minId =
if last.retweet.isNone: last.id if last.retweet.isNone: $last.id
else: get(last.retweet).id else: $(get(last.retweet).id)
proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.async.} = proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.async.} =
let let

View file

@ -68,7 +68,7 @@ proc getVideoFetch(tweet: Tweet; agent, token: string) {.async.} =
let let
headers = genHeaders({"authorization": auth, "x-guest-token": token}, headers = genHeaders({"authorization": auth, "x-guest-token": token},
agent, base / getLink(tweet), lang=false) agent, base / getLink(tweet), lang=false)
url = apiBase / (videoUrl % tweet.id) url = apiBase / (videoUrl % $tweet.id)
json = await fetchJson(url, headers) json = await fetchJson(url, headers)
if json == nil: if json == nil:
@ -106,7 +106,7 @@ proc getPoll*(tweet: Tweet; agent: string) {.async.} =
let let
headers = genHeaders(agent, base / getLink(tweet), auth=true) headers = genHeaders(agent, base / getLink(tweet), auth=true)
url = base / (pollUrl % tweet.id) url = base / (pollUrl % $tweet.id)
html = await fetchHtml(url, headers) html = await fetchHtml(url, headers)
if html == nil: return if html == nil: return

View file

@ -1,4 +1,5 @@
import strutils, strformat, sequtils, htmlgen, xmltree, times, uri, tables import strutils, strformat, sequtils, times, uri, tables
import xmltree, htmlparser, htmlgen
import regex import regex
import types, utils, query import types, utils, query
@ -15,6 +16,10 @@ const hostname {.strdefine.} = "nitter.net"
proc stripText*(text: string): string = proc stripText*(text: string): string =
text.replace(nbsp, " ").strip() text.replace(nbsp, " ").strip()
proc stripHtml*(text: string): string =
let html = parseHtml(text)
html.innerText()
proc shortLink*(text: string; length=28): string = proc shortLink*(text: string; length=28): string =
result = text.replace(re"https?://(www.)?", "") result = text.replace(re"https?://(www.)?", "")
if result.len > length: if result.len > length:
@ -27,7 +32,7 @@ proc replaceUrl*(url: string; prefs: Prefs; rss=false): string =
if prefs.replaceTwitter.len > 0: if prefs.replaceTwitter.len > 0:
result = result.replace(twRegex, prefs.replaceTwitter) result = result.replace(twRegex, prefs.replaceTwitter)
if rss: if rss:
result = result.replace("href=\"/", "href=\"" & hostname & "/") result = result.replace("href=\"/", "href=\"https://" & hostname & "/")
proc proxifyVideo*(manifest: string; proxy: bool): string = proc proxifyVideo*(manifest: string; proxy: bool): string =
proc cb(m: RegexMatch; s: string): string = proc cb(m: RegexMatch; s: string): string =
@ -42,7 +47,7 @@ proc getUserpic*(userpic: string; style=""): string =
proc getUserpic*(profile: Profile; style=""): string = proc getUserpic*(profile: Profile; style=""): string =
getUserPic(profile.userpic, style) getUserPic(profile.userpic, style)
proc getVideoEmbed*(id: string): string = proc getVideoEmbed*(id: int): string =
&"https://twitter.com/i/videos/{id}?embed_source=facebook" &"https://twitter.com/i/videos/{id}?embed_source=facebook"
proc pageTitle*(profile: Profile): string = proc pageTitle*(profile: Profile): string =
@ -67,7 +72,7 @@ proc getTweetTime*(tweet: Tweet): string =
tweet.time.format("h:mm tt' · 'MMM d', 'YYYY") tweet.time.format("h:mm tt' · 'MMM d', 'YYYY")
proc getLink*(tweet: Tweet | Quote): string = proc getLink*(tweet: Tweet | Quote): string =
if tweet.id.len == 0: return if tweet.id == 0: return
&"/{tweet.profile.username}/status/{tweet.id}" &"/{tweet.profile.username}/status/{tweet.id}"
proc getTombstone*(text: string): string = proc getTombstone*(text: string): string =

View file

@ -72,7 +72,7 @@ proc parseTweetProfile*(profile: XmlNode): Profile =
proc parseQuote*(quote: XmlNode): Quote = proc parseQuote*(quote: XmlNode): Quote =
result = Quote( result = Quote(
id: quote.attr("data-item-id"), id: parseInt(quote.attr("data-item-id")),
text: getQuoteText(quote), text: getQuoteText(quote),
reply: parseTweetReply(quote), reply: parseTweetReply(quote),
hasThread: quote.select(".self-thread-context") != nil, hasThread: quote.select(".self-thread-context") != nil,
@ -99,8 +99,8 @@ proc parseTweet*(node: XmlNode): Tweet =
return Tweet() return Tweet()
result = Tweet( result = Tweet(
id: tweet.attr("data-item-id"), id: parseInt(tweet.attr("data-item-id")),
threadId: tweet.attr("data-conversation-id"), threadId: parseInt(tweet.attr("data-conversation-id")),
text: getTweetText(tweet), text: getTweetText(tweet),
time: getTimestamp(tweet), time: getTimestamp(tweet),
shortTime: getShortTime(tweet), shortTime: getShortTime(tweet),
@ -119,7 +119,7 @@ proc parseTweet*(node: XmlNode): Tweet =
if by.len > 0: if by.len > 0:
result.retweet = some Retweet( result.retweet = some Retweet(
by: stripText(by), by: stripText(by),
id: tweet.attr("data-retweet-id") id: parseInt(tweet.attr("data-retweet-id"))
) )
let quote = tweet.select(".QuoteTweet-innerContainer") let quote = tweet.select(".QuoteTweet-innerContainer")
@ -191,7 +191,7 @@ proc parseTimeline*(node: XmlNode; after: string): Timeline =
beginning: after.len == 0 beginning: after.len == 0
) )
proc parseVideo*(node: JsonNode; tweetId: string): Video = proc parseVideo*(node: JsonNode; tweetId: int): Video =
let let
track = node{"track"} track = node{"track"}
cType = track["contentType"].to(string) cType = track["contentType"].to(string)
@ -216,7 +216,7 @@ proc parseVideo*(node: JsonNode; tweetId: string): Video =
else: else:
echo "Can't parse video of type ", cType echo "Can't parse video of type ", cType
result.videoId = tweetId result.videoId = $tweetId
result.thumb = node["posterImage"].to(string) result.thumb = node["posterImage"].to(string)
proc parsePoll*(node: XmlNode): Poll = proc parsePoll*(node: XmlNode): Poll =

View file

@ -57,7 +57,8 @@ proc parseText*(text: XmlNode; skipLink=""): string =
if "data-expanded-url" in el.attrs: if "data-expanded-url" in el.attrs:
let url = el.attr("data-expanded-url") let url = el.attr("data-expanded-url")
if url == skipLink: continue if url == skipLink: continue
elif "u-hidden" in class: result.add "\n" if "u-hidden" in class and result.len > 0:
result.add "\n"
result.add a(shortLink(url), href=url) result.add a(shortLink(url), href=url)
elif "ashtag" in class: elif "ashtag" in class:
let hash = el.innerText() let hash = el.innerText()
@ -240,7 +241,7 @@ proc getTweetCard*(tweet: Tweet; node: XmlNode) =
if cardDiv == nil: return if cardDiv == nil: return
var card = Card( var card = Card(
id: tweet.id, id: $tweet.id,
query: cardDiv.attr("data-src") query: cardDiv.attr("data-src")
) )

View file

@ -18,7 +18,7 @@ proc createStatusRouter*(cfg: Config) =
let prefs = cookiePrefs() let prefs = cookiePrefs()
let conversation = await getTweet(@"name", @"id", @"max_position", getAgent()) let conversation = await getTweet(@"name", @"id", @"max_position", getAgent())
if conversation == nil or conversation.tweet.id.len == 0: if conversation == nil or conversation.tweet.id == 0:
var error = "Tweet not found" var error = "Tweet not found"
if conversation != nil and conversation.tweet.tombstone.len > 0: if conversation != nil and conversation.tweet.tombstone.len > 0:
error = conversation.tweet.tombstone error = conversation.tweet.tombstone

View file

@ -115,7 +115,7 @@ type
video*: Option[Video] video*: Option[Video]
Quote* = object Quote* = object
id*: string id*: int
profile*: Profile profile*: Profile
text*: string text*: string
reply*: seq[string] reply*: seq[string]
@ -128,7 +128,7 @@ type
Retweet* = object Retweet* = object
by*: string by*: string
id*: string id*: int
TweetStats* = object TweetStats* = object
replies*: string replies*: string
@ -136,8 +136,8 @@ type
likes*: string likes*: string
Tweet* = ref object Tweet* = ref object
id*: string id*: int
threadId*: string threadId*: int
profile*: Profile profile*: Profile
text*: string text*: string
time*: Time time*: Time

View file

@ -57,7 +57,7 @@ proc renderMain*(body: VNode; req: Request; title="Nitter"; titleText=""; desc="
meta(name="viewport", content="width=device-width, initial-scale=1.0") meta(name="viewport", content="width=device-width, initial-scale=1.0")
meta(property="og:type", content=`type`) meta(property="og:type", content=`type`)
meta(property="og:title", content=titleText) meta(property="og:title", content=titleText)
meta(property="og:description", content=desc) meta(property="og:description", content=stripHtml(desc))
meta(property="og:site_name", content="Nitter") meta(property="og:site_name", content="Nitter")
for url in images: for url in images:

View file

@ -34,13 +34,13 @@ proc renderNoneFound(): VNode =
proc renderThread(thread: seq[Tweet]; prefs: Prefs; path: string): VNode = proc renderThread(thread: seq[Tweet]; prefs: Prefs; path: string): VNode =
buildHtml(tdiv(class="thread-line")): buildHtml(tdiv(class="thread-line")):
for i, threadTweet in thread.sortedByIt(it.time): for i, threadTweet in thread.sortedByIt(it.id):
let show = i == thread.len and thread[0].id != threadTweet.threadId let show = i == thread.len and thread[0].id != threadTweet.threadId
renderTweet(threadTweet, prefs, path, class="thread", renderTweet(threadTweet, prefs, path, class="thread",
index=i, total=thread.high, showThread=show) index=i, total=thread.high, showThread=show)
proc threadFilter(it: Tweet; tweetThread: string): bool = proc threadFilter(it: Tweet; thread: int): bool =
it.retweet.isNone and it.reply.len == 0 and it.threadId == tweetThread it.retweet.isNone and it.reply.len == 0 and it.threadId == thread
proc renderUser(user: Profile; prefs: Prefs): VNode = proc renderUser(user: Profile; prefs: Prefs): VNode =
buildHtml(tdiv(class="timeline-item")): buildHtml(tdiv(class="timeline-item")):
@ -81,8 +81,8 @@ proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string):
if results.content.len == 0: if results.content.len == 0:
renderNoneFound() renderNoneFound()
else: else:
var threads: seq[string] var threads: seq[int]
var retweets: seq[string] var retweets: seq[int]
for tweet in results.content: for tweet in results.content:
if tweet.threadId in threads or tweet.id in retweets: continue if tweet.threadId in threads or tweet.id in retweets: continue
let thread = results.content.filterIt(threadFilter(it, tweet.threadId)) let thread = results.content.filterIt(threadFilter(it, tweet.threadId))