Cynosphere/nitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Optimize RSS requests by disabling media fetching

Browse source
This commit is contained in:
Zed 2019-10-21 23:12:40 +02:00
parent 453beff09d
commit 6fb039dd79
5 changed files with 21 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/api/list.nim
View file

@ -4,7 +4,7 @@ import sequtils, strutils, json, uri
import ".."/[types, parser, parserutils, query]
import utils, consts, timeline, search
proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.async.} =
proc getListTimeline*(username, list, agent, after: string; media=true): Future[Timeline] {.async.} =
let url = base / (listUrl % [username, list])
var params = toSeq({
@ -17,7 +17,7 @@ proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.
params.add {"max_position": after}
let json = await fetchJson(url ? params, genHeaders(agent, url))
result = await finishTimeline(json, Query(), after, agent)
result = await finishTimeline(json, Query(), after, agent, media)
if result.content.len == 0:
return

4
src/api/search.nim
View file

@ -14,7 +14,7 @@ proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] =
beginning: after.len == 0
)
proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.async.} =
proc getSearch*[T](query: Query; after, agent: string; media=true): Future[Result[T]] {.async.} =
let
kind = if query.kind == users: "users" else: "tweets"
@ -44,7 +44,7 @@ proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.asyn
if json == nil or not json.hasKey("items_html"): return
when T is Tweet:
result = await finishTimeline(json, query, after, agent)
result = await finishTimeline(json, query, after, agent, media)
elif T is Profile:
let html = json["items_html"].to(string)
result.hasMore = html != "\n"

13
src/api/timeline.nim
View file

@ -9,7 +9,8 @@ proc getMedia(thread: Chain | Timeline; agent: string) {.async.} =
getCards(thread, agent),
getPolls(thread, agent))
proc finishTimeline*(json: JsonNode; query: Query; after, agent: string): Future[Timeline] {.async.} =
proc finishTimeline*(json: JsonNode; query: Query; after, agent: string;
media=true): Future[Timeline] {.async.} =
result = getResult[Tweet](json, query, after)
if json == nil: return
@ -19,10 +20,10 @@ proc finishTimeline*(json: JsonNode; query: Query; after, agent: string): Future
let html = parseHtml(json["items_html"].to(string))
let thread = parseChain(html)
await getMedia(thread, agent)
if media: await getMedia(thread, agent)
result.content = thread.content
proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} =
proc getTimeline*(username, after, agent: string; media=true): Future[Timeline] {.async.} =
var params = toSeq({
"include_available_features": "1",
"include_entities": "1",
@ -36,9 +37,9 @@ proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} =
let headers = genHeaders(agent, base / username, xml=true)
let json = await fetchJson(base / (timelineUrl % username) ? params, headers)
result = await finishTimeline(json, Query(), after, agent)
result = await finishTimeline(json, Query(), after, agent, media)
proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Timeline)] {.async.} =
proc getProfileAndTimeline*(username, agent, after: string; media=true): Future[(Profile, Timeline)] {.async.} =
var url = base / username
if after.len > 0:
url = url ? {"max_position": after}
@ -49,5 +50,5 @@ proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Ti
timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
profile = parseTimelineProfile(html)
await getMedia(timeline, agent)
if media: await getMedia(timeline, agent)
result = (profile, timeline)

8
src/routes/rss.nim
View file

@ -9,7 +9,9 @@ import ../views/general
include "../views/rss.nimf"
proc showRss*(name, hostname: string; query: Query): Future[string] {.async.} =
let (profile, timeline, _) = await fetchSingleTimeline(name, "", getAgent(), query)
let (profile, timeline, _) =
await fetchSingleTimeline(name, "", getAgent(), query, media=false)
if timeline != nil:
return renderTimelineRss(timeline, profile, hostname)
@ -28,7 +30,7 @@ proc createRssRouter*(cfg: Config) =
if query.kind != tweets:
resp Http400, showError("Only Tweet searches are allowed for RSS feeds.", cfg)
let tweets = await getSearch[Tweet](query, "", getAgent())
let tweets = await getSearch[Tweet](query, "", getAgent(), media=false)
respRss(renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg.hostname))
get "/@name/rss":
@ -49,5 +51,5 @@ proc createRssRouter*(cfg: Config) =
get "/@name/lists/@list/rss":
cond '.' notin @"name"
let list = await getListTimeline(@"name", @"list", getAgent(), "")
let list = await getListTimeline(@"name", @"list", getAgent(), "", media=false)
respRss(renderListRss(list.content, @"name", @"list", cfg.hostname))

10
src/routes/timeline.nim
View file

@ -13,8 +13,8 @@ export profile, timeline, status
type ProfileTimeline = (Profile, Timeline, seq[GalleryPhoto])
proc fetchSingleTimeline*(name, after, agent: string;
query: Query): Future[ProfileTimeline] {.async.} =
proc fetchSingleTimeline*(name, after, agent: string; query: Query;
media=true): Future[ProfileTimeline] {.async.} =
let railFut = getPhotoRail(name, agent)
var timeline: Timeline
@ -26,12 +26,12 @@ proc fetchSingleTimeline*(name, after, agent: string;
if query.kind == posts:
if cachedProfile.isSome:
timeline = await getTimeline(name, after, agent)
timeline = await getTimeline(name, after, agent, media)
else:
(profile, timeline) = await getProfileAndTimeline(name, agent, after)
(profile, timeline) = await getProfileAndTimeline(name, agent, after, media)
cache(profile)
else:
var timelineFut = getSearch[Tweet](query, after, agent)
var timelineFut = getSearch[Tweet](query, after, agent, media)
if cachedProfile.isNone:
profile = await getCachedProfile(name, agent)
timeline = await timelineFut