Optimize RSS requests by disabling media fetching
This commit is contained in:
parent
453beff09d
commit
6fb039dd79
5 changed files with 21 additions and 18 deletions
|
@ -4,7 +4,7 @@ import sequtils, strutils, json, uri
|
||||||
import ".."/[types, parser, parserutils, query]
|
import ".."/[types, parser, parserutils, query]
|
||||||
import utils, consts, timeline, search
|
import utils, consts, timeline, search
|
||||||
|
|
||||||
proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.async.} =
|
proc getListTimeline*(username, list, agent, after: string; media=true): Future[Timeline] {.async.} =
|
||||||
let url = base / (listUrl % [username, list])
|
let url = base / (listUrl % [username, list])
|
||||||
|
|
||||||
var params = toSeq({
|
var params = toSeq({
|
||||||
|
@ -17,7 +17,7 @@ proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.
|
||||||
params.add {"max_position": after}
|
params.add {"max_position": after}
|
||||||
|
|
||||||
let json = await fetchJson(url ? params, genHeaders(agent, url))
|
let json = await fetchJson(url ? params, genHeaders(agent, url))
|
||||||
result = await finishTimeline(json, Query(), after, agent)
|
result = await finishTimeline(json, Query(), after, agent, media)
|
||||||
if result.content.len == 0:
|
if result.content.len == 0:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
|
@ -14,7 +14,7 @@ proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] =
|
||||||
beginning: after.len == 0
|
beginning: after.len == 0
|
||||||
)
|
)
|
||||||
|
|
||||||
proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.async.} =
|
proc getSearch*[T](query: Query; after, agent: string; media=true): Future[Result[T]] {.async.} =
|
||||||
let
|
let
|
||||||
kind = if query.kind == users: "users" else: "tweets"
|
kind = if query.kind == users: "users" else: "tweets"
|
||||||
|
|
||||||
|
@ -44,7 +44,7 @@ proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.asyn
|
||||||
if json == nil or not json.hasKey("items_html"): return
|
if json == nil or not json.hasKey("items_html"): return
|
||||||
|
|
||||||
when T is Tweet:
|
when T is Tweet:
|
||||||
result = await finishTimeline(json, query, after, agent)
|
result = await finishTimeline(json, query, after, agent, media)
|
||||||
elif T is Profile:
|
elif T is Profile:
|
||||||
let html = json["items_html"].to(string)
|
let html = json["items_html"].to(string)
|
||||||
result.hasMore = html != "\n"
|
result.hasMore = html != "\n"
|
||||||
|
|
|
@ -9,7 +9,8 @@ proc getMedia(thread: Chain | Timeline; agent: string) {.async.} =
|
||||||
getCards(thread, agent),
|
getCards(thread, agent),
|
||||||
getPolls(thread, agent))
|
getPolls(thread, agent))
|
||||||
|
|
||||||
proc finishTimeline*(json: JsonNode; query: Query; after, agent: string): Future[Timeline] {.async.} =
|
proc finishTimeline*(json: JsonNode; query: Query; after, agent: string;
|
||||||
|
media=true): Future[Timeline] {.async.} =
|
||||||
result = getResult[Tweet](json, query, after)
|
result = getResult[Tweet](json, query, after)
|
||||||
if json == nil: return
|
if json == nil: return
|
||||||
|
|
||||||
|
@ -19,10 +20,10 @@ proc finishTimeline*(json: JsonNode; query: Query; after, agent: string): Future
|
||||||
let html = parseHtml(json["items_html"].to(string))
|
let html = parseHtml(json["items_html"].to(string))
|
||||||
let thread = parseChain(html)
|
let thread = parseChain(html)
|
||||||
|
|
||||||
await getMedia(thread, agent)
|
if media: await getMedia(thread, agent)
|
||||||
result.content = thread.content
|
result.content = thread.content
|
||||||
|
|
||||||
proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} =
|
proc getTimeline*(username, after, agent: string; media=true): Future[Timeline] {.async.} =
|
||||||
var params = toSeq({
|
var params = toSeq({
|
||||||
"include_available_features": "1",
|
"include_available_features": "1",
|
||||||
"include_entities": "1",
|
"include_entities": "1",
|
||||||
|
@ -36,9 +37,9 @@ proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} =
|
||||||
let headers = genHeaders(agent, base / username, xml=true)
|
let headers = genHeaders(agent, base / username, xml=true)
|
||||||
let json = await fetchJson(base / (timelineUrl % username) ? params, headers)
|
let json = await fetchJson(base / (timelineUrl % username) ? params, headers)
|
||||||
|
|
||||||
result = await finishTimeline(json, Query(), after, agent)
|
result = await finishTimeline(json, Query(), after, agent, media)
|
||||||
|
|
||||||
proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Timeline)] {.async.} =
|
proc getProfileAndTimeline*(username, agent, after: string; media=true): Future[(Profile, Timeline)] {.async.} =
|
||||||
var url = base / username
|
var url = base / username
|
||||||
if after.len > 0:
|
if after.len > 0:
|
||||||
url = url ? {"max_position": after}
|
url = url ? {"max_position": after}
|
||||||
|
@ -49,5 +50,5 @@ proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Ti
|
||||||
timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
|
timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
|
||||||
profile = parseTimelineProfile(html)
|
profile = parseTimelineProfile(html)
|
||||||
|
|
||||||
await getMedia(timeline, agent)
|
if media: await getMedia(timeline, agent)
|
||||||
result = (profile, timeline)
|
result = (profile, timeline)
|
||||||
|
|
|
@ -9,7 +9,9 @@ import ../views/general
|
||||||
include "../views/rss.nimf"
|
include "../views/rss.nimf"
|
||||||
|
|
||||||
proc showRss*(name, hostname: string; query: Query): Future[string] {.async.} =
|
proc showRss*(name, hostname: string; query: Query): Future[string] {.async.} =
|
||||||
let (profile, timeline, _) = await fetchSingleTimeline(name, "", getAgent(), query)
|
let (profile, timeline, _) =
|
||||||
|
await fetchSingleTimeline(name, "", getAgent(), query, media=false)
|
||||||
|
|
||||||
if timeline != nil:
|
if timeline != nil:
|
||||||
return renderTimelineRss(timeline, profile, hostname)
|
return renderTimelineRss(timeline, profile, hostname)
|
||||||
|
|
||||||
|
@ -28,7 +30,7 @@ proc createRssRouter*(cfg: Config) =
|
||||||
if query.kind != tweets:
|
if query.kind != tweets:
|
||||||
resp Http400, showError("Only Tweet searches are allowed for RSS feeds.", cfg)
|
resp Http400, showError("Only Tweet searches are allowed for RSS feeds.", cfg)
|
||||||
|
|
||||||
let tweets = await getSearch[Tweet](query, "", getAgent())
|
let tweets = await getSearch[Tweet](query, "", getAgent(), media=false)
|
||||||
respRss(renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg.hostname))
|
respRss(renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg.hostname))
|
||||||
|
|
||||||
get "/@name/rss":
|
get "/@name/rss":
|
||||||
|
@ -49,5 +51,5 @@ proc createRssRouter*(cfg: Config) =
|
||||||
|
|
||||||
get "/@name/lists/@list/rss":
|
get "/@name/lists/@list/rss":
|
||||||
cond '.' notin @"name"
|
cond '.' notin @"name"
|
||||||
let list = await getListTimeline(@"name", @"list", getAgent(), "")
|
let list = await getListTimeline(@"name", @"list", getAgent(), "", media=false)
|
||||||
respRss(renderListRss(list.content, @"name", @"list", cfg.hostname))
|
respRss(renderListRss(list.content, @"name", @"list", cfg.hostname))
|
||||||
|
|
|
@ -13,8 +13,8 @@ export profile, timeline, status
|
||||||
|
|
||||||
type ProfileTimeline = (Profile, Timeline, seq[GalleryPhoto])
|
type ProfileTimeline = (Profile, Timeline, seq[GalleryPhoto])
|
||||||
|
|
||||||
proc fetchSingleTimeline*(name, after, agent: string;
|
proc fetchSingleTimeline*(name, after, agent: string; query: Query;
|
||||||
query: Query): Future[ProfileTimeline] {.async.} =
|
media=true): Future[ProfileTimeline] {.async.} =
|
||||||
let railFut = getPhotoRail(name, agent)
|
let railFut = getPhotoRail(name, agent)
|
||||||
|
|
||||||
var timeline: Timeline
|
var timeline: Timeline
|
||||||
|
@ -26,12 +26,12 @@ proc fetchSingleTimeline*(name, after, agent: string;
|
||||||
|
|
||||||
if query.kind == posts:
|
if query.kind == posts:
|
||||||
if cachedProfile.isSome:
|
if cachedProfile.isSome:
|
||||||
timeline = await getTimeline(name, after, agent)
|
timeline = await getTimeline(name, after, agent, media)
|
||||||
else:
|
else:
|
||||||
(profile, timeline) = await getProfileAndTimeline(name, agent, after)
|
(profile, timeline) = await getProfileAndTimeline(name, agent, after, media)
|
||||||
cache(profile)
|
cache(profile)
|
||||||
else:
|
else:
|
||||||
var timelineFut = getSearch[Tweet](query, after, agent)
|
var timelineFut = getSearch[Tweet](query, after, agent, media)
|
||||||
if cachedProfile.isNone:
|
if cachedProfile.isNone:
|
||||||
profile = await getCachedProfile(name, agent)
|
profile = await getCachedProfile(name, agent)
|
||||||
timeline = await timelineFut
|
timeline = await timelineFut
|
||||||
|
|
Loading…
Reference in a new issue