Use random user agents
This commit is contained in:
Zed
parent
d0ee8e8403
commit
6b6e5b3a40
4 changed files with 143 additions and 53 deletions
90
src/agents.nim
Normal file
90
src/agents.nim
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
import random
|
||||
|
||||
const userAgents = [
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:53.0) Gecko/20100101 Firefox/53.0",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0",
|
||||
"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0",
|
||||
"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0",
|
||||
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/38.0",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0",
|
||||
"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko",
|
||||
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko",
|
||||
"Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 9.0; WOW64; Trident/7.0; rv:11.0) like Gecko",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64; rv:53.0) Gecko/20100101 Firefox/53.0",
|
||||
"Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0",
|
||||
"Mozilla/5.0 (X11; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0",
|
||||
"Mozilla/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0",
|
||||
"Mozilla/5.0 (X11; Linux x86_64; rv:50.0) Gecko/20100101 Firefox/50.0",
|
||||
"Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/38.0",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/40.1",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/43.0",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/50.0",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) like Gecko",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) like Gecko",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) like Gecko",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
|
||||
"Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00",
|
||||
"Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00",
|
||||
"Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14",
|
||||
"Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16"
|
||||
]
|
||||
|
||||
proc getAgent*(): string = sample(userAgents)
|
||||
87
src/api.nim
87
src/api.nim
|
|
@ -4,7 +4,6 @@ import sequtils, strutils, json, xmltree, uri
|
|||
import types, parser, parserutils, formatters, search
|
||||
|
||||
const
|
||||
agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
|
||||
lang = "en-US,en;q=0.9"
|
||||
auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
|
||||
cardAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
|
||||
|
|
@ -61,7 +60,7 @@ proc fetchJson(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} =
|
|||
except:
|
||||
return nil
|
||||
|
||||
proc getGuestToken(force=false): Future[string] {.async.} =
|
||||
proc getGuestToken(agent: string; force=false): Future[string] {.async.} =
|
||||
if getTime() - tokenUpdated < tokenLifetime and
|
||||
not force and tokenUses < tokenMaxUses:
|
||||
return guestToken
|
||||
|
|
@ -85,7 +84,7 @@ proc getGuestToken(force=false): Future[string] {.async.} =
|
|||
result = json["guest_token"].to(string)
|
||||
guestToken = result
|
||||
|
||||
proc getVideo*(tweet: Tweet; token: string) {.async.} =
|
||||
proc getVideo*(tweet: Tweet; token, agent: string) {.async.} =
|
||||
if tweet.video.isNone(): return
|
||||
|
||||
let headers = newHttpHeaders({
|
||||
|
|
@ -102,8 +101,8 @@ proc getVideo*(tweet: Tweet; token: string) {.async.} =
|
|||
if json == nil:
|
||||
if getTime() - tokenUpdated > initDuration(seconds=1):
|
||||
tokenUpdated = getTime()
|
||||
discard await getGuestToken(force=true)
|
||||
await getVideo(tweet, guestToken)
|
||||
discard await getGuestToken(agent, force=true)
|
||||
await getVideo(tweet, guestToken, agent)
|
||||
return
|
||||
|
||||
if tweet.card.isNone:
|
||||
|
|
@ -113,31 +112,31 @@ proc getVideo*(tweet: Tweet; token: string) {.async.} =
|
|||
tweet.video = none(Video)
|
||||
tokenUses.inc
|
||||
|
||||
proc getVideos*(thread: Thread; token="") {.async.} =
|
||||
proc getVideos*(thread: Thread; agent: string; token="") {.async.} =
|
||||
if thread == nil: return
|
||||
|
||||
var gToken = token
|
||||
if gToken.len == 0:
|
||||
gToken = await getGuestToken()
|
||||
gToken = await getGuestToken(agent)
|
||||
|
||||
var videoFuts: seq[Future[void]]
|
||||
for tweet in thread.tweets.filterIt(it.video.isSome):
|
||||
videoFuts.add getVideo(tweet, gToken)
|
||||
videoFuts.add getVideo(tweet, gToken, agent)
|
||||
|
||||
await all(videoFuts)
|
||||
|
||||
proc getConversationVideos*(convo: Conversation) {.async.} =
|
||||
var token = await getGuestToken()
|
||||
proc getConversationVideos*(convo: Conversation; agent: string) {.async.} =
|
||||
var token = await getGuestToken(agent)
|
||||
var futs: seq[Future[void]]
|
||||
|
||||
futs.add getVideo(convo.tweet, token)
|
||||
futs.add convo.replies.mapIt(getVideos(it, token))
|
||||
futs.add getVideos(convo.before, token)
|
||||
futs.add getVideos(convo.after, token)
|
||||
futs.add getVideo(convo.tweet, token, agent)
|
||||
futs.add convo.replies.mapIt(getVideos(it, token, agent))
|
||||
futs.add getVideos(convo.before, token, agent)
|
||||
futs.add getVideos(convo.after, token, agent)
|
||||
|
||||
await all(futs)
|
||||
|
||||
proc getPoll*(tweet: Tweet) {.async.} =
|
||||
proc getPoll*(tweet: Tweet; agent: string) {.async.} =
|
||||
if tweet.poll.isNone(): return
|
||||
|
||||
let headers = newHttpHeaders({
|
||||
|
|
@ -154,20 +153,20 @@ proc getPoll*(tweet: Tweet) {.async.} =
|
|||
|
||||
tweet.poll = some(parsePoll(html))
|
||||
|
||||
proc getPolls*(thread: Thread) {.async.} =
|
||||
proc getPolls*(thread: Thread; agent: string) {.async.} =
|
||||
if thread == nil: return
|
||||
var polls = thread.tweets.filterIt(it.poll.isSome)
|
||||
await all(polls.map(getPoll))
|
||||
await all(polls.mapIt(getPoll(it, agent)))
|
||||
|
||||
proc getConversationPolls*(convo: Conversation) {.async.} =
|
||||
proc getConversationPolls*(convo: Conversation; agent: string) {.async.} =
|
||||
var futs: seq[Future[void]]
|
||||
futs.add getPoll(convo.tweet)
|
||||
futs.add getPolls(convo.before)
|
||||
futs.add getPolls(convo.after)
|
||||
futs.add convo.replies.map(getPolls)
|
||||
futs.add getPoll(convo.tweet, agent)
|
||||
futs.add getPolls(convo.before, agent)
|
||||
futs.add getPolls(convo.after, agent)
|
||||
futs.add convo.replies.mapIt(getPolls(it, agent))
|
||||
await all(futs)
|
||||
|
||||
proc getCard*(tweet: Tweet) {.async.} =
|
||||
proc getCard*(tweet: Tweet; agent: string) {.async.} =
|
||||
if tweet.card.isNone(): return
|
||||
|
||||
let headers = newHttpHeaders({
|
||||
|
|
@ -184,20 +183,20 @@ proc getCard*(tweet: Tweet) {.async.} =
|
|||
|
||||
parseCard(get(tweet.card), html)
|
||||
|
||||
proc getCards*(thread: Thread) {.async.} =
|
||||
proc getCards*(thread: Thread; agent: string) {.async.} =
|
||||
if thread == nil: return
|
||||
var cards = thread.tweets.filterIt(it.card.isSome)
|
||||
await all(cards.map(getCard))
|
||||
await all(cards.mapIt(getCard(it, agent)))
|
||||
|
||||
proc getConversationCards*(convo: Conversation) {.async.} =
|
||||
proc getConversationCards*(convo: Conversation; agent: string) {.async.} =
|
||||
var futs: seq[Future[void]]
|
||||
futs.add getCard(convo.tweet)
|
||||
futs.add getCards(convo.before)
|
||||
futs.add getCards(convo.after)
|
||||
futs.add convo.replies.map(getCards)
|
||||
futs.add getCard(convo.tweet, agent)
|
||||
futs.add getCards(convo.before, agent)
|
||||
futs.add getCards(convo.after, agent)
|
||||
futs.add convo.replies.mapIt(getCards(it, agent))
|
||||
await all(futs)
|
||||
|
||||
proc getPhotoRail*(username: string): Future[seq[GalleryPhoto]] {.async.} =
|
||||
proc getPhotoRail*(username, agent: string): Future[seq[GalleryPhoto]] {.async.} =
|
||||
let headers = newHttpHeaders({
|
||||
"Accept": jsonAccept,
|
||||
"Referer": $(base / username),
|
||||
|
|
@ -222,7 +221,7 @@ proc getProfileFallback(username: string; headers: HttpHeaders): Future[Profile]
|
|||
|
||||
result = parseIntentProfile(html)
|
||||
|
||||
proc getProfile*(username: string): Future[Profile] {.async.} =
|
||||
proc getProfile*(username, agent: string): Future[Profile] {.async.} =
|
||||
let headers = newHttpHeaders({
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9",
|
||||
"Referer": $(base / username),
|
||||
|
|
@ -248,7 +247,7 @@ proc getProfile*(username: string): Future[Profile] {.async.} =
|
|||
|
||||
result = parsePopupProfile(html)
|
||||
|
||||
proc getTweet*(username, id: string): Future[Conversation] {.async.} =
|
||||
proc getTweet*(username, id, agent: string): Future[Conversation] {.async.} =
|
||||
let headers = newHttpHeaders({
|
||||
"Accept": jsonAccept,
|
||||
"Referer": $base,
|
||||
|
|
@ -269,13 +268,13 @@ proc getTweet*(username, id: string): Future[Conversation] {.async.} =
|
|||
result = parseConversation(html)
|
||||
|
||||
let
|
||||
vidsFut = getConversationVideos(result)
|
||||
pollFut = getConversationPolls(result)
|
||||
cardFut = getConversationCards(result)
|
||||
vidsFut = getConversationVideos(result, agent)
|
||||
pollFut = getConversationPolls(result, agent)
|
||||
cardFut = getConversationCards(result, agent)
|
||||
|
||||
await all(vidsFut, pollFut, cardFut)
|
||||
|
||||
proc finishTimeline(json: JsonNode; query: Option[Query]; after: string): Future[Timeline] {.async.} =
|
||||
proc finishTimeline(json: JsonNode; query: Option[Query]; after, agent: string): Future[Timeline] {.async.} =
|
||||
if json == nil: return Timeline()
|
||||
|
||||
result = Timeline(
|
||||
|
|
@ -292,14 +291,14 @@ proc finishTimeline(json: JsonNode; query: Option[Query]; after: string): Future
|
|||
let
|
||||
html = parseHtml(json["items_html"].to(string))
|
||||
thread = parseThread(html)
|
||||
vidsFut = getVideos(thread)
|
||||
pollFut = getPolls(thread)
|
||||
cardFut = getCards(thread)
|
||||
vidsFut = getVideos(thread, agent)
|
||||
pollFut = getPolls(thread, agent)
|
||||
cardFut = getCards(thread, agent)
|
||||
|
||||
await all(vidsFut, pollFut, cardFut)
|
||||
result.tweets = thread.tweets
|
||||
|
||||
proc getTimeline*(username, after: string): Future[Timeline] {.async.} =
|
||||
proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} =
|
||||
let headers = newHttpHeaders({
|
||||
"Accept": jsonAccept,
|
||||
"Referer": $(base / username),
|
||||
|
|
@ -320,9 +319,9 @@ proc getTimeline*(username, after: string): Future[Timeline] {.async.} =
|
|||
params.add {"max_position": after}
|
||||
|
||||
let json = await fetchJson(base / (timelineUrl % username) ? params, headers)
|
||||
result = await finishTimeline(json, none(Query), after)
|
||||
result = await finishTimeline(json, none(Query), after, agent)
|
||||
|
||||
proc getTimelineSearch*(username, after: string; query: Query): Future[Timeline] {.async.} =
|
||||
proc getTimelineSearch*(username, after, agent: string; query: Query): Future[Timeline] {.async.} =
|
||||
let queryParam = genQueryParam(query)
|
||||
let queryEncoded = encodeUrl(queryParam, usePlus=false)
|
||||
|
||||
|
|
@ -347,4 +346,4 @@ proc getTimelineSearch*(username, after: string; query: Query): Future[Timeline]
|
|||
}
|
||||
|
||||
let json = await fetchJson(base / timelineSearchUrl ? params, headers)
|
||||
result = await finishTimeline(json, some(query), after)
|
||||
result = await finishTimeline(json, some(query), after, agent)
|
||||
|
|
|
|||
|
|
@ -12,18 +12,18 @@ var profileCacheTime = initDuration(minutes=10)
|
|||
proc outdated(profile: Profile): bool =
|
||||
getTime() - profile.updated > profileCacheTime
|
||||
|
||||
proc getCachedProfile*(username: string; force=false): Future[Profile] {.async.} =
|
||||
proc getCachedProfile*(username, agent: string; force=false): Future[Profile] {.async.} =
|
||||
withDb:
|
||||
try:
|
||||
result.getOne("username = ?", username)
|
||||
doAssert not result.outdated()
|
||||
except AssertionError:
|
||||
var profile = await getProfile(username)
|
||||
var profile = await getProfile(username, agent)
|
||||
profile.id = result.id
|
||||
result = profile
|
||||
result.update()
|
||||
except KeyError:
|
||||
result = await getProfile(username)
|
||||
result = await getProfile(username, agent)
|
||||
if result.username.len > 0:
|
||||
result.insert()
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from net import Port
|
|||
|
||||
import jester, regex
|
||||
|
||||
import api, utils, types, cache, formatters, search, config
|
||||
import api, utils, types, cache, formatters, search, config, agents
|
||||
import views/[general, profile, status]
|
||||
|
||||
const configPath {.strdefine.} = "./nitter.conf"
|
||||
|
|
@ -11,15 +11,16 @@ let cfg = getConfig(configPath)
|
|||
|
||||
proc showTimeline(name, after: string; query: Option[Query]): Future[string] {.async.} =
|
||||
let
|
||||
agent = getAgent()
|
||||
username = name.strip(chars={'/'})
|
||||
profileFut = getCachedProfile(username)
|
||||
railFut = getPhotoRail(username)
|
||||
profileFut = getCachedProfile(username, agent)
|
||||
railFut = getPhotoRail(username, agent)
|
||||
|
||||
var timelineFut: Future[Timeline]
|
||||
if query.isNone:
|
||||
timelineFut = getTimeline(username, after)
|
||||
timelineFut = getTimeline(username, after, agent)
|
||||
else:
|
||||
timelineFut = getTimelineSearch(username, after, get(query))
|
||||
timelineFut = getTimelineSearch(username, after, agent, get(query))
|
||||
|
||||
let profile = await profileFut
|
||||
if profile.username.len == 0:
|
||||
|
|
@ -69,7 +70,7 @@ routes:
|
|||
get "/@name/status/@id":
|
||||
cond '.' notin @"name"
|
||||
|
||||
let conversation = await getTweet(@"name", @"id")
|
||||
let conversation = await getTweet(@"name", @"id", getAgent())
|
||||
if conversation == nil or conversation.tweet.id.len == 0:
|
||||
resp Http404, showError("Tweet not found", cfg.title)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue