Use random user agents
This commit is contained in:
parent
d0ee8e8403
commit
6b6e5b3a40
4 changed files with 143 additions and 53 deletions
90
src/agents.nim
Normal file
90
src/agents.nim
Normal file
|
@ -0,0 +1,90 @@
|
||||||
|
import random
|
||||||
|
|
||||||
|
const userAgents = [
|
||||||
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:53.0) Gecko/20100101 Firefox/53.0",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0",
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0",
|
||||||
|
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/38.0",
|
||||||
|
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
|
||||||
|
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0",
|
||||||
|
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0",
|
||||||
|
"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko",
|
||||||
|
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko",
|
||||||
|
"Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (Windows NT 9.0; WOW64; Trident/7.0; rv:11.0) like Gecko",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64; rv:53.0) Gecko/20100101 Firefox/53.0",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64; rv:50.0) Gecko/20100101 Firefox/50.0",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/38.0",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/40.1",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/43.0",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/50.0",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) like Gecko",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) like Gecko",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) like Gecko",
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
|
||||||
|
"Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00",
|
||||||
|
"Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00",
|
||||||
|
"Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14",
|
||||||
|
"Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16"
|
||||||
|
]
|
||||||
|
|
||||||
|
proc getAgent*(): string = sample(userAgents)
|
87
src/api.nim
87
src/api.nim
|
@ -4,7 +4,6 @@ import sequtils, strutils, json, xmltree, uri
|
||||||
import types, parser, parserutils, formatters, search
|
import types, parser, parserutils, formatters, search
|
||||||
|
|
||||||
const
|
const
|
||||||
agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
|
|
||||||
lang = "en-US,en;q=0.9"
|
lang = "en-US,en;q=0.9"
|
||||||
auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
|
auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
|
||||||
cardAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
|
cardAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
|
||||||
|
@ -61,7 +60,7 @@ proc fetchJson(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} =
|
||||||
except:
|
except:
|
||||||
return nil
|
return nil
|
||||||
|
|
||||||
proc getGuestToken(force=false): Future[string] {.async.} =
|
proc getGuestToken(agent: string; force=false): Future[string] {.async.} =
|
||||||
if getTime() - tokenUpdated < tokenLifetime and
|
if getTime() - tokenUpdated < tokenLifetime and
|
||||||
not force and tokenUses < tokenMaxUses:
|
not force and tokenUses < tokenMaxUses:
|
||||||
return guestToken
|
return guestToken
|
||||||
|
@ -85,7 +84,7 @@ proc getGuestToken(force=false): Future[string] {.async.} =
|
||||||
result = json["guest_token"].to(string)
|
result = json["guest_token"].to(string)
|
||||||
guestToken = result
|
guestToken = result
|
||||||
|
|
||||||
proc getVideo*(tweet: Tweet; token: string) {.async.} =
|
proc getVideo*(tweet: Tweet; token, agent: string) {.async.} =
|
||||||
if tweet.video.isNone(): return
|
if tweet.video.isNone(): return
|
||||||
|
|
||||||
let headers = newHttpHeaders({
|
let headers = newHttpHeaders({
|
||||||
|
@ -102,8 +101,8 @@ proc getVideo*(tweet: Tweet; token: string) {.async.} =
|
||||||
if json == nil:
|
if json == nil:
|
||||||
if getTime() - tokenUpdated > initDuration(seconds=1):
|
if getTime() - tokenUpdated > initDuration(seconds=1):
|
||||||
tokenUpdated = getTime()
|
tokenUpdated = getTime()
|
||||||
discard await getGuestToken(force=true)
|
discard await getGuestToken(agent, force=true)
|
||||||
await getVideo(tweet, guestToken)
|
await getVideo(tweet, guestToken, agent)
|
||||||
return
|
return
|
||||||
|
|
||||||
if tweet.card.isNone:
|
if tweet.card.isNone:
|
||||||
|
@ -113,31 +112,31 @@ proc getVideo*(tweet: Tweet; token: string) {.async.} =
|
||||||
tweet.video = none(Video)
|
tweet.video = none(Video)
|
||||||
tokenUses.inc
|
tokenUses.inc
|
||||||
|
|
||||||
proc getVideos*(thread: Thread; token="") {.async.} =
|
proc getVideos*(thread: Thread; agent: string; token="") {.async.} =
|
||||||
if thread == nil: return
|
if thread == nil: return
|
||||||
|
|
||||||
var gToken = token
|
var gToken = token
|
||||||
if gToken.len == 0:
|
if gToken.len == 0:
|
||||||
gToken = await getGuestToken()
|
gToken = await getGuestToken(agent)
|
||||||
|
|
||||||
var videoFuts: seq[Future[void]]
|
var videoFuts: seq[Future[void]]
|
||||||
for tweet in thread.tweets.filterIt(it.video.isSome):
|
for tweet in thread.tweets.filterIt(it.video.isSome):
|
||||||
videoFuts.add getVideo(tweet, gToken)
|
videoFuts.add getVideo(tweet, gToken, agent)
|
||||||
|
|
||||||
await all(videoFuts)
|
await all(videoFuts)
|
||||||
|
|
||||||
proc getConversationVideos*(convo: Conversation) {.async.} =
|
proc getConversationVideos*(convo: Conversation; agent: string) {.async.} =
|
||||||
var token = await getGuestToken()
|
var token = await getGuestToken(agent)
|
||||||
var futs: seq[Future[void]]
|
var futs: seq[Future[void]]
|
||||||
|
|
||||||
futs.add getVideo(convo.tweet, token)
|
futs.add getVideo(convo.tweet, token, agent)
|
||||||
futs.add convo.replies.mapIt(getVideos(it, token))
|
futs.add convo.replies.mapIt(getVideos(it, token, agent))
|
||||||
futs.add getVideos(convo.before, token)
|
futs.add getVideos(convo.before, token, agent)
|
||||||
futs.add getVideos(convo.after, token)
|
futs.add getVideos(convo.after, token, agent)
|
||||||
|
|
||||||
await all(futs)
|
await all(futs)
|
||||||
|
|
||||||
proc getPoll*(tweet: Tweet) {.async.} =
|
proc getPoll*(tweet: Tweet; agent: string) {.async.} =
|
||||||
if tweet.poll.isNone(): return
|
if tweet.poll.isNone(): return
|
||||||
|
|
||||||
let headers = newHttpHeaders({
|
let headers = newHttpHeaders({
|
||||||
|
@ -154,20 +153,20 @@ proc getPoll*(tweet: Tweet) {.async.} =
|
||||||
|
|
||||||
tweet.poll = some(parsePoll(html))
|
tweet.poll = some(parsePoll(html))
|
||||||
|
|
||||||
proc getPolls*(thread: Thread) {.async.} =
|
proc getPolls*(thread: Thread; agent: string) {.async.} =
|
||||||
if thread == nil: return
|
if thread == nil: return
|
||||||
var polls = thread.tweets.filterIt(it.poll.isSome)
|
var polls = thread.tweets.filterIt(it.poll.isSome)
|
||||||
await all(polls.map(getPoll))
|
await all(polls.mapIt(getPoll(it, agent)))
|
||||||
|
|
||||||
proc getConversationPolls*(convo: Conversation) {.async.} =
|
proc getConversationPolls*(convo: Conversation; agent: string) {.async.} =
|
||||||
var futs: seq[Future[void]]
|
var futs: seq[Future[void]]
|
||||||
futs.add getPoll(convo.tweet)
|
futs.add getPoll(convo.tweet, agent)
|
||||||
futs.add getPolls(convo.before)
|
futs.add getPolls(convo.before, agent)
|
||||||
futs.add getPolls(convo.after)
|
futs.add getPolls(convo.after, agent)
|
||||||
futs.add convo.replies.map(getPolls)
|
futs.add convo.replies.mapIt(getPolls(it, agent))
|
||||||
await all(futs)
|
await all(futs)
|
||||||
|
|
||||||
proc getCard*(tweet: Tweet) {.async.} =
|
proc getCard*(tweet: Tweet; agent: string) {.async.} =
|
||||||
if tweet.card.isNone(): return
|
if tweet.card.isNone(): return
|
||||||
|
|
||||||
let headers = newHttpHeaders({
|
let headers = newHttpHeaders({
|
||||||
|
@ -184,20 +183,20 @@ proc getCard*(tweet: Tweet) {.async.} =
|
||||||
|
|
||||||
parseCard(get(tweet.card), html)
|
parseCard(get(tweet.card), html)
|
||||||
|
|
||||||
proc getCards*(thread: Thread) {.async.} =
|
proc getCards*(thread: Thread; agent: string) {.async.} =
|
||||||
if thread == nil: return
|
if thread == nil: return
|
||||||
var cards = thread.tweets.filterIt(it.card.isSome)
|
var cards = thread.tweets.filterIt(it.card.isSome)
|
||||||
await all(cards.map(getCard))
|
await all(cards.mapIt(getCard(it, agent)))
|
||||||
|
|
||||||
proc getConversationCards*(convo: Conversation) {.async.} =
|
proc getConversationCards*(convo: Conversation; agent: string) {.async.} =
|
||||||
var futs: seq[Future[void]]
|
var futs: seq[Future[void]]
|
||||||
futs.add getCard(convo.tweet)
|
futs.add getCard(convo.tweet, agent)
|
||||||
futs.add getCards(convo.before)
|
futs.add getCards(convo.before, agent)
|
||||||
futs.add getCards(convo.after)
|
futs.add getCards(convo.after, agent)
|
||||||
futs.add convo.replies.map(getCards)
|
futs.add convo.replies.mapIt(getCards(it, agent))
|
||||||
await all(futs)
|
await all(futs)
|
||||||
|
|
||||||
proc getPhotoRail*(username: string): Future[seq[GalleryPhoto]] {.async.} =
|
proc getPhotoRail*(username, agent: string): Future[seq[GalleryPhoto]] {.async.} =
|
||||||
let headers = newHttpHeaders({
|
let headers = newHttpHeaders({
|
||||||
"Accept": jsonAccept,
|
"Accept": jsonAccept,
|
||||||
"Referer": $(base / username),
|
"Referer": $(base / username),
|
||||||
|
@ -222,7 +221,7 @@ proc getProfileFallback(username: string; headers: HttpHeaders): Future[Profile]
|
||||||
|
|
||||||
result = parseIntentProfile(html)
|
result = parseIntentProfile(html)
|
||||||
|
|
||||||
proc getProfile*(username: string): Future[Profile] {.async.} =
|
proc getProfile*(username, agent: string): Future[Profile] {.async.} =
|
||||||
let headers = newHttpHeaders({
|
let headers = newHttpHeaders({
|
||||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9",
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9",
|
||||||
"Referer": $(base / username),
|
"Referer": $(base / username),
|
||||||
|
@ -248,7 +247,7 @@ proc getProfile*(username: string): Future[Profile] {.async.} =
|
||||||
|
|
||||||
result = parsePopupProfile(html)
|
result = parsePopupProfile(html)
|
||||||
|
|
||||||
proc getTweet*(username, id: string): Future[Conversation] {.async.} =
|
proc getTweet*(username, id, agent: string): Future[Conversation] {.async.} =
|
||||||
let headers = newHttpHeaders({
|
let headers = newHttpHeaders({
|
||||||
"Accept": jsonAccept,
|
"Accept": jsonAccept,
|
||||||
"Referer": $base,
|
"Referer": $base,
|
||||||
|
@ -269,13 +268,13 @@ proc getTweet*(username, id: string): Future[Conversation] {.async.} =
|
||||||
result = parseConversation(html)
|
result = parseConversation(html)
|
||||||
|
|
||||||
let
|
let
|
||||||
vidsFut = getConversationVideos(result)
|
vidsFut = getConversationVideos(result, agent)
|
||||||
pollFut = getConversationPolls(result)
|
pollFut = getConversationPolls(result, agent)
|
||||||
cardFut = getConversationCards(result)
|
cardFut = getConversationCards(result, agent)
|
||||||
|
|
||||||
await all(vidsFut, pollFut, cardFut)
|
await all(vidsFut, pollFut, cardFut)
|
||||||
|
|
||||||
proc finishTimeline(json: JsonNode; query: Option[Query]; after: string): Future[Timeline] {.async.} =
|
proc finishTimeline(json: JsonNode; query: Option[Query]; after, agent: string): Future[Timeline] {.async.} =
|
||||||
if json == nil: return Timeline()
|
if json == nil: return Timeline()
|
||||||
|
|
||||||
result = Timeline(
|
result = Timeline(
|
||||||
|
@ -292,14 +291,14 @@ proc finishTimeline(json: JsonNode; query: Option[Query]; after: string): Future
|
||||||
let
|
let
|
||||||
html = parseHtml(json["items_html"].to(string))
|
html = parseHtml(json["items_html"].to(string))
|
||||||
thread = parseThread(html)
|
thread = parseThread(html)
|
||||||
vidsFut = getVideos(thread)
|
vidsFut = getVideos(thread, agent)
|
||||||
pollFut = getPolls(thread)
|
pollFut = getPolls(thread, agent)
|
||||||
cardFut = getCards(thread)
|
cardFut = getCards(thread, agent)
|
||||||
|
|
||||||
await all(vidsFut, pollFut, cardFut)
|
await all(vidsFut, pollFut, cardFut)
|
||||||
result.tweets = thread.tweets
|
result.tweets = thread.tweets
|
||||||
|
|
||||||
proc getTimeline*(username, after: string): Future[Timeline] {.async.} =
|
proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} =
|
||||||
let headers = newHttpHeaders({
|
let headers = newHttpHeaders({
|
||||||
"Accept": jsonAccept,
|
"Accept": jsonAccept,
|
||||||
"Referer": $(base / username),
|
"Referer": $(base / username),
|
||||||
|
@ -320,9 +319,9 @@ proc getTimeline*(username, after: string): Future[Timeline] {.async.} =
|
||||||
params.add {"max_position": after}
|
params.add {"max_position": after}
|
||||||
|
|
||||||
let json = await fetchJson(base / (timelineUrl % username) ? params, headers)
|
let json = await fetchJson(base / (timelineUrl % username) ? params, headers)
|
||||||
result = await finishTimeline(json, none(Query), after)
|
result = await finishTimeline(json, none(Query), after, agent)
|
||||||
|
|
||||||
proc getTimelineSearch*(username, after: string; query: Query): Future[Timeline] {.async.} =
|
proc getTimelineSearch*(username, after, agent: string; query: Query): Future[Timeline] {.async.} =
|
||||||
let queryParam = genQueryParam(query)
|
let queryParam = genQueryParam(query)
|
||||||
let queryEncoded = encodeUrl(queryParam, usePlus=false)
|
let queryEncoded = encodeUrl(queryParam, usePlus=false)
|
||||||
|
|
||||||
|
@ -347,4 +346,4 @@ proc getTimelineSearch*(username, after: string; query: Query): Future[Timeline]
|
||||||
}
|
}
|
||||||
|
|
||||||
let json = await fetchJson(base / timelineSearchUrl ? params, headers)
|
let json = await fetchJson(base / timelineSearchUrl ? params, headers)
|
||||||
result = await finishTimeline(json, some(query), after)
|
result = await finishTimeline(json, some(query), after, agent)
|
||||||
|
|
|
@ -12,18 +12,18 @@ var profileCacheTime = initDuration(minutes=10)
|
||||||
proc outdated(profile: Profile): bool =
|
proc outdated(profile: Profile): bool =
|
||||||
getTime() - profile.updated > profileCacheTime
|
getTime() - profile.updated > profileCacheTime
|
||||||
|
|
||||||
proc getCachedProfile*(username: string; force=false): Future[Profile] {.async.} =
|
proc getCachedProfile*(username, agent: string; force=false): Future[Profile] {.async.} =
|
||||||
withDb:
|
withDb:
|
||||||
try:
|
try:
|
||||||
result.getOne("username = ?", username)
|
result.getOne("username = ?", username)
|
||||||
doAssert not result.outdated()
|
doAssert not result.outdated()
|
||||||
except AssertionError:
|
except AssertionError:
|
||||||
var profile = await getProfile(username)
|
var profile = await getProfile(username, agent)
|
||||||
profile.id = result.id
|
profile.id = result.id
|
||||||
result = profile
|
result = profile
|
||||||
result.update()
|
result.update()
|
||||||
except KeyError:
|
except KeyError:
|
||||||
result = await getProfile(username)
|
result = await getProfile(username, agent)
|
||||||
if result.username.len > 0:
|
if result.username.len > 0:
|
||||||
result.insert()
|
result.insert()
|
||||||
|
|
||||||
|
|
|
@ -3,7 +3,7 @@ from net import Port
|
||||||
|
|
||||||
import jester, regex
|
import jester, regex
|
||||||
|
|
||||||
import api, utils, types, cache, formatters, search, config
|
import api, utils, types, cache, formatters, search, config, agents
|
||||||
import views/[general, profile, status]
|
import views/[general, profile, status]
|
||||||
|
|
||||||
const configPath {.strdefine.} = "./nitter.conf"
|
const configPath {.strdefine.} = "./nitter.conf"
|
||||||
|
@ -11,15 +11,16 @@ let cfg = getConfig(configPath)
|
||||||
|
|
||||||
proc showTimeline(name, after: string; query: Option[Query]): Future[string] {.async.} =
|
proc showTimeline(name, after: string; query: Option[Query]): Future[string] {.async.} =
|
||||||
let
|
let
|
||||||
|
agent = getAgent()
|
||||||
username = name.strip(chars={'/'})
|
username = name.strip(chars={'/'})
|
||||||
profileFut = getCachedProfile(username)
|
profileFut = getCachedProfile(username, agent)
|
||||||
railFut = getPhotoRail(username)
|
railFut = getPhotoRail(username, agent)
|
||||||
|
|
||||||
var timelineFut: Future[Timeline]
|
var timelineFut: Future[Timeline]
|
||||||
if query.isNone:
|
if query.isNone:
|
||||||
timelineFut = getTimeline(username, after)
|
timelineFut = getTimeline(username, after, agent)
|
||||||
else:
|
else:
|
||||||
timelineFut = getTimelineSearch(username, after, get(query))
|
timelineFut = getTimelineSearch(username, after, agent, get(query))
|
||||||
|
|
||||||
let profile = await profileFut
|
let profile = await profileFut
|
||||||
if profile.username.len == 0:
|
if profile.username.len == 0:
|
||||||
|
@ -69,7 +70,7 @@ routes:
|
||||||
get "/@name/status/@id":
|
get "/@name/status/@id":
|
||||||
cond '.' notin @"name"
|
cond '.' notin @"name"
|
||||||
|
|
||||||
let conversation = await getTweet(@"name", @"id")
|
let conversation = await getTweet(@"name", @"id", getAgent())
|
||||||
if conversation == nil or conversation.tweet.id.len == 0:
|
if conversation == nil or conversation.tweet.id.len == 0:
|
||||||
resp Http404, showError("Tweet not found", cfg.title)
|
resp Http404, showError("Tweet not found", cfg.title)
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue