Refactor api code
This commit is contained in:
Zed
parent
014f01bf88
commit
43bf6735d4
13 changed files with 414 additions and 397 deletions
390
src/api.nim
390
src/api.nim
|
|
@ -1,388 +1,2 @@
|
||||||
import httpclient, asyncdispatch, htmlparser, times
|
import api/[media, profile, timeline, tweet, search]
|
||||||
import sequtils, strutils, json, xmltree, uri
|
export profile, timeline, tweet, search, media
|
||||||
|
|
||||||
import types, parser, parserutils, formatters, search
|
|
||||||
|
|
||||||
const
|
|
||||||
lang = "en-US,en;q=0.9"
|
|
||||||
auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
|
|
||||||
accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
|
|
||||||
jsonAccept = "application/json, text/javascript, */*; q=0.01"
|
|
||||||
|
|
||||||
base = parseUri("https://twitter.com/")
|
|
||||||
apiBase = parseUri("https://api.twitter.com/1.1/")
|
|
||||||
|
|
||||||
timelineUrl = "i/profiles/show/$1/timeline/tweets"
|
|
||||||
timelineMediaUrl = "i/profiles/show/$1/media_timeline"
|
|
||||||
profilePopupUrl = "i/profiles/popup"
|
|
||||||
profileIntentUrl = "intent/user"
|
|
||||||
searchUrl = "i/search/timeline"
|
|
||||||
tweetUrl = "status"
|
|
||||||
videoUrl = "videos/tweet/config/$1.json"
|
|
||||||
tokenUrl = "guest/activate.json"
|
|
||||||
cardUrl = "i/cards/tfw/v1/$1"
|
|
||||||
pollUrl = cardUrl & "?cardname=poll2choice_text_only&lang=en"
|
|
||||||
|
|
||||||
var
|
|
||||||
guestToken = ""
|
|
||||||
tokenUses = 0
|
|
||||||
tokenMaxUses = 230
|
|
||||||
tokenUpdated: Time
|
|
||||||
tokenLifetime = initDuration(minutes=20)
|
|
||||||
|
|
||||||
macro genMediaGet(media: untyped; token=false) =
|
|
||||||
let
|
|
||||||
mediaName = capitalizeAscii($media)
|
|
||||||
multi = ident("get" & mediaName & "s")
|
|
||||||
convo = ident("getConversation" & mediaName & "s")
|
|
||||||
single = ident("get" & mediaName)
|
|
||||||
|
|
||||||
quote do:
|
|
||||||
proc `multi`(thread: Thread | Timeline; agent: string; token="") {.async.} =
|
|
||||||
if thread == nil: return
|
|
||||||
var `media` = thread.content.filterIt(it.`media`.isSome)
|
|
||||||
when `token`:
|
|
||||||
var gToken = token
|
|
||||||
if gToken.len == 0: gToken = await getGuestToken(agent)
|
|
||||||
await all(`media`.mapIt(`single`(it, token, agent)))
|
|
||||||
else:
|
|
||||||
await all(`media`.mapIt(`single`(it, agent)))
|
|
||||||
|
|
||||||
proc `convo`(convo: Conversation; agent: string) {.async.} =
|
|
||||||
var futs: seq[Future[void]]
|
|
||||||
when `token`:
|
|
||||||
var token = await getGuestToken(agent)
|
|
||||||
futs.add `single`(convo.tweet, agent, token)
|
|
||||||
futs.add `multi`(convo.before, agent, token=token)
|
|
||||||
futs.add `multi`(convo.after, agent, token=token)
|
|
||||||
futs.add convo.replies.mapIt(`multi`(it, agent, token=token))
|
|
||||||
else:
|
|
||||||
futs.add `single`(convo.tweet, agent)
|
|
||||||
futs.add `multi`(convo.before, agent)
|
|
||||||
futs.add `multi`(convo.after, agent)
|
|
||||||
futs.add convo.replies.mapIt(`multi`(it, agent))
|
|
||||||
await all(futs)
|
|
||||||
|
|
||||||
template newClient() {.dirty.} =
|
|
||||||
var client = newAsyncHttpClient()
|
|
||||||
defer: client.close()
|
|
||||||
client.headers = headers
|
|
||||||
|
|
||||||
proc fetchHtml(url: Uri; headers: HttpHeaders; jsonKey = ""): Future[XmlNode] {.async.} =
|
|
||||||
newClient()
|
|
||||||
|
|
||||||
var resp = ""
|
|
||||||
try:
|
|
||||||
resp = await client.getContent($url)
|
|
||||||
except:
|
|
||||||
return nil
|
|
||||||
|
|
||||||
if jsonKey.len > 0:
|
|
||||||
let json = parseJson(resp)[jsonKey].str
|
|
||||||
return parseHtml(json)
|
|
||||||
else:
|
|
||||||
return parseHtml(resp)
|
|
||||||
|
|
||||||
proc fetchJson(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} =
|
|
||||||
newClient()
|
|
||||||
|
|
||||||
var resp = ""
|
|
||||||
try:
|
|
||||||
resp = await client.getContent($url)
|
|
||||||
result = parseJson(resp)
|
|
||||||
except:
|
|
||||||
return nil
|
|
||||||
|
|
||||||
proc getGuestToken(agent: string; force=false): Future[string] {.async.} =
|
|
||||||
if getTime() - tokenUpdated < tokenLifetime and
|
|
||||||
not force and tokenUses < tokenMaxUses:
|
|
||||||
return guestToken
|
|
||||||
|
|
||||||
tokenUpdated = getTime()
|
|
||||||
tokenUses = 0
|
|
||||||
|
|
||||||
let headers = newHttpHeaders({
|
|
||||||
"Accept": jsonAccept,
|
|
||||||
"Referer": $base,
|
|
||||||
"User-Agent": agent,
|
|
||||||
"Authorization": auth
|
|
||||||
})
|
|
||||||
|
|
||||||
newClient()
|
|
||||||
|
|
||||||
let
|
|
||||||
url = apiBase / tokenUrl
|
|
||||||
json = parseJson(await client.postContent($url))
|
|
||||||
|
|
||||||
result = json["guest_token"].to(string)
|
|
||||||
guestToken = result
|
|
||||||
|
|
||||||
proc getVideoFetch*(tweet: Tweet; agent, token: string) {.async.} =
|
|
||||||
if tweet.video.isNone(): return
|
|
||||||
|
|
||||||
let headers = newHttpHeaders({
|
|
||||||
"Accept": jsonAccept,
|
|
||||||
"Referer": $(base / getLink(tweet)),
|
|
||||||
"User-Agent": agent,
|
|
||||||
"Authorization": auth,
|
|
||||||
"x-guest-token": token
|
|
||||||
})
|
|
||||||
|
|
||||||
let url = apiBase / (videoUrl % tweet.id)
|
|
||||||
let json = await fetchJson(url, headers)
|
|
||||||
|
|
||||||
if json == nil:
|
|
||||||
if getTime() - tokenUpdated > initDuration(seconds=1):
|
|
||||||
tokenUpdated = getTime()
|
|
||||||
discard await getGuestToken(agent, force=true)
|
|
||||||
await getVideoFetch(tweet, agent, guestToken)
|
|
||||||
return
|
|
||||||
|
|
||||||
if tweet.card.isNone:
|
|
||||||
tweet.video = some(parseVideo(json, tweet.id))
|
|
||||||
else:
|
|
||||||
get(tweet.card).video = some(parseVideo(json, tweet.id))
|
|
||||||
tweet.video = none(Video)
|
|
||||||
tokenUses.inc
|
|
||||||
|
|
||||||
proc getVideoVar*(tweet: Tweet): var Option[Video] =
|
|
||||||
if tweet.card.isSome():
|
|
||||||
return get(tweet.card).video
|
|
||||||
else:
|
|
||||||
return tweet.video
|
|
||||||
|
|
||||||
proc getVideo*(tweet: Tweet; agent, token: string; force=false) {.async.} =
|
|
||||||
withDb:
|
|
||||||
try:
|
|
||||||
getVideoVar(tweet) = some(Video.getOne("videoId = ?", tweet.id))
|
|
||||||
except KeyError:
|
|
||||||
await getVideoFetch(tweet, agent, token)
|
|
||||||
var video = getVideoVar(tweet)
|
|
||||||
if video.isSome():
|
|
||||||
get(video).insert()
|
|
||||||
|
|
||||||
proc getPoll*(tweet: Tweet; agent: string) {.async.} =
|
|
||||||
if tweet.poll.isNone(): return
|
|
||||||
|
|
||||||
let headers = newHttpHeaders({
|
|
||||||
"Accept": accept,
|
|
||||||
"Referer": $(base / getLink(tweet)),
|
|
||||||
"User-Agent": agent,
|
|
||||||
"Authority": "twitter.com",
|
|
||||||
"Accept-Language": lang,
|
|
||||||
})
|
|
||||||
|
|
||||||
let url = base / (pollUrl % tweet.id)
|
|
||||||
let html = await fetchHtml(url, headers)
|
|
||||||
if html == nil: return
|
|
||||||
|
|
||||||
tweet.poll = some(parsePoll(html))
|
|
||||||
|
|
||||||
proc getCard*(tweet: Tweet; agent: string) {.async.} =
|
|
||||||
if tweet.card.isNone(): return
|
|
||||||
|
|
||||||
let headers = newHttpHeaders({
|
|
||||||
"Accept": accept,
|
|
||||||
"Referer": $(base / getLink(tweet)),
|
|
||||||
"User-Agent": agent,
|
|
||||||
"Authority": "twitter.com",
|
|
||||||
"Accept-Language": lang,
|
|
||||||
})
|
|
||||||
|
|
||||||
let query = get(tweet.card).query.replace("sensitive=true", "sensitive=false")
|
|
||||||
let html = await fetchHtml(base / query, headers)
|
|
||||||
if html == nil: return
|
|
||||||
|
|
||||||
parseCard(get(tweet.card), html)
|
|
||||||
|
|
||||||
genMediaGet(video, token=true)
|
|
||||||
genMediaGet(poll)
|
|
||||||
genMediaGet(card)
|
|
||||||
|
|
||||||
proc getPhotoRail*(username, agent: string): Future[seq[GalleryPhoto]] {.async.} =
|
|
||||||
let headers = newHttpHeaders({
|
|
||||||
"Accept": jsonAccept,
|
|
||||||
"Referer": $(base / username),
|
|
||||||
"User-Agent": agent,
|
|
||||||
"X-Requested-With": "XMLHttpRequest"
|
|
||||||
})
|
|
||||||
|
|
||||||
let params = {
|
|
||||||
"for_photo_rail": "true",
|
|
||||||
"oldest_unread_id": "0"
|
|
||||||
}
|
|
||||||
|
|
||||||
let url = base / (timelineMediaUrl % username) ? params
|
|
||||||
let html = await fetchHtml(url, headers, jsonKey="items_html")
|
|
||||||
|
|
||||||
result = parsePhotoRail(html)
|
|
||||||
|
|
||||||
proc getProfileFallback(username: string; headers: HttpHeaders): Future[Profile] {.async.} =
|
|
||||||
let url = base / profileIntentUrl ? {"screen_name": username}
|
|
||||||
let html = await fetchHtml(url, headers)
|
|
||||||
if html == nil: return Profile()
|
|
||||||
|
|
||||||
result = parseIntentProfile(html)
|
|
||||||
|
|
||||||
proc getProfile*(username, agent: string): Future[Profile] {.async.} =
|
|
||||||
let headers = newHttpHeaders({
|
|
||||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9",
|
|
||||||
"Referer": $(base / username),
|
|
||||||
"User-Agent": agent,
|
|
||||||
"X-Twitter-Active-User": "yes",
|
|
||||||
"X-Requested-With": "XMLHttpRequest",
|
|
||||||
"Accept-Language": lang
|
|
||||||
})
|
|
||||||
|
|
||||||
let
|
|
||||||
params = {
|
|
||||||
"screen_name": username,
|
|
||||||
"wants_hovercard": "true",
|
|
||||||
"_": $(epochTime().int)
|
|
||||||
}
|
|
||||||
url = base / profilePopupUrl ? params
|
|
||||||
html = await fetchHtml(url, headers, jsonKey="html")
|
|
||||||
|
|
||||||
if html == nil: return Profile()
|
|
||||||
|
|
||||||
if html.select(".ProfileCard-sensitiveWarningContainer") != nil:
|
|
||||||
return await getProfileFallback(username, headers)
|
|
||||||
|
|
||||||
result = parsePopupProfile(html)
|
|
||||||
|
|
||||||
proc getTweet*(username, id, agent: string): Future[Conversation] {.async.} =
|
|
||||||
let headers = newHttpHeaders({
|
|
||||||
"Accept": jsonAccept,
|
|
||||||
"Referer": $base,
|
|
||||||
"User-Agent": agent,
|
|
||||||
"X-Twitter-Active-User": "yes",
|
|
||||||
"X-Requested-With": "XMLHttpRequest",
|
|
||||||
"Accept-Language": lang,
|
|
||||||
"pragma": "no-cache",
|
|
||||||
"x-previous-page-name": "profile"
|
|
||||||
})
|
|
||||||
|
|
||||||
let
|
|
||||||
url = base / username / tweetUrl / id
|
|
||||||
html = await fetchHtml(url, headers)
|
|
||||||
|
|
||||||
if html == nil: return
|
|
||||||
|
|
||||||
result = parseConversation(html)
|
|
||||||
|
|
||||||
let
|
|
||||||
vidsFut = getConversationVideos(result, agent)
|
|
||||||
pollFut = getConversationPolls(result, agent)
|
|
||||||
cardFut = getConversationCards(result, agent)
|
|
||||||
|
|
||||||
await all(vidsFut, pollFut, cardFut)
|
|
||||||
|
|
||||||
proc finishTimeline(json: JsonNode; query: Option[Query]; after, agent: string): Future[Timeline] {.async.} =
|
|
||||||
if json == nil: return Timeline()
|
|
||||||
|
|
||||||
result = Timeline(
|
|
||||||
hasMore: json["has_more_items"].to(bool),
|
|
||||||
maxId: json.getOrDefault("max_position").getStr(""),
|
|
||||||
minId: json.getOrDefault("min_position").getStr("").cleanPos(),
|
|
||||||
query: query,
|
|
||||||
beginning: after.len == 0
|
|
||||||
)
|
|
||||||
|
|
||||||
if json["new_latent_count"].to(int) == 0: return
|
|
||||||
if not json.hasKey("items_html"): return
|
|
||||||
|
|
||||||
let
|
|
||||||
html = parseHtml(json["items_html"].to(string))
|
|
||||||
thread = parseThread(html)
|
|
||||||
vidsFut = getVideos(thread, agent)
|
|
||||||
pollFut = getPolls(thread, agent)
|
|
||||||
cardFut = getCards(thread, agent)
|
|
||||||
|
|
||||||
await all(vidsFut, pollFut, cardFut)
|
|
||||||
result.content = thread.content
|
|
||||||
|
|
||||||
proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} =
|
|
||||||
let headers = newHttpHeaders({
|
|
||||||
"Accept": jsonAccept,
|
|
||||||
"Referer": $(base / username),
|
|
||||||
"User-Agent": agent,
|
|
||||||
"X-Twitter-Active-User": "yes",
|
|
||||||
"X-Requested-With": "XMLHttpRequest",
|
|
||||||
"Accept-Language": lang
|
|
||||||
})
|
|
||||||
|
|
||||||
var params = toSeq({
|
|
||||||
"include_available_features": "1",
|
|
||||||
"include_entities": "1",
|
|
||||||
"include_new_items_bar": "false",
|
|
||||||
"reset_error_state": "false"
|
|
||||||
})
|
|
||||||
|
|
||||||
if after.len > 0:
|
|
||||||
params.add {"max_position": after}
|
|
||||||
|
|
||||||
let json = await fetchJson(base / (timelineUrl % username) ? params, headers)
|
|
||||||
result = await finishTimeline(json, none(Query), after, agent)
|
|
||||||
|
|
||||||
proc getTimelineSearch*(query: Query; after, agent: string): Future[Timeline] {.async.} =
|
|
||||||
let queryParam = genQueryParam(query)
|
|
||||||
let queryEncoded = encodeUrl(queryParam, usePlus=false)
|
|
||||||
|
|
||||||
let headers = newHttpHeaders({
|
|
||||||
"Accept": jsonAccept,
|
|
||||||
"Referer": $(base / ("search?f=tweets&vertical=default&q=$1&src=typd" % queryEncoded)),
|
|
||||||
"User-Agent": agent,
|
|
||||||
"X-Requested-With": "XMLHttpRequest",
|
|
||||||
"Authority": "twitter.com",
|
|
||||||
"Accept-Language": lang
|
|
||||||
})
|
|
||||||
|
|
||||||
let params = {
|
|
||||||
"f": "tweets",
|
|
||||||
"vertical": "default",
|
|
||||||
"q": queryParam,
|
|
||||||
"src": "typd",
|
|
||||||
"include_available_features": "1",
|
|
||||||
"include_entities": "1",
|
|
||||||
"max_position": if after.len > 0: genPos(after) else: "0",
|
|
||||||
"reset_error_state": "false"
|
|
||||||
}
|
|
||||||
|
|
||||||
let json = await fetchJson(base / searchUrl ? params, headers)
|
|
||||||
result = await finishTimeline(json, some(query), after, agent)
|
|
||||||
|
|
||||||
proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Timeline)] {.async.} =
|
|
||||||
let headers = newHttpHeaders({
|
|
||||||
"authority": "twitter.com",
|
|
||||||
"accept": accept,
|
|
||||||
"referer": "https://twitter.com/" & username,
|
|
||||||
"accept-language": lang
|
|
||||||
})
|
|
||||||
|
|
||||||
var url = base / username
|
|
||||||
if after.len > 0:
|
|
||||||
url = url ? {"max_position": after}
|
|
||||||
|
|
||||||
let
|
|
||||||
html = await fetchHtml(url, headers)
|
|
||||||
timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
|
|
||||||
profile = parseTimelineProfile(html)
|
|
||||||
|
|
||||||
vidsFut = getVideos(timeline, agent)
|
|
||||||
pollFut = getPolls(timeline, agent)
|
|
||||||
cardFut = getCards(timeline, agent)
|
|
||||||
|
|
||||||
await all(vidsFut, pollFut, cardFut)
|
|
||||||
result = (profile, timeline)
|
|
||||||
|
|
||||||
proc getProfileFull*(username: string): Future[Profile] {.async.} =
|
|
||||||
let headers = newHttpHeaders({
|
|
||||||
"authority": "twitter.com",
|
|
||||||
"accept": accept,
|
|
||||||
"referer": "https://twitter.com/" & username,
|
|
||||||
"accept-language": lang
|
|
||||||
})
|
|
||||||
|
|
||||||
let html = await fetchHtml(base / username, headers)
|
|
||||||
if html == nil: return
|
|
||||||
result = parseTimelineProfile(html)
|
|
||||||
|
|
|
||||||
21
src/api/consts.nim
Normal file
21
src/api/consts.nim
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
import uri
|
||||||
|
|
||||||
|
const
|
||||||
|
lang* = "en-US,en;q=0.9"
|
||||||
|
auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
|
||||||
|
htmlAccept* = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"
|
||||||
|
jsonAccept* = "application/json, text/javascript, */*; q=0.01"
|
||||||
|
|
||||||
|
base* = parseUri("https://twitter.com/")
|
||||||
|
apiBase* = parseUri("https://api.twitter.com/1.1/")
|
||||||
|
|
||||||
|
timelineUrl* = "i/profiles/show/$1/timeline/tweets"
|
||||||
|
timelineMediaUrl* = "i/profiles/show/$1/media_timeline"
|
||||||
|
profilePopupUrl* = "i/profiles/popup"
|
||||||
|
profileIntentUrl* = "intent/user"
|
||||||
|
searchUrl* = "i/search/timeline"
|
||||||
|
tweetUrl* = "status"
|
||||||
|
videoUrl* = "videos/tweet/config/$1.json"
|
||||||
|
tokenUrl* = "guest/activate.json"
|
||||||
|
cardUrl* = "i/cards/tfw/v1/$1"
|
||||||
|
pollUrl* = cardUrl & "?cardname=poll2choice_text_only&lang=en"
|
||||||
168
src/api/media.nim
Normal file
168
src/api/media.nim
Normal file
|
|
@ -0,0 +1,168 @@
|
||||||
|
import httpclient, asyncdispatch, times, sequtils, strutils, json, uri
|
||||||
|
|
||||||
|
import ".."/[types, parser, formatters]
|
||||||
|
import utils, consts
|
||||||
|
|
||||||
|
var
|
||||||
|
guestToken = ""
|
||||||
|
tokenUses = 0
|
||||||
|
tokenMaxUses = 230
|
||||||
|
tokenUpdated: Time
|
||||||
|
tokenLifetime = initDuration(minutes=20)
|
||||||
|
|
||||||
|
macro genMediaGet(media: untyped; token=false) =
|
||||||
|
let
|
||||||
|
mediaName = capitalizeAscii($media)
|
||||||
|
multi = ident("get" & mediaName & "s")
|
||||||
|
convo = ident("getConversation" & mediaName & "s")
|
||||||
|
single = ident("get" & mediaName)
|
||||||
|
|
||||||
|
quote do:
|
||||||
|
proc `multi`*(thread: Thread | Timeline; agent: string; token="") {.async.} =
|
||||||
|
if thread == nil: return
|
||||||
|
var `media` = thread.content.filterIt(it.`media`.isSome)
|
||||||
|
when `token`:
|
||||||
|
var gToken = token
|
||||||
|
if gToken.len == 0: gToken = await getGuestToken(agent)
|
||||||
|
await all(`media`.mapIt(`single`(it, token, agent)))
|
||||||
|
else:
|
||||||
|
await all(`media`.mapIt(`single`(it, agent)))
|
||||||
|
|
||||||
|
proc `convo`*(convo: Conversation; agent: string) {.async.} =
|
||||||
|
var futs: seq[Future[void]]
|
||||||
|
when `token`:
|
||||||
|
var token = await getGuestToken(agent)
|
||||||
|
futs.add `single`(convo.tweet, agent, token)
|
||||||
|
futs.add `multi`(convo.before, agent, token=token)
|
||||||
|
futs.add `multi`(convo.after, agent, token=token)
|
||||||
|
futs.add convo.replies.mapIt(`multi`(it, agent, token=token))
|
||||||
|
else:
|
||||||
|
futs.add `single`(convo.tweet, agent)
|
||||||
|
futs.add `multi`(convo.before, agent)
|
||||||
|
futs.add `multi`(convo.after, agent)
|
||||||
|
futs.add convo.replies.mapIt(`multi`(it, agent))
|
||||||
|
await all(futs)
|
||||||
|
|
||||||
|
proc getGuestToken(agent: string; force=false): Future[string] {.async.} =
|
||||||
|
if getTime() - tokenUpdated < tokenLifetime and
|
||||||
|
not force and tokenUses < tokenMaxUses:
|
||||||
|
return guestToken
|
||||||
|
|
||||||
|
tokenUpdated = getTime()
|
||||||
|
tokenUses = 0
|
||||||
|
|
||||||
|
let headers = newHttpHeaders({
|
||||||
|
"Accept": jsonAccept,
|
||||||
|
"Referer": $base,
|
||||||
|
"User-Agent": agent,
|
||||||
|
"Authorization": auth
|
||||||
|
})
|
||||||
|
|
||||||
|
newClient()
|
||||||
|
|
||||||
|
let
|
||||||
|
url = apiBase / tokenUrl
|
||||||
|
json = parseJson(await client.postContent($url))
|
||||||
|
|
||||||
|
result = json["guest_token"].to(string)
|
||||||
|
guestToken = result
|
||||||
|
|
||||||
|
proc getVideoFetch(tweet: Tweet; agent, token: string) {.async.} =
|
||||||
|
if tweet.video.isNone(): return
|
||||||
|
|
||||||
|
let headers = newHttpHeaders({
|
||||||
|
"Accept": jsonAccept,
|
||||||
|
"Referer": $(base / getLink(tweet)),
|
||||||
|
"User-Agent": agent,
|
||||||
|
"Authorization": auth,
|
||||||
|
"x-guest-token": token
|
||||||
|
})
|
||||||
|
|
||||||
|
let url = apiBase / (videoUrl % tweet.id)
|
||||||
|
let json = await fetchJson(url, headers)
|
||||||
|
|
||||||
|
if json == nil:
|
||||||
|
if getTime() - tokenUpdated > initDuration(seconds=1):
|
||||||
|
tokenUpdated = getTime()
|
||||||
|
discard await getGuestToken(agent, force=true)
|
||||||
|
await getVideoFetch(tweet, agent, guestToken)
|
||||||
|
return
|
||||||
|
|
||||||
|
if tweet.card.isNone:
|
||||||
|
tweet.video = some(parseVideo(json, tweet.id))
|
||||||
|
else:
|
||||||
|
get(tweet.card).video = some(parseVideo(json, tweet.id))
|
||||||
|
tweet.video = none(Video)
|
||||||
|
tokenUses.inc
|
||||||
|
|
||||||
|
proc getVideoVar(tweet: Tweet): var Option[Video] =
|
||||||
|
if tweet.card.isSome():
|
||||||
|
return get(tweet.card).video
|
||||||
|
else:
|
||||||
|
return tweet.video
|
||||||
|
|
||||||
|
proc getVideo*(tweet: Tweet; agent, token: string; force=false) {.async.} =
|
||||||
|
withDb:
|
||||||
|
try:
|
||||||
|
getVideoVar(tweet) = some(Video.getOne("videoId = ?", tweet.id))
|
||||||
|
except KeyError:
|
||||||
|
await getVideoFetch(tweet, agent, token)
|
||||||
|
var video = getVideoVar(tweet)
|
||||||
|
if video.isSome():
|
||||||
|
get(video).insert()
|
||||||
|
|
||||||
|
proc getPoll*(tweet: Tweet; agent: string) {.async.} =
|
||||||
|
if tweet.poll.isNone(): return
|
||||||
|
|
||||||
|
let headers = newHttpHeaders({
|
||||||
|
"Accept": htmlAccept,
|
||||||
|
"Referer": $(base / getLink(tweet)),
|
||||||
|
"User-Agent": agent,
|
||||||
|
"Authority": "twitter.com",
|
||||||
|
"Accept-Language": lang,
|
||||||
|
})
|
||||||
|
|
||||||
|
let url = base / (pollUrl % tweet.id)
|
||||||
|
let html = await fetchHtml(url, headers)
|
||||||
|
if html == nil: return
|
||||||
|
|
||||||
|
tweet.poll = some(parsePoll(html))
|
||||||
|
|
||||||
|
proc getCard*(tweet: Tweet; agent: string) {.async.} =
|
||||||
|
if tweet.card.isNone(): return
|
||||||
|
|
||||||
|
let headers = newHttpHeaders({
|
||||||
|
"Accept": htmlAccept,
|
||||||
|
"Referer": $(base / getLink(tweet)),
|
||||||
|
"User-Agent": agent,
|
||||||
|
"Authority": "twitter.com",
|
||||||
|
"Accept-Language": lang,
|
||||||
|
})
|
||||||
|
|
||||||
|
let query = get(tweet.card).query.replace("sensitive=true", "sensitive=false")
|
||||||
|
let html = await fetchHtml(base / query, headers)
|
||||||
|
if html == nil: return
|
||||||
|
|
||||||
|
parseCard(get(tweet.card), html)
|
||||||
|
|
||||||
|
proc getPhotoRail*(username, agent: string): Future[seq[GalleryPhoto]] {.async.} =
|
||||||
|
let headers = newHttpHeaders({
|
||||||
|
"Accept": jsonAccept,
|
||||||
|
"Referer": $(base / username),
|
||||||
|
"User-Agent": agent,
|
||||||
|
"X-Requested-With": "XMLHttpRequest"
|
||||||
|
})
|
||||||
|
|
||||||
|
let params = {
|
||||||
|
"for_photo_rail": "true",
|
||||||
|
"oldest_unread_id": "0"
|
||||||
|
}
|
||||||
|
|
||||||
|
let url = base / (timelineMediaUrl % username) ? params
|
||||||
|
let html = await fetchHtml(url, headers, jsonKey="items_html")
|
||||||
|
|
||||||
|
result = parsePhotoRail(html)
|
||||||
|
|
||||||
|
genMediaGet(video, token=true)
|
||||||
|
genMediaGet(poll)
|
||||||
|
genMediaGet(card)
|
||||||
50
src/api/profile.nim
Normal file
50
src/api/profile.nim
Normal file
|
|
@ -0,0 +1,50 @@
|
||||||
|
import httpclient, asyncdispatch, times, strutils, uri
|
||||||
|
|
||||||
|
import ".."/[types, parser, parserutils]
|
||||||
|
|
||||||
|
import utils, consts
|
||||||
|
|
||||||
|
proc getProfileFallback(username: string; headers: HttpHeaders): Future[Profile] {.async.} =
|
||||||
|
let url = base / profileIntentUrl ? {"screen_name": username}
|
||||||
|
let html = await fetchHtml(url, headers)
|
||||||
|
if html == nil: return Profile()
|
||||||
|
|
||||||
|
result = parseIntentProfile(html)
|
||||||
|
|
||||||
|
proc getProfile*(username, agent: string): Future[Profile] {.async.} =
|
||||||
|
let headers = newHttpHeaders({
|
||||||
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9",
|
||||||
|
"Referer": $(base / username),
|
||||||
|
"User-Agent": agent,
|
||||||
|
"X-Twitter-Active-User": "yes",
|
||||||
|
"X-Requested-With": "XMLHttpRequest",
|
||||||
|
"Accept-Language": lang
|
||||||
|
})
|
||||||
|
|
||||||
|
let
|
||||||
|
params = {
|
||||||
|
"screen_name": username,
|
||||||
|
"wants_hovercard": "true",
|
||||||
|
"_": $(epochTime().int)
|
||||||
|
}
|
||||||
|
url = base / profilePopupUrl ? params
|
||||||
|
html = await fetchHtml(url, headers, jsonKey="html")
|
||||||
|
|
||||||
|
if html == nil: return Profile()
|
||||||
|
|
||||||
|
if html.select(".ProfileCard-sensitiveWarningContainer") != nil:
|
||||||
|
return await getProfileFallback(username, headers)
|
||||||
|
|
||||||
|
result = parsePopupProfile(html)
|
||||||
|
|
||||||
|
proc getProfileFull*(username: string): Future[Profile] {.async.} =
|
||||||
|
let headers = newHttpHeaders({
|
||||||
|
"authority": "twitter.com",
|
||||||
|
"accept": htmlAccept,
|
||||||
|
"referer": "https://twitter.com/" & username,
|
||||||
|
"accept-language": lang
|
||||||
|
})
|
||||||
|
|
||||||
|
let html = await fetchHtml(base / username, headers)
|
||||||
|
if html == nil: return
|
||||||
|
result = parseTimelineProfile(html)
|
||||||
32
src/api/search.nim
Normal file
32
src/api/search.nim
Normal file
|
|
@ -0,0 +1,32 @@
|
||||||
|
import httpclient, asyncdispatch, htmlparser
|
||||||
|
import sequtils, strutils, json, xmltree, uri
|
||||||
|
|
||||||
|
import ".."/[types, parser, parserutils, formatters, search]
|
||||||
|
import utils, consts, media, timeline
|
||||||
|
|
||||||
|
proc getTimelineSearch*(query: Query; after, agent: string): Future[Timeline] {.async.} =
|
||||||
|
let queryParam = genQueryParam(query)
|
||||||
|
let queryEncoded = encodeUrl(queryParam, usePlus=false)
|
||||||
|
|
||||||
|
let headers = newHttpHeaders({
|
||||||
|
"Accept": jsonAccept,
|
||||||
|
"Referer": $(base / ("search?f=tweets&vertical=default&q=$1&src=typd" % queryEncoded)),
|
||||||
|
"User-Agent": agent,
|
||||||
|
"X-Requested-With": "XMLHttpRequest",
|
||||||
|
"Authority": "twitter.com",
|
||||||
|
"Accept-Language": lang
|
||||||
|
})
|
||||||
|
|
||||||
|
let params = {
|
||||||
|
"f": "tweets",
|
||||||
|
"vertical": "default",
|
||||||
|
"q": queryParam,
|
||||||
|
"src": "typd",
|
||||||
|
"include_available_features": "1",
|
||||||
|
"include_entities": "1",
|
||||||
|
"max_position": if after.len > 0: genPos(after) else: "0",
|
||||||
|
"reset_error_state": "false"
|
||||||
|
}
|
||||||
|
|
||||||
|
let json = await fetchJson(base / searchUrl ? params, headers)
|
||||||
|
result = await finishTimeline(json, some(query), after, agent)
|
||||||
76
src/api/timeline.nim
Normal file
76
src/api/timeline.nim
Normal file
|
|
@ -0,0 +1,76 @@
|
||||||
|
import httpclient, asyncdispatch, htmlparser
|
||||||
|
import sequtils, strutils, json, xmltree, uri
|
||||||
|
|
||||||
|
import ".."/[types, parser, parserutils, formatters, search]
|
||||||
|
import utils, consts, media
|
||||||
|
|
||||||
|
proc finishTimeline*(json: JsonNode; query: Option[Query]; after, agent: string): Future[Timeline] {.async.} =
|
||||||
|
if json == nil: return Timeline()
|
||||||
|
|
||||||
|
result = Timeline(
|
||||||
|
hasMore: json["has_more_items"].to(bool),
|
||||||
|
maxId: json.getOrDefault("max_position").getStr(""),
|
||||||
|
minId: json.getOrDefault("min_position").getStr("").cleanPos(),
|
||||||
|
query: query,
|
||||||
|
beginning: after.len == 0
|
||||||
|
)
|
||||||
|
|
||||||
|
if json["new_latent_count"].to(int) == 0: return
|
||||||
|
if not json.hasKey("items_html"): return
|
||||||
|
|
||||||
|
let
|
||||||
|
html = parseHtml(json["items_html"].to(string))
|
||||||
|
thread = parseThread(html)
|
||||||
|
vidsFut = getVideos(thread, agent)
|
||||||
|
pollFut = getPolls(thread, agent)
|
||||||
|
cardFut = getCards(thread, agent)
|
||||||
|
|
||||||
|
await all(vidsFut, pollFut, cardFut)
|
||||||
|
result.content = thread.content
|
||||||
|
|
||||||
|
proc getTimeline*(username, after, agent: string): Future[Timeline] {.async.} =
|
||||||
|
let headers = newHttpHeaders({
|
||||||
|
"Accept": jsonAccept,
|
||||||
|
"Referer": $(base / username),
|
||||||
|
"User-Agent": agent,
|
||||||
|
"X-Twitter-Active-User": "yes",
|
||||||
|
"X-Requested-With": "XMLHttpRequest",
|
||||||
|
"Accept-Language": lang
|
||||||
|
})
|
||||||
|
|
||||||
|
var params = toSeq({
|
||||||
|
"include_available_features": "1",
|
||||||
|
"include_entities": "1",
|
||||||
|
"include_new_items_bar": "false",
|
||||||
|
"reset_error_state": "false"
|
||||||
|
})
|
||||||
|
|
||||||
|
if after.len > 0:
|
||||||
|
params.add {"max_position": after}
|
||||||
|
|
||||||
|
let json = await fetchJson(base / (timelineUrl % username) ? params, headers)
|
||||||
|
result = await finishTimeline(json, none(Query), after, agent)
|
||||||
|
|
||||||
|
proc getProfileAndTimeline*(username, agent, after: string): Future[(Profile, Timeline)] {.async.} =
|
||||||
|
let headers = newHttpHeaders({
|
||||||
|
"authority": "twitter.com",
|
||||||
|
"accept": htmlAccept,
|
||||||
|
"referer": "https://twitter.com/" & username,
|
||||||
|
"accept-language": lang
|
||||||
|
})
|
||||||
|
|
||||||
|
var url = base / username
|
||||||
|
if after.len > 0:
|
||||||
|
url = url ? {"max_position": after}
|
||||||
|
|
||||||
|
let
|
||||||
|
html = await fetchHtml(url, headers)
|
||||||
|
timeline = parseTimeline(html.select("#timeline > .stream-container"), after)
|
||||||
|
profile = parseTimelineProfile(html)
|
||||||
|
|
||||||
|
vidsFut = getVideos(timeline, agent)
|
||||||
|
pollFut = getPolls(timeline, agent)
|
||||||
|
cardFut = getCards(timeline, agent)
|
||||||
|
|
||||||
|
await all(vidsFut, pollFut, cardFut)
|
||||||
|
result = (profile, timeline)
|
||||||
31
src/api/tweet.nim
Normal file
31
src/api/tweet.nim
Normal file
|
|
@ -0,0 +1,31 @@
|
||||||
|
import httpclient, asyncdispatch, strutils, uri
|
||||||
|
|
||||||
|
import ".."/[types, parser]
|
||||||
|
import utils, consts, media
|
||||||
|
|
||||||
|
proc getTweet*(username, id, agent: string): Future[Conversation] {.async.} =
|
||||||
|
let headers = newHttpHeaders({
|
||||||
|
"Accept": jsonAccept,
|
||||||
|
"Referer": $base,
|
||||||
|
"User-Agent": agent,
|
||||||
|
"X-Twitter-Active-User": "yes",
|
||||||
|
"X-Requested-With": "XMLHttpRequest",
|
||||||
|
"Accept-Language": lang,
|
||||||
|
"pragma": "no-cache",
|
||||||
|
"x-previous-page-name": "profile"
|
||||||
|
})
|
||||||
|
|
||||||
|
let
|
||||||
|
url = base / username / tweetUrl / id
|
||||||
|
html = await fetchHtml(url, headers)
|
||||||
|
|
||||||
|
if html == nil: return
|
||||||
|
|
||||||
|
result = parseConversation(html)
|
||||||
|
|
||||||
|
let
|
||||||
|
vidsFut = getConversationVideos(result, agent)
|
||||||
|
pollFut = getConversationPolls(result, agent)
|
||||||
|
cardFut = getConversationCards(result, agent)
|
||||||
|
|
||||||
|
await all(vidsFut, pollFut, cardFut)
|
||||||
32
src/api/utils.nim
Normal file
32
src/api/utils.nim
Normal file
|
|
@ -0,0 +1,32 @@
|
||||||
|
import httpclient, asyncdispatch, htmlparser
|
||||||
|
import strutils, json, xmltree, uri
|
||||||
|
|
||||||
|
template newClient*() {.dirty.} =
|
||||||
|
var client = newAsyncHttpClient()
|
||||||
|
defer: client.close()
|
||||||
|
client.headers = headers
|
||||||
|
|
||||||
|
proc fetchHtml*(url: Uri; headers: HttpHeaders; jsonKey = ""): Future[XmlNode] {.async.} =
|
||||||
|
newClient()
|
||||||
|
|
||||||
|
var resp = ""
|
||||||
|
try:
|
||||||
|
resp = await client.getContent($url)
|
||||||
|
except:
|
||||||
|
return nil
|
||||||
|
|
||||||
|
if jsonKey.len > 0:
|
||||||
|
let json = parseJson(resp)[jsonKey].str
|
||||||
|
return parseHtml(json)
|
||||||
|
else:
|
||||||
|
return parseHtml(resp)
|
||||||
|
|
||||||
|
proc fetchJson*(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} =
|
||||||
|
newClient()
|
||||||
|
|
||||||
|
var resp = ""
|
||||||
|
try:
|
||||||
|
resp = await client.getContent($url)
|
||||||
|
result = parseJson(resp)
|
||||||
|
except:
|
||||||
|
return nil
|
||||||
|
|
@ -1,5 +1,3 @@
|
||||||
import ../utils
|
|
||||||
|
|
||||||
template cookiePrefs*(): untyped {.dirty.} =
|
template cookiePrefs*(): untyped {.dirty.} =
|
||||||
getPrefs(request.cookies.getOrDefault("preferences"))
|
getPrefs(request.cookies.getOrDefault("preferences"))
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -80,20 +80,17 @@ proc createTimelineRouter*(cfg: Config) =
|
||||||
|
|
||||||
get "/@name/search":
|
get "/@name/search":
|
||||||
cond '.' notin @"name"
|
cond '.' notin @"name"
|
||||||
let prefs = cookiePrefs()
|
|
||||||
let query = initQuery(@"filter", @"include", @"not", @"sep", @"name")
|
let query = initQuery(@"filter", @"include", @"not", @"sep", @"name")
|
||||||
respTimeline(await showTimeline(@"name", @"after", some(query),
|
respTimeline(await showTimeline(@"name", @"after", some(query),
|
||||||
cookiePrefs(), getPath(), cfg.title))
|
cookiePrefs(), getPath(), cfg.title))
|
||||||
|
|
||||||
get "/@name/replies":
|
get "/@name/replies":
|
||||||
cond '.' notin @"name"
|
cond '.' notin @"name"
|
||||||
let prefs = cookiePrefs()
|
|
||||||
respTimeline(await showTimeline(@"name", @"after", some(getReplyQuery(@"name")),
|
respTimeline(await showTimeline(@"name", @"after", some(getReplyQuery(@"name")),
|
||||||
cookiePrefs(), getPath(), cfg.title))
|
cookiePrefs(), getPath(), cfg.title))
|
||||||
|
|
||||||
get "/@name/media":
|
get "/@name/media":
|
||||||
cond '.' notin @"name"
|
cond '.' notin @"name"
|
||||||
let prefs = cookiePrefs()
|
|
||||||
respTimeline(await showTimeline(@"name", @"after", some(getMediaQuery(@"name")),
|
respTimeline(await showTimeline(@"name", @"after", some(getMediaQuery(@"name")),
|
||||||
cookiePrefs(), getPath(), cfg.title))
|
cookiePrefs(), getPath(), cfg.title))
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,5 @@
|
||||||
import times, sequtils, options
|
import times, sequtils, options
|
||||||
import norm/sqlite
|
import norm/sqlite
|
||||||
import prefs_impl
|
|
||||||
|
|
||||||
export sqlite, options
|
export sqlite, options
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
import strutils
|
import strutils
|
||||||
import karax/[karaxdsl, vdom, vstyles]
|
import karax/[karaxdsl, vdom]
|
||||||
|
|
||||||
import ../types, ../utils
|
import ../types, ../utils
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,7 @@
|
||||||
import strutils, strformat
|
|
||||||
import karax/[karaxdsl, vdom]
|
import karax/[karaxdsl, vdom]
|
||||||
|
|
||||||
import ../types
|
import ../types
|
||||||
import tweet, renderutils
|
import tweet
|
||||||
|
|
||||||
proc renderMoreReplies(thread: Thread): VNode =
|
proc renderMoreReplies(thread: Thread): VNode =
|
||||||
let num = if thread.more != -1: $thread.more & " " else: ""
|
let num = if thread.more != -1: $thread.more & " " else: ""
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue