update parser

This commit is contained in:
cmj 2024-04-06 10:03:35 -07:00
parent 26ee35b7fc
commit f14409f5f5
5 changed files with 49 additions and 17 deletions

View file

@ -71,10 +71,20 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
proc getFavorites*(id: string; cfg: Config; after=""): Future[Profile] {.async.} = proc getFavorites*(id: string; cfg: Config; after=""): Future[Profile] {.async.} =
if id.len == 0: return if id.len == 0: return
let var
ps = genParams({"userId": id}, after) variables = %*{
url = consts.favorites / (id & ".json") ? ps "userId": id,
result = parseTimeline(await fetch(url, Api.favorites), after) "includePromotedContent":false,
"withClientEventToken":false,
"withBirdwatchNotes":false,
"withVoice":true,
"withV2Timeline":false
}
if after.len > 0:
variables["cursor"] = % after
let
url = consts.favorites ? {"variables": $variables, "features": gqlFeatures}
result = parseGraphTimeline(await fetch(url, Api.favorites), after)
proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} = proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
if id.len == 0: return if id.len == 0: return

View file

@ -49,7 +49,7 @@ proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
proc genHeaders*(url, oauthToken, oauthTokenSecret: string): HttpHeaders = proc genHeaders*(url, oauthToken, oauthTokenSecret: string): HttpHeaders =
let header = getOauthHeader(url, oauthToken, oauthTokenSecret) let header = getOauthHeader(url, oauthToken, oauthTokenSecret)
result = newHttpHeaders({ result = newHttpHeaders({
"connection": "keep-alive", "connection": "keep-alive",
"authorization": header, "authorization": header,
@ -149,18 +149,27 @@ template retry(bod) =
proc fetch*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[JsonNode] {.async.} = proc fetch*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[JsonNode] {.async.} =
if len(cfg.cookieHeader) != 0:
additional_headers.add("Cookie", cfg.cookieHeader)
if len(cfg.xCsrfToken) != 0:
additional_headers.add("x-csrf-token", cfg.xCsrfToken)
retry: retry:
var body: string var body: string
fetchImpl(body, additional_headers): fetchImpl(body, additional_headers):
if body.startsWith('{') or body.startsWith('['): if body.startsWith('{') or body.startsWith('['):
result = parseJson(body) result = parseJson(body)
echo "-------------"
echo "api: ", api
echo "-------------"
echo "url: ", url
echo "-------------"
echo "added headers: ", additional_headers
echo "-------------"
else: else:
echo resp.status, ": ", body, " --- url: ", url echo resp.status, ": ", body, " --- url: ", url
echo "*************"
echo url
echo "*************"
echo api
echo "*************"
echo additional_headers
echo "*************"
result = newJNull() result = newJNull()
let error = result.getError let error = result.getError

View file

@ -41,9 +41,7 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
enableRss: cfg.get("Config", "enableRSS", true), enableRss: cfg.get("Config", "enableRSS", true),
enableDebug: cfg.get("Config", "enableDebug", false), enableDebug: cfg.get("Config", "enableDebug", false),
proxy: cfg.get("Config", "proxy", ""), proxy: cfg.get("Config", "proxy", ""),
proxyAuth: cfg.get("Config", "proxyAuth", ""), proxyAuth: cfg.get("Config", "proxyAuth", "")
cookieHeader: cfg.get("Config", "cookieHeader", ""),
xCsrfToken: cfg.get("Config", "xCsrfToken", "")
) )
return (conf, cfg) return (conf, cfg)

View file

@ -33,6 +33,7 @@ proc parseGraphUser(js: JsonNode): User =
var user = js{"user_result", "result"} var user = js{"user_result", "result"}
if user.isNull: if user.isNull:
user = ? js{"user_results", "result"} user = ? js{"user_results", "result"}
result = parseUser(user{"legacy"}) result = parseUser(user{"legacy"})
if result.verifiedType == VerifiedType.none and user{"is_blue_verified"}.getBool(false): if result.verifiedType == VerifiedType.none and user{"is_blue_verified"}.getBool(false):
@ -534,7 +535,8 @@ proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile =
let instructions = let instructions =
if root == "list": ? js{"data", "list", "timeline_response", "timeline", "instructions"} if root == "list": ? js{"data", "list", "timeline_response", "timeline", "instructions"}
else: ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"} elif root == "user": ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
else: ? js{"data", "user", "result", "timeline", "timeline", "instructions"}
if instructions.len == 0: if instructions.len == 0:
return return
@ -554,6 +556,21 @@ proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile =
result.tweets.content.add thread.content result.tweets.content.add thread.content
elif entryId.startsWith("cursor-bottom"): elif entryId.startsWith("cursor-bottom"):
result.tweets.bottom = e{"content", "value"}.getStr result.tweets.bottom = e{"content", "value"}.getStr
# TODO cleanup
if i{"type"}.getStr == "TimelineAddEntries":
for e in i{"entries"}:
let entryId = e{"entryId"}.getStr
if entryId.startsWith("tweet"):
with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
let tweet = parseGraphTweet(tweetResult, false)
if not tweet.available:
tweet.id = parseBiggestInt(entryId.getId())
result.tweets.content.add tweet
elif "-conversation-" in entryId or entryId.startsWith("homeConversation"):
let (thread, self) = parseGraphThread(e)
result.tweets.content.add thread.content
elif entryId.startsWith("cursor-bottom"):
result.tweets.bottom = e{"content", "value"}.getStr
if after.len == 0 and i{"__typename"}.getStr == "TimelinePinEntry": if after.len == 0 and i{"__typename"}.getStr == "TimelinePinEntry":
with tweetResult, i{"entry", "content", "content", "tweetResult", "result"}: with tweetResult, i{"entry", "content", "content", "tweetResult", "result"}:
let tweet = parseGraphTweet(tweetResult, false) let tweet = parseGraphTweet(tweetResult, false)

View file

@ -282,9 +282,7 @@ type
redisConns*: int redisConns*: int
redisMaxConns*: int redisMaxConns*: int
redisPassword*: string redisPassword*: string
redisDb*: int
cookieHeader*: string
xCsrfToken*: string
Rss* = object Rss* = object
feed*, cursor*: string feed*, cursor*: string