Cache profiles
This commit is contained in:
parent
63d7528b8f
commit
6103db6893
8 changed files with 78 additions and 100 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -1,2 +1,3 @@
|
||||||
nitter
|
nitter
|
||||||
*.html
|
*.html
|
||||||
|
*.db
|
|
@ -1,74 +1,30 @@
|
||||||
import sharedtables, times, hashes
|
import asyncdispatch, times
|
||||||
import types, api
|
import types, api
|
||||||
|
|
||||||
# var
|
withDb:
|
||||||
# profileCache: SharedTable[int, Profile]
|
try:
|
||||||
# profileCacheTime = initDuration(seconds=10)
|
createTables()
|
||||||
|
except DbError:
|
||||||
|
discard
|
||||||
|
|
||||||
# profileCache.init()
|
var profileCacheTime = initDuration(seconds=60)
|
||||||
|
|
||||||
proc getCachedProfile*(username: string; force=false): Profile =
|
proc outdated(profile: Profile): bool =
|
||||||
return getProfile(username)
|
getTime() - profile.updated > profileCacheTime
|
||||||
# let index = username.hash
|
|
||||||
|
|
||||||
# try:
|
|
||||||
# result = profileCache.mget(index)
|
|
||||||
# # if force or getTime() - result.lastUpdated > profileCacheTime:
|
|
||||||
# # result = getProfile(username)
|
|
||||||
# # profileCache[username.hash] = deepCopy(result)
|
|
||||||
# # return
|
|
||||||
# except KeyError:
|
|
||||||
# # result = getProfile(username)
|
|
||||||
# # profileCache.add(username.hash, deepCopy(result))
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# var profile: Profile
|
|
||||||
# profileCache.withKey(index) do (k: int, v: var Profile, pairExists: var bool):
|
|
||||||
# v = getProfile(username)
|
|
||||||
# profile = v
|
|
||||||
# echo v
|
|
||||||
# pairExists = true
|
|
||||||
# echo profile.username
|
|
||||||
# return profile
|
|
||||||
|
|
||||||
# profileCache.withValue(hash(username), value) do:
|
|
||||||
# if getTime() - value.lastUpdated > profileCacheTime or force:
|
|
||||||
# result = getProfile(username)
|
|
||||||
# value = result
|
|
||||||
# else:
|
|
||||||
# result = value
|
|
||||||
# do:
|
|
||||||
# result = getProfile(username)
|
|
||||||
# value = result
|
|
||||||
|
|
||||||
# var profile: Profile
|
|
||||||
|
|
||||||
# profileCache.withKey(username.hash) do (k: int, v: var Profile, pairExists: var bool):
|
|
||||||
# if pairExists and getTime() - v.lastUpdated < profileCacheTime and not force:
|
|
||||||
# profile = deepCopy(v)
|
|
||||||
# echo "cached"
|
|
||||||
# else:
|
|
||||||
# profile = getProfile(username)
|
|
||||||
# v = deepCopy(profile)
|
|
||||||
# pairExists = true
|
|
||||||
# echo "fetched"
|
|
||||||
|
|
||||||
# return profile
|
|
||||||
|
|
||||||
# try:
|
|
||||||
# result = profileCache.mget(username.hash)
|
|
||||||
# if force or getTime() - result.lastUpdated > profileCacheTime:
|
|
||||||
# result = getProfile(username)
|
|
||||||
# profileCache[username.hash] = deepCopy(result)
|
|
||||||
# return
|
|
||||||
# except KeyError:
|
|
||||||
# result = getProfile(username)
|
|
||||||
# profileCache.add(username.hash, deepCopy(result))
|
|
||||||
|
|
||||||
# if not result.isNil or force or
|
|
||||||
# getTime() - result.lastUpdated > profileCacheTime:
|
|
||||||
# result = getProfile(username)
|
|
||||||
# profileCache[username] = result
|
|
||||||
# return
|
|
||||||
|
|
||||||
|
proc getCachedProfile*(username: string; force=false): Future[Profile] {.async.} =
|
||||||
|
withDb:
|
||||||
|
try:
|
||||||
|
result.getOne("username = ?", username)
|
||||||
|
doAssert(not result.outdated())
|
||||||
|
except:
|
||||||
|
if result.id == 0:
|
||||||
|
result = await getProfile(username)
|
||||||
|
result.insert()
|
||||||
|
elif result.outdated():
|
||||||
|
let
|
||||||
|
profile = await getProfile(username)
|
||||||
|
oldId = result.id
|
||||||
|
result = profile
|
||||||
|
result.id = oldId
|
||||||
|
result.update()
|
||||||
|
|
|
@ -64,10 +64,10 @@ proc getUserpic*(profile: Profile; style=""): string =
|
||||||
getUserPic(profile.userpic, style)
|
getUserPic(profile.userpic, style)
|
||||||
|
|
||||||
proc getGifSrc*(tweet: Tweet): string =
|
proc getGifSrc*(tweet: Tweet): string =
|
||||||
fmt"https://video.twimg.com/tweet_video/{tweet.gif}.mp4"
|
fmt"https://video.twimg.com/tweet_video/{tweet.gif.get()}.mp4"
|
||||||
|
|
||||||
proc getGifThumb*(tweet: Tweet): string =
|
proc getGifThumb*(tweet: Tweet): string =
|
||||||
fmt"https://pbs.twimg.com/tweet_video_thumb/{tweet.gif}.jpg"
|
fmt"https://pbs.twimg.com/tweet_video_thumb/{tweet.gif.get()}.jpg"
|
||||||
|
|
||||||
proc formatName*(profile: Profile): string =
|
proc formatName*(profile: Profile): string =
|
||||||
result = xmltree.escape(profile.fullname)
|
result = xmltree.escape(profile.fullname)
|
||||||
|
|
|
@ -1,13 +1,13 @@
|
||||||
import asyncdispatch, httpclient, times, strutils, hashes, random, uri
|
import asyncdispatch, httpclient, times, strutils, hashes, random, uri
|
||||||
import jester, regex
|
import jester, regex
|
||||||
|
|
||||||
import api, utils, types
|
import api, utils, types, cache
|
||||||
import views/[user, general, conversation]
|
import views/[user, general, conversation]
|
||||||
|
|
||||||
proc showTimeline(name: string; num=""): Future[string] {.async.} =
|
proc showTimeline(name: string; num=""): Future[string] {.async.} =
|
||||||
let
|
let
|
||||||
username = name.strip(chars={'/'})
|
username = name.strip(chars={'/'})
|
||||||
profileFut = getProfile(username)
|
profileFut = getCachedProfile(username)
|
||||||
tweetsFut = getTimeline(username, after=num)
|
tweetsFut = getTimeline(username, after=num)
|
||||||
|
|
||||||
let profile = await profileFut
|
let profile = await profileFut
|
||||||
|
|
|
@ -47,7 +47,6 @@ proc parseTweet*(tweet: XmlNode): Tweet =
|
||||||
result.id = tweet.getAttr("data-item-id")
|
result.id = tweet.getAttr("data-item-id")
|
||||||
result.link = tweet.getAttr("data-permalink-path")
|
result.link = tweet.getAttr("data-permalink-path")
|
||||||
result.text = tweet.selectText(".tweet-text").stripTwitterUrls()
|
result.text = tweet.selectText(".tweet-text").stripTwitterUrls()
|
||||||
result.retweetBy = tweet.selectText(".js-retweet-text > a > b")
|
|
||||||
result.pinned = "pinned" in tweet.getAttr("class")
|
result.pinned = "pinned" in tweet.getAttr("class")
|
||||||
result.profile = parseTweetProfile(tweet)
|
result.profile = parseTweetProfile(tweet)
|
||||||
|
|
||||||
|
@ -70,6 +69,10 @@ proc parseTweet*(tweet: XmlNode): Tweet =
|
||||||
of "retweets": result.retweets = num
|
of "retweets": result.retweets = num
|
||||||
else: discard
|
else: discard
|
||||||
|
|
||||||
|
let by = tweet.selectText(".js-retweet-text > a > b")
|
||||||
|
if by.len > 0:
|
||||||
|
result.retweetBy = some(by)
|
||||||
|
|
||||||
for photo in tweet.querySelectorAll(".AdaptiveMedia-photoContainer"):
|
for photo in tweet.querySelectorAll(".AdaptiveMedia-photoContainer"):
|
||||||
result.photos.add photo.attrs["data-image-url"]
|
result.photos.add photo.attrs["data-image-url"]
|
||||||
|
|
||||||
|
@ -77,9 +80,9 @@ proc parseTweet*(tweet: XmlNode): Tweet =
|
||||||
if player.len > 0:
|
if player.len > 0:
|
||||||
let thumb = player.replace(re".+:url\('([^']+)'\)", "$1")
|
let thumb = player.replace(re".+:url\('([^']+)'\)", "$1")
|
||||||
if "tweet_video" in thumb:
|
if "tweet_video" in thumb:
|
||||||
result.gif = thumb.replace(re".+thumb/([^\.']+)\.jpg.+", "$1")
|
result.gif = some(thumb.replace(re".+thumb/([^\.']+)\.jpg.+", "$1"))
|
||||||
else:
|
else:
|
||||||
result.videoThumb = thumb
|
result.videoThumb = some(thumb)
|
||||||
|
|
||||||
proc parseTweets*(node: XmlNode): Tweets =
|
proc parseTweets*(node: XmlNode): Tweets =
|
||||||
if node.isNil: return
|
if node.isNil: return
|
||||||
|
|
|
@ -1,5 +1,9 @@
|
||||||
import times, sequtils
|
import times, sequtils, strutils, options
|
||||||
|
import norm/sqlite
|
||||||
|
|
||||||
|
export sqlite, options
|
||||||
|
|
||||||
|
db("cache.db", "", "", ""):
|
||||||
type
|
type
|
||||||
Profile* = object
|
Profile* = object
|
||||||
username*: string
|
username*: string
|
||||||
|
@ -10,9 +14,23 @@ type
|
||||||
following*: string
|
following*: string
|
||||||
followers*: string
|
followers*: string
|
||||||
tweets*: string
|
tweets*: string
|
||||||
verified*: bool
|
verified* {.
|
||||||
protected*: bool
|
dbType: "STRING",
|
||||||
|
parseIt: parseBool(it.s)
|
||||||
|
formatIt: $it
|
||||||
|
.}: bool
|
||||||
|
protected* {.
|
||||||
|
dbType: "STRING",
|
||||||
|
parseIt: parseBool(it.s)
|
||||||
|
formatIt: $it
|
||||||
|
.}: bool
|
||||||
|
updated* {.
|
||||||
|
dbType: "INTEGER",
|
||||||
|
parseIt: it.i.fromUnix(),
|
||||||
|
formatIt: getTime().toUnix()
|
||||||
|
.}: Time
|
||||||
|
|
||||||
|
type
|
||||||
Tweet* = object
|
Tweet* = object
|
||||||
id*: string
|
id*: string
|
||||||
profile*: Profile
|
profile*: Profile
|
||||||
|
@ -23,12 +41,12 @@ type
|
||||||
replies*: string
|
replies*: string
|
||||||
retweets*: string
|
retweets*: string
|
||||||
likes*: string
|
likes*: string
|
||||||
retweetBy*: string
|
|
||||||
pinned*: bool
|
pinned*: bool
|
||||||
photos*: seq[string]
|
photos*: seq[string]
|
||||||
gif*: string
|
retweetBy*: Option[string]
|
||||||
video*: string
|
gif*: Option[string]
|
||||||
videoThumb*: string
|
video*: Option[string]
|
||||||
|
videoThumb*: Option[string]
|
||||||
|
|
||||||
Tweets* = seq[Tweet]
|
Tweets* = seq[Tweet]
|
||||||
|
|
||||||
|
|
|
@ -3,9 +3,9 @@
|
||||||
#import ../types, ../formatters, ../utils
|
#import ../types, ../formatters, ../utils
|
||||||
#
|
#
|
||||||
#proc renderHeading(tweet: Tweet): string =
|
#proc renderHeading(tweet: Tweet): string =
|
||||||
#if tweet.retweetBy != "":
|
#if tweet.retweetBy.isSome:
|
||||||
<div class="retweet">
|
<div class="retweet">
|
||||||
<span>🔄 ${tweet.retweetBy} retweeted</span>
|
<span>🔄 ${tweet.retweetBy.get()} retweeted</span>
|
||||||
</div>
|
</div>
|
||||||
#end if
|
#end if
|
||||||
#if tweet.pinned:
|
#if tweet.pinned:
|
||||||
|
@ -59,7 +59,7 @@
|
||||||
<div class="attachments media-body">
|
<div class="attachments media-body">
|
||||||
<div class="gallery-row" style="max-height: unset;">
|
<div class="gallery-row" style="max-height: unset;">
|
||||||
<div class="attachment image">
|
<div class="attachment image">
|
||||||
<video poster=${tweet.videoThumb} style="width: 100%; height: 100%;" autoplay muted loop></video>
|
<video poster=${tweet.videoThumb.get()} style="width: 100%; height: 100%;" autoplay muted loop></video>
|
||||||
<div class="video-overlay">
|
<div class="video-overlay">
|
||||||
<p>Video playback not supported</p>
|
<p>Video playback not supported</p>
|
||||||
</div>
|
</div>
|
||||||
|
@ -102,9 +102,9 @@
|
||||||
</div>
|
</div>
|
||||||
#if tweet.photos.len > 0:
|
#if tweet.photos.len > 0:
|
||||||
${renderMediaGroup(tweet)}
|
${renderMediaGroup(tweet)}
|
||||||
#elif tweet.videoThumb.len > 0:
|
#elif tweet.videoThumb.isSome:
|
||||||
${renderVideo(tweet)}
|
${renderVideo(tweet)}
|
||||||
#elif tweet.gif.len > 0:
|
#elif tweet.gif.isSome:
|
||||||
${renderGif(tweet)}
|
${renderGif(tweet)}
|
||||||
#end if
|
#end if
|
||||||
${renderStats(tweet)}
|
${renderStats(tweet)}
|
||||||
|
|
|
@ -73,7 +73,7 @@
|
||||||
#for tweet in tweets:
|
#for tweet in tweets:
|
||||||
#if tweet in retweets: continue
|
#if tweet in retweets: continue
|
||||||
#end if
|
#end if
|
||||||
#if tweet.retweetBy.len > 0: retweets.add tweet
|
#if tweet.retweetBy.isSome: retweets.add tweet
|
||||||
#end if
|
#end if
|
||||||
${renderTweet(tweet, "timeline-tweet")}
|
${renderTweet(tweet, "timeline-tweet")}
|
||||||
#end for
|
#end for
|
||||||
|
|
Loading…
Reference in a new issue