Cache profiles

This commit is contained in:
Zed 2019-06-20 20:04:18 +02:00
parent 63d7528b8f
commit 6103db6893
8 changed files with 78 additions and 100 deletions

1
.gitignore vendored
View file

@ -1,2 +1,3 @@
nitter nitter
*.html *.html
*.db

View file

@ -1,74 +1,30 @@
import sharedtables, times, hashes import asyncdispatch, times
import types, api import types, api
# var withDb:
# profileCache: SharedTable[int, Profile] try:
# profileCacheTime = initDuration(seconds=10) createTables()
except DbError:
discard
# profileCache.init() var profileCacheTime = initDuration(seconds=60)
proc getCachedProfile*(username: string; force=false): Profile = proc outdated(profile: Profile): bool =
return getProfile(username) getTime() - profile.updated > profileCacheTime
# let index = username.hash
# try:
# result = profileCache.mget(index)
# # if force or getTime() - result.lastUpdated > profileCacheTime:
# # result = getProfile(username)
# # profileCache[username.hash] = deepCopy(result)
# # return
# except KeyError:
# # result = getProfile(username)
# # profileCache.add(username.hash, deepCopy(result))
# var profile: Profile
# profileCache.withKey(index) do (k: int, v: var Profile, pairExists: var bool):
# v = getProfile(username)
# profile = v
# echo v
# pairExists = true
# echo profile.username
# return profile
# profileCache.withValue(hash(username), value) do:
# if getTime() - value.lastUpdated > profileCacheTime or force:
# result = getProfile(username)
# value = result
# else:
# result = value
# do:
# result = getProfile(username)
# value = result
# var profile: Profile
# profileCache.withKey(username.hash) do (k: int, v: var Profile, pairExists: var bool):
# if pairExists and getTime() - v.lastUpdated < profileCacheTime and not force:
# profile = deepCopy(v)
# echo "cached"
# else:
# profile = getProfile(username)
# v = deepCopy(profile)
# pairExists = true
# echo "fetched"
# return profile
# try:
# result = profileCache.mget(username.hash)
# if force or getTime() - result.lastUpdated > profileCacheTime:
# result = getProfile(username)
# profileCache[username.hash] = deepCopy(result)
# return
# except KeyError:
# result = getProfile(username)
# profileCache.add(username.hash, deepCopy(result))
# if not result.isNil or force or
# getTime() - result.lastUpdated > profileCacheTime:
# result = getProfile(username)
# profileCache[username] = result
# return
proc getCachedProfile*(username: string; force=false): Future[Profile] {.async.} =
withDb:
try:
result.getOne("username = ?", username)
doAssert(not result.outdated())
except:
if result.id == 0:
result = await getProfile(username)
result.insert()
elif result.outdated():
let
profile = await getProfile(username)
oldId = result.id
result = profile
result.id = oldId
result.update()

View file

@ -64,10 +64,10 @@ proc getUserpic*(profile: Profile; style=""): string =
getUserPic(profile.userpic, style) getUserPic(profile.userpic, style)
proc getGifSrc*(tweet: Tweet): string = proc getGifSrc*(tweet: Tweet): string =
fmt"https://video.twimg.com/tweet_video/{tweet.gif}.mp4" fmt"https://video.twimg.com/tweet_video/{tweet.gif.get()}.mp4"
proc getGifThumb*(tweet: Tweet): string = proc getGifThumb*(tweet: Tweet): string =
fmt"https://pbs.twimg.com/tweet_video_thumb/{tweet.gif}.jpg" fmt"https://pbs.twimg.com/tweet_video_thumb/{tweet.gif.get()}.jpg"
proc formatName*(profile: Profile): string = proc formatName*(profile: Profile): string =
result = xmltree.escape(profile.fullname) result = xmltree.escape(profile.fullname)

View file

@ -1,13 +1,13 @@
import asyncdispatch, httpclient, times, strutils, hashes, random, uri import asyncdispatch, httpclient, times, strutils, hashes, random, uri
import jester, regex import jester, regex
import api, utils, types import api, utils, types, cache
import views/[user, general, conversation] import views/[user, general, conversation]
proc showTimeline(name: string; num=""): Future[string] {.async.} = proc showTimeline(name: string; num=""): Future[string] {.async.} =
let let
username = name.strip(chars={'/'}) username = name.strip(chars={'/'})
profileFut = getProfile(username) profileFut = getCachedProfile(username)
tweetsFut = getTimeline(username, after=num) tweetsFut = getTimeline(username, after=num)
let profile = await profileFut let profile = await profileFut

View file

@ -47,7 +47,6 @@ proc parseTweet*(tweet: XmlNode): Tweet =
result.id = tweet.getAttr("data-item-id") result.id = tweet.getAttr("data-item-id")
result.link = tweet.getAttr("data-permalink-path") result.link = tweet.getAttr("data-permalink-path")
result.text = tweet.selectText(".tweet-text").stripTwitterUrls() result.text = tweet.selectText(".tweet-text").stripTwitterUrls()
result.retweetBy = tweet.selectText(".js-retweet-text > a > b")
result.pinned = "pinned" in tweet.getAttr("class") result.pinned = "pinned" in tweet.getAttr("class")
result.profile = parseTweetProfile(tweet) result.profile = parseTweetProfile(tweet)
@ -70,6 +69,10 @@ proc parseTweet*(tweet: XmlNode): Tweet =
of "retweets": result.retweets = num of "retweets": result.retweets = num
else: discard else: discard
let by = tweet.selectText(".js-retweet-text > a > b")
if by.len > 0:
result.retweetBy = some(by)
for photo in tweet.querySelectorAll(".AdaptiveMedia-photoContainer"): for photo in tweet.querySelectorAll(".AdaptiveMedia-photoContainer"):
result.photos.add photo.attrs["data-image-url"] result.photos.add photo.attrs["data-image-url"]
@ -77,9 +80,9 @@ proc parseTweet*(tweet: XmlNode): Tweet =
if player.len > 0: if player.len > 0:
let thumb = player.replace(re".+:url\('([^']+)'\)", "$1") let thumb = player.replace(re".+:url\('([^']+)'\)", "$1")
if "tweet_video" in thumb: if "tweet_video" in thumb:
result.gif = thumb.replace(re".+thumb/([^\.']+)\.jpg.+", "$1") result.gif = some(thumb.replace(re".+thumb/([^\.']+)\.jpg.+", "$1"))
else: else:
result.videoThumb = thumb result.videoThumb = some(thumb)
proc parseTweets*(node: XmlNode): Tweets = proc parseTweets*(node: XmlNode): Tweets =
if node.isNil: return if node.isNil: return

View file

@ -1,5 +1,9 @@
import times, sequtils import times, sequtils, strutils, options
import norm/sqlite
export sqlite, options
db("cache.db", "", "", ""):
type type
Profile* = object Profile* = object
username*: string username*: string
@ -10,9 +14,23 @@ type
following*: string following*: string
followers*: string followers*: string
tweets*: string tweets*: string
verified*: bool verified* {.
protected*: bool dbType: "STRING",
parseIt: parseBool(it.s)
formatIt: $it
.}: bool
protected* {.
dbType: "STRING",
parseIt: parseBool(it.s)
formatIt: $it
.}: bool
updated* {.
dbType: "INTEGER",
parseIt: it.i.fromUnix(),
formatIt: getTime().toUnix()
.}: Time
type
Tweet* = object Tweet* = object
id*: string id*: string
profile*: Profile profile*: Profile
@ -23,12 +41,12 @@ type
replies*: string replies*: string
retweets*: string retweets*: string
likes*: string likes*: string
retweetBy*: string
pinned*: bool pinned*: bool
photos*: seq[string] photos*: seq[string]
gif*: string retweetBy*: Option[string]
video*: string gif*: Option[string]
videoThumb*: string video*: Option[string]
videoThumb*: Option[string]
Tweets* = seq[Tweet] Tweets* = seq[Tweet]

View file

@ -3,9 +3,9 @@
#import ../types, ../formatters, ../utils #import ../types, ../formatters, ../utils
# #
#proc renderHeading(tweet: Tweet): string = #proc renderHeading(tweet: Tweet): string =
#if tweet.retweetBy != "": #if tweet.retweetBy.isSome:
<div class="retweet"> <div class="retweet">
<span>🔄 ${tweet.retweetBy} retweeted</span> <span>🔄 ${tweet.retweetBy.get()} retweeted</span>
</div> </div>
#end if #end if
#if tweet.pinned: #if tweet.pinned:
@ -59,7 +59,7 @@
<div class="attachments media-body"> <div class="attachments media-body">
<div class="gallery-row" style="max-height: unset;"> <div class="gallery-row" style="max-height: unset;">
<div class="attachment image"> <div class="attachment image">
<video poster=${tweet.videoThumb} style="width: 100%; height: 100%;" autoplay muted loop></video> <video poster=${tweet.videoThumb.get()} style="width: 100%; height: 100%;" autoplay muted loop></video>
<div class="video-overlay"> <div class="video-overlay">
<p>Video playback not supported</p> <p>Video playback not supported</p>
</div> </div>
@ -102,9 +102,9 @@
</div> </div>
#if tweet.photos.len > 0: #if tweet.photos.len > 0:
${renderMediaGroup(tweet)} ${renderMediaGroup(tweet)}
#elif tweet.videoThumb.len > 0: #elif tweet.videoThumb.isSome:
${renderVideo(tweet)} ${renderVideo(tweet)}
#elif tweet.gif.len > 0: #elif tweet.gif.isSome:
${renderGif(tweet)} ${renderGif(tweet)}
#end if #end if
${renderStats(tweet)} ${renderStats(tweet)}

View file

@ -73,7 +73,7 @@
#for tweet in tweets: #for tweet in tweets:
#if tweet in retweets: continue #if tweet in retweets: continue
#end if #end if
#if tweet.retweetBy.len > 0: retweets.add tweet #if tweet.retweetBy.isSome: retweets.add tweet
#end if #end if
${renderTweet(tweet, "timeline-tweet")} ${renderTweet(tweet, "timeline-tweet")}
#end for #end for