diff --git a/.gitignore b/.gitignore index 3c7021e..0af7e4c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ nitter -*.html \ No newline at end of file +*.html +*.db \ No newline at end of file diff --git a/src/cache.nim b/src/cache.nim index 8ac3ada..b9d6fed 100644 --- a/src/cache.nim +++ b/src/cache.nim @@ -1,74 +1,30 @@ -import sharedtables, times, hashes +import asyncdispatch, times import types, api -# var -# profileCache: SharedTable[int, Profile] -# profileCacheTime = initDuration(seconds=10) +withDb: + try: + createTables() + except DbError: + discard -# profileCache.init() +var profileCacheTime = initDuration(seconds=60) -proc getCachedProfile*(username: string; force=false): Profile = - return getProfile(username) - # let index = username.hash - - # try: - # result = profileCache.mget(index) - # # if force or getTime() - result.lastUpdated > profileCacheTime: - # # result = getProfile(username) - # # profileCache[username.hash] = deepCopy(result) - # # return - # except KeyError: - # # result = getProfile(username) - # # profileCache.add(username.hash, deepCopy(result)) - - - - # var profile: Profile - # profileCache.withKey(index) do (k: int, v: var Profile, pairExists: var bool): - # v = getProfile(username) - # profile = v - # echo v - # pairExists = true - # echo profile.username - # return profile - - # profileCache.withValue(hash(username), value) do: - # if getTime() - value.lastUpdated > profileCacheTime or force: - # result = getProfile(username) - # value = result - # else: - # result = value - # do: - # result = getProfile(username) - # value = result - - # var profile: Profile - - # profileCache.withKey(username.hash) do (k: int, v: var Profile, pairExists: var bool): - # if pairExists and getTime() - v.lastUpdated < profileCacheTime and not force: - # profile = deepCopy(v) - # echo "cached" - # else: - # profile = getProfile(username) - # v = deepCopy(profile) - # pairExists = true - # echo "fetched" - - # return profile - - # try: - # result = profileCache.mget(username.hash) - # if force or getTime() - result.lastUpdated > profileCacheTime: - # result = getProfile(username) - # profileCache[username.hash] = deepCopy(result) - # return - # except KeyError: - # result = getProfile(username) - # profileCache.add(username.hash, deepCopy(result)) - - # if not result.isNil or force or - # getTime() - result.lastUpdated > profileCacheTime: - # result = getProfile(username) - # profileCache[username] = result - # return +proc outdated(profile: Profile): bool = + getTime() - profile.updated > profileCacheTime +proc getCachedProfile*(username: string; force=false): Future[Profile] {.async.} = + withDb: + try: + result.getOne("username = ?", username) + doAssert(not result.outdated()) + except: + if result.id == 0: + result = await getProfile(username) + result.insert() + elif result.outdated(): + let + profile = await getProfile(username) + oldId = result.id + result = profile + result.id = oldId + result.update() diff --git a/src/formatters.nim b/src/formatters.nim index b07668b..11373e8 100644 --- a/src/formatters.nim +++ b/src/formatters.nim @@ -64,10 +64,10 @@ proc getUserpic*(profile: Profile; style=""): string = getUserPic(profile.userpic, style) proc getGifSrc*(tweet: Tweet): string = - fmt"https://video.twimg.com/tweet_video/{tweet.gif}.mp4" + fmt"https://video.twimg.com/tweet_video/{tweet.gif.get()}.mp4" proc getGifThumb*(tweet: Tweet): string = - fmt"https://pbs.twimg.com/tweet_video_thumb/{tweet.gif}.jpg" + fmt"https://pbs.twimg.com/tweet_video_thumb/{tweet.gif.get()}.jpg" proc formatName*(profile: Profile): string = result = xmltree.escape(profile.fullname) diff --git a/src/nitter.nim b/src/nitter.nim index 4fef4e0..259400e 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -1,13 +1,13 @@ import asyncdispatch, httpclient, times, strutils, hashes, random, uri import jester, regex -import api, utils, types +import api, utils, types, cache import views/[user, general, conversation] proc showTimeline(name: string; num=""): Future[string] {.async.} = let username = name.strip(chars={'/'}) - profileFut = getProfile(username) + profileFut = getCachedProfile(username) tweetsFut = getTimeline(username, after=num) let profile = await profileFut diff --git a/src/parser.nim b/src/parser.nim index 76715f2..9decc14 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -47,7 +47,6 @@ proc parseTweet*(tweet: XmlNode): Tweet = result.id = tweet.getAttr("data-item-id") result.link = tweet.getAttr("data-permalink-path") result.text = tweet.selectText(".tweet-text").stripTwitterUrls() - result.retweetBy = tweet.selectText(".js-retweet-text > a > b") result.pinned = "pinned" in tweet.getAttr("class") result.profile = parseTweetProfile(tweet) @@ -70,6 +69,10 @@ proc parseTweet*(tweet: XmlNode): Tweet = of "retweets": result.retweets = num else: discard + let by = tweet.selectText(".js-retweet-text > a > b") + if by.len > 0: + result.retweetBy = some(by) + for photo in tweet.querySelectorAll(".AdaptiveMedia-photoContainer"): result.photos.add photo.attrs["data-image-url"] @@ -77,9 +80,9 @@ proc parseTweet*(tweet: XmlNode): Tweet = if player.len > 0: let thumb = player.replace(re".+:url\('([^']+)'\)", "$1") if "tweet_video" in thumb: - result.gif = thumb.replace(re".+thumb/([^\.']+)\.jpg.+", "$1") + result.gif = some(thumb.replace(re".+thumb/([^\.']+)\.jpg.+", "$1")) else: - result.videoThumb = thumb + result.videoThumb = some(thumb) proc parseTweets*(node: XmlNode): Tweets = if node.isNil: return diff --git a/src/types.nim b/src/types.nim index 9da08ac..a2807e8 100644 --- a/src/types.nim +++ b/src/types.nim @@ -1,18 +1,36 @@ -import times, sequtils +import times, sequtils, strutils, options +import norm/sqlite + +export sqlite, options + +db("cache.db", "", "", ""): + type + Profile* = object + username*: string + fullname*: string + description*: string + userpic*: string + banner*: string + following*: string + followers*: string + tweets*: string + verified* {. + dbType: "STRING", + parseIt: parseBool(it.s) + formatIt: $it + .}: bool + protected* {. + dbType: "STRING", + parseIt: parseBool(it.s) + formatIt: $it + .}: bool + updated* {. + dbType: "INTEGER", + parseIt: it.i.fromUnix(), + formatIt: getTime().toUnix() + .}: Time type - Profile* = object - username*: string - fullname*: string - description*: string - userpic*: string - banner*: string - following*: string - followers*: string - tweets*: string - verified*: bool - protected*: bool - Tweet* = object id*: string profile*: Profile @@ -23,12 +41,12 @@ type replies*: string retweets*: string likes*: string - retweetBy*: string pinned*: bool photos*: seq[string] - gif*: string - video*: string - videoThumb*: string + retweetBy*: Option[string] + gif*: Option[string] + video*: Option[string] + videoThumb*: Option[string] Tweets* = seq[Tweet] diff --git a/src/views/tweet.nim b/src/views/tweet.nim index d26bc1d..6bd8879 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -3,9 +3,9 @@ #import ../types, ../formatters, ../utils # #proc renderHeading(tweet: Tweet): string = -#if tweet.retweetBy != "": +#if tweet.retweetBy.isSome:
- 🔄 ${tweet.retweetBy} retweeted + 🔄 ${tweet.retweetBy.get()} retweeted
#end if #if tweet.pinned: @@ -59,7 +59,7 @@
- +

Video playback not supported

@@ -102,9 +102,9 @@
#if tweet.photos.len > 0: ${renderMediaGroup(tweet)} - #elif tweet.videoThumb.len > 0: + #elif tweet.videoThumb.isSome: ${renderVideo(tweet)} - #elif tweet.gif.len > 0: + #elif tweet.gif.isSome: ${renderGif(tweet)} #end if ${renderStats(tweet)} diff --git a/src/views/user.nim b/src/views/user.nim index 39ede0c..1330cb2 100644 --- a/src/views/user.nim +++ b/src/views/user.nim @@ -73,7 +73,7 @@ #for tweet in tweets: #if tweet in retweets: continue #end if - #if tweet.retweetBy.len > 0: retweets.add tweet + #if tweet.retweetBy.isSome: retweets.add tweet #end if ${renderTweet(tweet, "timeline-tweet")} #end for