Add experimental user parser

config.nims

@@ -7,6 +7,7 @@
 # disable annoying warnings
 warning("GcUnsafe2", off)
 hint("XDeclaredButNotUsed", off)
+hint("XCannotRaiseY", off)
 hint("User", off)
 
 const

src/api.nim

@@ -2,6 +2,7 @@
 import asyncdispatch, httpclient, uri, strutils
 import packedjson
 import types, query, formatters, consts, apiutils, parser
+import experimental/parser/user
 
 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
   let
@@ -32,14 +33,14 @@ proc getListMembers*(list: List; after=""): Future[Result[Profile]] {.async.} =
 proc getProfile*(username: string): Future[Profile] {.async.} =
   let
     ps = genParams({"screen_name": username})
-    js = await fetch(userShow ? ps, Api.userShow)
+    json = await fetchRaw(userShow ? ps, Api.userShow)
-  result = parseUserShow(js, username=username)
+  result = parseUser(json)
 
 proc getProfileById*(userId: string): Future[Profile] {.async.} =
   let
     ps = genParams({"user_id": userId})
-    js = await fetch(userShow ? ps, Api.userShow)
+    json = await fetchRaw(userShow ? ps, Api.userShow)
-  result = parseUserShow(js, id=userId)
+  result = parseUser(json)
 
 proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
   let

src/apiutils.nim

@@ -1,7 +1,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import httpclient, asyncdispatch, options, times, strutils, uri
+import httpclient, asyncdispatch, options, sequtils, strutils, uri
-import packedjson, zippy
+import jsony, packedjson, zippy
 import types, tokens, consts, parserutils, http_pool
+from experimental/types/common import Errors, ErrorObj
 
 const
   rlRemaining = "x-rate-limit-remaining"
@@ -40,7 +41,14 @@ proc genHeaders*(token: Token = nil): HttpHeaders =
     "DNT": "1"
   })
 
-proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
+template updateToken() =
+  if api != Api.search and resp.headers.hasKey(rlRemaining):
+    let
+      remaining = parseInt(resp.headers[rlRemaining])
+      reset = parseInt(resp.headers[rlReset])
+    token.setRateLimit(api, remaining, reset)
+
+template fetchImpl(result, fetchBody) {.dirty.} =
   once:
     pool = HttpPool()
 
@@ -48,37 +56,21 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
   if token.tok.len == 0:
     raise rateLimitError()
 
-  let headers = genHeaders(token)
   try:
     var resp: AsyncResponse
-    var body = pool.use(headers):
+    result = pool.use(genHeaders(token)):
       resp = await c.get($url)
       await resp.body
 
-    if body.len > 0:
+    if result.len > 0:
       if resp.headers.getOrDefault("content-encoding") == "gzip":
-        body = uncompress(body, dfGzip)
+        result = uncompress(result, dfGzip)
       else:
-        echo "non-gzip body, url: ", url, ", body: ", body
+        echo "non-gzip body, url: ", url, ", body: ", result
 
-    if body.startsWith('{') or body.startsWith('['):
+    fetchBody
-      result = parseJson(body)
-    else:
-      echo resp.status, ": ", body
-      result = newJNull()
 
-    if api != Api.search and resp.headers.hasKey(rlRemaining):
+    release(token, used=true)
-      let
-        remaining = parseInt(resp.headers[rlRemaining])
-        reset = parseInt(resp.headers[rlReset])
-      token.setRateLimit(api, remaining, reset)
-
-    if result.getError notin {invalidToken, forbidden, badToken}:
-      release(token, used=true)
-    else:
-      echo "fetch error: ", result.getError
-      release(token, invalid=true)
-      raise rateLimitError()
 
     if resp.status == $Http400:
       raise newException(InternalError, $url)
@@ -89,3 +81,35 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
     if "length" notin e.msg and "descriptor" notin e.msg:
       release(token, invalid=true)
     raise rateLimitError()
+
+proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
+  var body: string
+  fetchImpl body:
+    if body.startsWith('{') or body.startsWith('['):
+      result = parseJson(body)
+    else:
+      echo resp.status, ": ", body
+      result = newJNull()
+
+    updateToken()
+
+    let error = result.getError
+    if error in {invalidToken, forbidden, badToken}:
+      echo "fetch error: ", result.getError
+      release(token, invalid=true)
+      raise rateLimitError()
+
+proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
+  fetchImpl result:
+    if not (result.startsWith('{') or result.startsWith('[')):
+      echo resp.status, ": ", result
+      result.setLen(0)
+
+    updateToken()
+
+    if result.startsWith("{\"errors"):
+      let errors = result.fromJson(Errors).errors
+      if errors.anyIt(it.code in {invalidToken, forbidden, badToken}):
+        echo "fetch error: ", errors
+        release(token, invalid=true)
+        raise rateLimitError()

src/experimental/parser/slices.nim

@@ -0,0 +1,67 @@
+import std/[macros, htmlgen, unicode]
+import ../types/common
+import ".."/../[formatters, utils]
+
+type
+  ReplaceSliceKind = enum
+    rkRemove, rkUrl, rkHashtag, rkMention
+
+  ReplaceSlice* = object
+    slice: Slice[int]
+    kind: ReplaceSliceKind
+    url, display: string
+
+proc cmp*(x, y: ReplaceSlice): int = cmp(x.slice.a, y.slice.b)
+
+proc dedupSlices*(s: var seq[ReplaceSlice]) =
+  var
+    len = s.len
+    i = 0
+  while i < len:
+    var j = i + 1
+    while j < len:
+      if s[i].slice.a == s[j].slice.a:
+        s.del j
+        dec len
+      else:
+        inc j
+    inc i
+
+proc extractUrls*(result: var seq[ReplaceSlice]; url: Url;
+                  textLen: int; hideTwitter = false) =
+  let
+    link = url.expandedUrl
+    slice = url.indices[0] ..< url.indices[1]
+
+  if hideTwitter and slice.b.succ >= textLen and link.isTwitterUrl:
+    if slice.a < textLen:
+      result.add ReplaceSlice(kind: rkRemove, slice: slice)
+  else:
+    result.add ReplaceSlice(kind: rkUrl, url: link,
+                            display: link.shortLink, slice: slice)
+
+proc replacedWith*(runes: seq[Rune]; repls: openArray[ReplaceSlice];
+                   textSlice: Slice[int]): string =
+  template extractLowerBound(i: int; idx): int =
+    if i > 0: repls[idx].slice.b.succ else: textSlice.a
+
+  result = newStringOfCap(runes.len)
+
+  for i, rep in repls:
+    result.add $runes[extractLowerBound(i, i - 1) ..< rep.slice.a]
+    case rep.kind
+    of rkHashtag:
+      let
+        name = $runes[rep.slice.a.succ .. rep.slice.b]
+        symbol = $runes[rep.slice.a]
+      result.add a(symbol & name, href = "/search?q=%23" & name)
+    of rkMention:
+      result.add a($runes[rep.slice], href = rep.url, title = rep.display)
+    of rkUrl:
+      result.add a(rep.display, href = rep.url)
+    of rkRemove:
+      discard
+
+  let rest = extractLowerBound(repls.len, ^1) ..< textSlice.b
+  if rest.a <= rest.b:
+    result.add $runes[rest]

src/experimental/parser/user.nim

@@ -0,0 +1,68 @@
+import std/[algorithm, unicode, re, strutils]
+import jsony
+import utils, slices
+import ../types/user as userType
+from ../../types import Profile, Error
+
+let
+  unRegex = re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
+  unReplace = "$1<a href=\"/$2\">@$2</a>"
+
+  htRegex = re"(^|[^\w-_./?])([#＃$])([\w_]+)"
+  htReplace = "$1<a href=\"/search?q=%23$3\">$2$3</a>"
+
+proc expandProfileEntities(profile: var Profile; user: User) =
+  let
+    orig = profile.bio.toRunes
+    ent = user.entities
+
+  if ent.url.urls.len > 0:
+    profile.website = ent.url.urls[0].expandedUrl
+
+  var replacements = newSeq[ReplaceSlice]()
+
+  for u in ent.description.urls:
+    replacements.extractUrls(u, orig.high)
+
+  replacements.dedupSlices
+  replacements.sort(cmp)
+
+  profile.bio = orig.replacedWith(replacements, 0 .. orig.len)
+                    .replacef(unRegex, unReplace)
+                    .replacef(htRegex, htReplace)
+
+proc getBanner(user: User): string =
+  if user.profileBannerUrl.len > 0:
+    return user.profileBannerUrl & "/1500x500"
+  if user.profileLinkColor.len > 0:
+    return '#' & user.profileLinkColor
+  return "#161616"
+
+proc parseUser*(json: string): Profile =
+  handleErrors:
+    case error
+    of suspended: return Profile(suspended: true)
+    of userNotFound: return
+    else: echo "[error - parseUser]: ", error
+
+  let user = json.fromJson(User)
+
+  result = Profile(
+    id: user.idStr,
+    username: user.screenName,
+    fullname: user.name,
+    location: user.location,
+    bio: user.description,
+    following: user.friendsCount,
+    followers: user.followersCount,
+    tweets: user.statusesCount,
+    likes: user.favouritesCount,
+    media: user.mediaCount,
+    verified: user.verified,
+    protected: user.protected,
+    joinDate: parseTwitterDate(user.createdAt),
+    banner: getBanner(user),
+    userPic: getImageUrl(user.profileImageUrlHttps).replace("_normal", "")
+  )
+
+  result.expandProfileEntities(user)

src/experimental/parser/utils.nim

@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import std/[sugar, strutils, times]
+import ../types/common
+import ../../utils as uutils
+
+template parseTime(time: string; f: static string; flen: int): DateTime =
+  if time.len != flen: return
+  parse(time, f, utc())
+
+proc parseIsoDate*(date: string): DateTime =
+  date.parseTime("yyyy-MM-dd\'T\'HH:mm:ss\'Z\'", 20)
+
+proc parseTwitterDate*(date: string): DateTime =
+  date.parseTime("ddd MMM dd hh:mm:ss \'+0000\' yyyy", 30)
+
+proc getImageUrl*(url: string): string =
+  url.dup(removePrefix(twimg), removePrefix(https))
+
+template handleErrors*(body) =
+  if json.startsWith("{\"errors"):
+    let error {.inject.} = json.fromJson(Errors).errors[0].code
+    body

src/experimental/types/common.nim

@@ -0,0 +1,30 @@
+from ../../types import Error
+
+type
+  Url* = object
+    url*: string
+    expandedUrl*: string
+    displayUrl*: string
+    indices*: array[2, int]
+
+  ErrorCode* = enum
+    null = 0
+    noUserMatches = 17
+    protectedUser = 22
+    couldntAuth = 32
+    doesntExist = 34
+    userNotFound = 50
+    suspended = 63
+    rateLimited = 88
+    invalidToken = 89
+    listIdOrSlug = 112
+    forbidden = 200
+    badToken = 239
+    noCsrf = 353
+
+  ErrorObj* = object
+    code*: Error
+    message*: string
+
+  Errors* = object
+    errors*: seq[ErrorObj]

src/experimental/types/user.nim

@@ -0,0 +1,28 @@
+import common
+
+type
+  User* = object
+    idStr*: string
+    name*: string
+    screenName*: string
+    location*: string
+    description*: string
+    entities*: Entities
+    createdAt*: string
+    followersCount*: int
+    friendsCount*: int
+    favouritesCount*: int
+    statusesCount*: int
+    mediaCount*: int
+    verified*: bool
+    protected*: bool
+    profileBannerUrl*: string
+    profileImageUrlHttps*: string
+    profileLinkColor*: string
+
+  Entities* = object
+    url*: Urls
+    description*: Urls
+
+  Urls* = object
+    urls*: seq[Url]