nitter/src/parser.nim

import xmltree, sequtils, strtabs, strutils, strformat, json

import types, parserutils, formatters

proc parsePopupProfile*(node: XmlNode): Profile =
  let profile = node.select(".profile-card")
  if profile == nil: return

  result = Profile(
    fullname:  profile.getName(".fullname"),
    username:  profile.getUsername(".username"),
    bio:       profile.getBio(".bio"),
    userpic:   profile.getAvatar(".ProfileCard-avatarImage"),
    verified:  isVerified(profile),
    protected: isProtected(profile),
    banner:    getBanner(profile)
  )

  result.getPopupStats(profile)

proc parseIntentProfile*(profile: XmlNode): Profile =
  result = Profile(
    fullname:  profile.getName("a.fn.url.alternate-context"),
    username:  profile.getUsername(".nickname"),
    bio:       profile.getBio("p.note"),
    userpic:   profile.select(".profile.summary").getAvatar("img.photo"),
    verified:  profile.select("li.verified") != nil,
    protected: profile.select("li.protected") != nil,
    banner:    getBanner(profile)
  )

  result.getIntentStats(profile)

proc parseTweetProfile*(profile: XmlNode): Profile =
  result = Profile(
    fullname: profile.attr("data-name").stripText(),
    username: profile.attr("data-screen-name"),
    userpic:  profile.getAvatar(".avatar"),
    verified: isVerified(profile)
  )

proc parseQuote*(quote: XmlNode): Quote =
  result = Quote(
    id:    quote.attr("data-item-id"),
    text:  getQuoteText(quote),
    reply: parseTweetReply(quote),
    hasThread: quote.select(".self-thread-context") != nil,
    available: true
  )

  result.profile = Profile(
    fullname: quote.selectText(".QuoteTweet-fullname").stripText(),
    username: quote.attr("data-screen-name"),
    verified: isVerified(quote)
  )

  result.getQuoteMedia(quote)

proc parseTweet*(node: XmlNode): Tweet =
  let tweet = node.select(".tweet")
  if tweet == nil: return Tweet()

  result = Tweet(
    id:        tweet.attr("data-item-id"),
    threadId:  tweet.attr("data-conversation-id"),
    text:      getTweetText(tweet),
    time:      getTimestamp(tweet),
    shortTime: getShortTime(tweet),
    profile:   parseTweetProfile(tweet),
    stats:     parseTweetStats(tweet),
    reply:     parseTweetReply(tweet),
    hasThread: tweet.select(".content > .self-thread-context") != nil,
    pinned:    "pinned" in tweet.attr("class"),
    available: true
  )

  result.getTweetMedia(tweet)
  result.getTweetCard(tweet)

  let by = tweet.selectText(".js-retweet-text > a > b")
  if by.len > 0:
    result.retweet = some(Retweet(
      by: stripText(by),
      id: tweet.attr("data-retweet-id")
    ))

  let quote = tweet.select(".QuoteTweet-innerContainer")
  if quote != nil:
    result.quote = some(parseQuote(quote))

  let tombstone = tweet.select(".Tombstone")
  if tombstone != nil:
    if "unavailable" in tombstone.innerText():
      result.quote = some(Quote())

proc parseThread*(nodes: XmlNode): Thread =
  if nodes == nil: return
  result = Thread()
  for n in nodes.filterIt(it.kind != xnText):
    let class = n.attr("class").toLower()
    if "tombstone" in class or "unavailable" in class:
      result.tweets.add Tweet()
    elif "morereplies" in class:
      result.more = getMoreReplies(n)
    else:
      result.tweets.add parseTweet(n)

proc parseConversation*(node: XmlNode): Conversation =
  result = Conversation(
    tweet:  parseTweet(node.select(".permalink-tweet-container")),
    before: parseThread(node.select(".in-reply-to .stream-items"))
  )

  let replies = node.select(".replies-to .stream-items")
  if replies == nil: return

  for i, reply in replies.filterIt(it.kind != xnText):
    let class = reply.attr("class").toLower()
    let thread = reply.select(".stream-items")

    if i == 0 and "self" in class:
      result.after = parseThread(thread)
    elif "lone" in class:
      result.replies.add parseThread(reply)
    else:
      result.replies.add parseThread(thread)

proc parseVideo*(node: JsonNode): Video =
  let
    track = node{"track"}
    cType = track["contentType"].to(string)
    pType = track["playbackType"].to(string)

  case cType
  of "media_entity":
    result = Video(
      playbackType: if "mp4" in pType: mp4 else: m3u8,
      contentId: track["contentId"].to(string),
      durationMs: track["durationMs"].to(int),
      views: track["viewCount"].to(string),
      url: track["playbackUrl"].to(string),
      available: track{"mediaAvailability"}["status"].to(string) == "available")
  of "vmap":
    result = Video(
      playbackType: vmap,
      durationMs: track["durationMs"].to(int),
      url: track["vmapUrl"].to(string))
  else:
    echo "Can't parse video of type ", cType

  result.thumb = node["posterImage"].to(string)

proc parsePoll*(node: XmlNode): Poll =
  let
    choices = node.selectAll(".PollXChoice-choice")
    votes = node.selectText(".PollXChoice-footer--total")

  result.votes = votes.strip().split(" ")[0]
  result.status = node.selectText(".PollXChoice-footer--time")

  for choice in choices:
    for span in choice.select(".PollXChoice-choice--text").filterIt(it.kind != xnText):
      if span.attr("class").len == 0:
        result.options.add span.innerText()
      elif "progress" in span.attr("class"):
        result.values.add parseInt(span.innerText()[0 .. ^2])

  var highest = 0
  for i, n in result.values:
    if n > highest:
      highest = n
      result.leader = i

proc parsePhotoRail*(node: XmlNode): seq[GalleryPhoto] =
  for img in node.selectAll(".tweet-media-img-placeholder"):
    result.add GalleryPhoto(
      url: img.attr("data-image-url"),
      tweetId: img.attr("data-tweet-id"),
      color: img.attr("background-color").replace("style: ", "")
    )

proc parseCard*(card: var Card; node: XmlNode) =
  let cardKind = node.select("head > meta[name*=card_name]").attr("content")

  if "summary_large_image" in cardKind:
    card.kind = summaryLarge
  elif "summary" in cardKind:
    card.kind = summary
  elif "live_event" in cardKind:
    card.kind = liveEvent
  elif "player" in cardKind:
    card.kind = player
  elif "promo_website" in cardKind:
    card.kind = promoWebsite

  card.title = node.selectText("h2.TwitterCard-title")
  card.text = node.selectText("p.tcu-resetMargin")
  card.dest = node.selectText("span.SummaryCard-destination")

  if card.url.len == 0:
    card.url = node.select("a").attr("href")

  let image = node.select(".tcu-imageWrapper img")
  if image != nil:
    # workaround for issue 11713
    card.image = image.attr("data-src").replace("gname", "g&name")
  else:
    echo card.id

  if card.kind == liveEvent:
    card.text = card.title
    card.title = node.selectText(".TwitterCard-attribution--category")