Cynosphere/nitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add list support

Browse source
This commit is contained in:
Zed 2019-09-21 01:08:30 +02:00
parent d1fbcef64d
commit 9e3138e51b
25 changed files with 224 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/api.nim
View file

@ -1,2 +1,2 @@
import api/[profile, timeline, tweet, search, media]
export profile, timeline, tweet, search, media
import api/[profile, timeline, tweet, search, media, list]
export profile, timeline, tweet, search, media, list

2
src/api/consts.nim
View file

@ -11,6 +11,8 @@ const
timelineUrl* = "i/profiles/show/$1/timeline/tweets"
timelineMediaUrl* = "i/profiles/show/$1/media_timeline"
listUrl* = "$1/lists/$2/timeline"
listMembersUrl* = "$1/lists/$2/members"
profilePopupUrl* = "i/profiles/popup"
profileIntentUrl* = "intent/user"
searchUrl* = "i/search/timeline"

83
src/api/list.nim Normal file
View file

@ -0,0 +1,83 @@
import httpclient, asyncdispatch, htmlparser, strformat
import sequtils, strutils, json, uri
import ".."/[types, parser, parserutils, query]
import utils, consts, timeline, search
proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.async.} =
let url = base / (listUrl % [username, list])
let headers = newHttpHeaders({
"Accept": jsonAccept,
"Referer": $url,
"User-Agent": agent,
"X-Twitter-Active-User": "yes",
"X-Requested-With": "XMLHttpRequest",
"Accept-Language": lang
})
var params = toSeq({
"include_available_features": "1",
"include_entities": "1",
"reset_error_state": "false"
})
if after.len > 0:
params.add {"max_position": after}
let json = await fetchJson(url ? params, headers)
result = await finishTimeline(json, Query(), after, agent)
if result.content.len > 0:
result.minId = result.content[^1].id
proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.async.} =
let url = base / (listMembersUrl % [username, list])
let headers = newHttpHeaders({
"Accept": htmlAccept,
"Referer": $(base / &"{username}/lists/{list}/members"),
"User-Agent": agent,
"Accept-Language": lang
})
let html = await fetchHtml(url, headers)
result = Result[Profile](
minId: html.selectAttr(".stream-container", "data-min-position"),
hasMore: html.select(".has-more-items") != nil,
beginning: true,
query: Query(kind: users),
content: html.selectAll(".account").map(parseListProfile)
)
proc getListMembersSearch*(username, list, agent, after: string): Future[Result[Profile]] {.async.} =
let url = base / ((listMembersUrl & "/timeline") % [username, list])
let headers = newHttpHeaders({
"Accept": jsonAccept,
"Referer": $(base / &"{username}/lists/{list}/members"),
"User-Agent": agent,
"X-Twitter-Active-User": "yes",
"X-Requested-With": "XMLHttpRequest",
"X-Push-With": "XMLHttpRequest",
"Accept-Language": lang
})
var params = toSeq({
"include_available_features": "1",
"include_entities": "1",
"reset_error_state": "false"
})
if after.len > 0:
params.add {"max_position": after}
let json = await fetchJson(url ? params, headers)
result = getResult[Profile](json, Query(kind: users), after)
if json == nil or not json.hasKey("items_html"): return
let html = json["items_html"].to(string)
result.hasMore = html != "\n"
for p in parseHtml(html).selectAll(".account"):
result.content.add parseListProfile(p)

7
src/api/search.nim
View file

@ -7,7 +7,7 @@ import utils, consts, timeline
proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] =
if json == nil: return Result[T](beginning: true, query: query)
Result[T](
hasMore: json["has_more_items"].to(bool),
hasMore: json.getOrDefault("has_more_items").getBool(false),
maxId: json.getOrDefault("max_position").getStr(""),
minId: json.getOrDefault("min_position").getStr("").cleanPos(),
query: query,
@ -16,7 +16,7 @@ proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] =
proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.async.} =
let
kind = if query.kind == users: "users" else: "tweets"
kind = if query.kind == userSearch: "users" else: "tweets"
pos = when T is Tweet: genPos(after) else: after
param = genQueryParam(query)
@ -46,10 +46,9 @@ proc getSearch*[T](query: Query; after, agent: string): Future[Result[T]] {.asyn
return Result[T](query: query, beginning: true)
let json = await fetchJson(base / searchUrl ? params, headers)
if json == nil: return Result[T](query: query, beginning: true)
result = getResult[T](json, query, after)
if not json.hasKey("items_html"): return
if json == nil or not json.hasKey("items_html"): return
when T is Tweet:
result = await finishTimeline(json, query, after, agent)

2
src/api/timeline.nim
View file

@ -1,4 +1,4 @@
import httpclient, asyncdispatch, htmlparser
import httpclient, asyncdispatch, htmlparser, strformat
import sequtils, strutils, json, xmltree, uri
import ".."/[types, parser, parserutils, formatters, query]

8
src/nitter.nim
View file

@ -5,13 +5,14 @@ import jester
import types, config, prefs
import views/[general, about]
import routes/[preferences, timeline, status, media, search, rss]
import routes/[preferences, timeline, status, media, search, rss, list]
const configPath {.strdefine.} = "./nitter.conf"
let cfg = getConfig(configPath)
createPrefRouter(cfg)
createTimelineRouter(cfg)
createListRouter(cfg)
createStatusRouter(cfg)
createSearchRouter(cfg)
createMediaRouter(cfg)
@ -24,15 +25,16 @@ settings:
routes:
get "/":
resp renderMain(renderSearch(), Prefs(), cfg.title)
resp renderMain(renderSearch(), request, cfg.title)
get "/about":
resp renderMain(renderAbout(), Prefs(), cfg.title)
resp renderMain(renderAbout(), request, cfg.title)
extend preferences, ""
extend rss, ""
extend search, ""
extend timeline, ""
extend list, ""
extend status, ""
extend media, ""

10
src/parser.nim
View file

@ -39,6 +39,16 @@ proc parsePopupProfile*(node: XmlNode; selector=".profile-card"): Profile =
result.getPopupStats(profile)
proc parseListProfile*(profile: XmlNode): Profile =
result = Profile(
fullname: profile.getName(".fullname"),
username: profile.getUsername(".username"),
bio: profile.getBio(".bio"),
userpic: profile.getAvatar(".avatar"),
verified: isVerified(profile),
protected: isProtected(profile),
)
proc parseIntentProfile*(profile: XmlNode): Profile =
result = Profile(
fullname: profile.getName("a.fn.url.alternate-context"),

4
src/query.nim
View file

@ -58,7 +58,7 @@ proc genQueryParam*(query: Query): string =
var filters: seq[string]
var param: string
if query.kind == users:
if query.kind == userSearch:
return query.text
for i, user in query.fromUser:
@ -84,7 +84,7 @@ proc genQueryParam*(query: Query): string =
result &= " " & query.text
proc genQueryUrl*(query: Query): string =
if query.kind notin {custom, users}: return
if query.kind notin {custom, userSearch}: return
var params = @[&"kind={query.kind}"]
if query.text.len > 0:

34
src/routes/list.nim Normal file
View file

@ -0,0 +1,34 @@
import strutils
import jester
import router_utils
import ".."/[query, types, api, agents]
import ../views/[general, timeline, list]
template respList*(list, timeline: typed) =
if list.minId.len == 0:
resp Http404, showError("List \"" & @"list" & "\" not found", cfg.title)
let html = renderList(timeline, list.query, @"name", @"list")
let rss = "/$1/lists/$2/rss" % [@"name", @"list"]
resp renderMain(html, request, cfg.title, rss=rss)
proc createListRouter*(cfg: Config) =
router list:
get "/@name/lists/@list":
cond '.' notin @"name"
let
list = await getListTimeline(@"name", @"list", getAgent(), @"after")
tweets = renderTimelineTweets(list, cookiePrefs(), request.path)
respList list, tweets
get "/@name/lists/@list/members":
cond '.' notin @"name"
let list =
if @"after".len == 0:
await getListMembers(@"name", @"list", getAgent())
else:
await getListMembersSearch(@"name", @"list", getAgent(), @"after")
let users = renderTimelineUsers(list, cookiePrefs(), request.path)
respList list, users

2
src/routes/media.nim
View file

@ -3,7 +3,7 @@ import asyncfile, uri, strutils, httpclient, os
import jester, regex
import router_utils
import ".."/[types, formatters, prefs]
import ".."/[types, formatters]
import ../views/general
export asyncfile, httpclient, os, strutils

2
src/routes/preferences.nim
View file

@ -3,7 +3,7 @@ import strutils, uri
import jester
import router_utils
import ".."/[prefs, types]
import ".."/[types]
import ../views/[general, preferences]
export preferences

4
src/routes/router_utils.nim
View file

@ -1,5 +1,5 @@
import ../utils
export utils
import ../utils, ../prefs
export utils, prefs
template cookiePrefs*(): untyped {.dirty.} =
getPrefs(request.cookies.getOrDefault("preferences"))

5
src/routes/rss.nim
View file

@ -34,3 +34,8 @@ proc createRssRouter*(cfg: Config) =
get "/@name/search/rss":
cond '.' notin @"name"
respRss(await showRss(@"name", initQuery(params(request), name=(@"name"))))
get "/@name/lists/@list/rss":
cond '.' notin @"name"
let list = await getListTimeline(@"name", @"list", getAgent(), "")
respRss(renderListRss(list.content, @"name", @"list"))

4
src/routes/search.nim
View file

@ -3,7 +3,7 @@ import strutils, sequtils, uri
import jester
import router_utils
import ".."/[query, types, api, agents, prefs]
import ".."/[query, types, api, agents]
import ../views/[general, search]
export search
@ -18,7 +18,7 @@ proc createSearchRouter*(cfg: Config) =
let query = initQuery(params(request))
case query.kind
of users:
of userSearch:
if "," in @"text":
redirect("/" & @"text")
let users = await getSearch[Profile](query, @"after", getAgent())

2
src/routes/status.nim
View file

@ -3,7 +3,7 @@ import asyncdispatch, strutils, sequtils, uri
import jester
import router_utils
import ".."/[api, prefs, types, formatters, agents]
import ".."/[api, types, formatters, agents]
import ../views/[general, status]
export uri, sequtils

2
src/routes/timeline.nim
View file

@ -3,7 +3,7 @@ import asyncdispatch, strutils, sequtils, uri
import jester
import router_utils
import ".."/[api, prefs, types, cache, formatters, agents, query]
import ".."/[api, types, cache, formatters, agents, query]
import ../views/[general, profile, timeline, status, search]
export uri, sequtils

5
src/sass/tweet/_base.scss
View file

@ -85,7 +85,10 @@
.replying-to {
color: $fg_dark;
margin: -2px 0 4px;
pointer-events: all;
a {
pointer-events: all;
}
}
.retweet, .pinned, .tweet-stats {

2
src/types.nim
View file

@ -57,7 +57,7 @@ dbFromTypes("cache.db", "", "", "", [Profile, Video])
type
QueryKind* = enum
posts, replies, media, users, custom
posts, replies, media, users, userSearch, custom
Query* = object
kind*: QueryKind

2
src/utils.nim
View file

@ -41,7 +41,7 @@ proc cleanFilename*(filename: string): string =
filename.replace(reg, "_")
proc filterParams*(params: Table): seq[(string, string)] =
let filter = ["name", "id"]
let filter = ["name", "id", "list"]
toSeq(params.pairs()).filterIt(it[0] notin filter and it[1].len > 0)
proc isTwitterUrl*(url: string): bool =

4
src/views/general.nim
View file

@ -71,5 +71,5 @@ proc renderError*(error: string): VNode =
tdiv(class="error-panel"):
span: text error
proc showError*(error, title: string): string =
renderMain(renderError(error), Request(), title, "Error")
template showError*(error, title: string): string =
renderMain(renderError(error), request, title, "Error")

20
src/views/list.nim Normal file
View file

@ -0,0 +1,20 @@
import strformat
import karax/[karaxdsl, vdom]
import renderutils
import ".."/[types]
proc renderListTabs*(query: Query; path: string): VNode =
buildHtml(ul(class="tab")):
li(class=query.getTabClass(posts)):
a(href=(path)): text "Tweets"
li(class=query.getTabClass(users)):
a(href=(path & "/members")): text "Members"
proc renderList*(body: VNode; query: Query; name, list: string): VNode =
buildHtml(tdiv(class="timeline-container")):
tdiv(class="timeline-header"):
text &"\"{list}\" by @{name}"
renderListTabs(query, &"/{name}/lists/{list}")
body

13
src/views/renderutils.nim
View file

@ -30,10 +30,6 @@ proc linkUser*(profile: Profile, class=""): VNode =
text " "
icon "lock-circled", title="Protected account"
proc genImg*(url: string; class=""): VNode =
buildHtml():
img(src=getPicUrl(url), class=class, alt="Image")
proc linkText*(text: string; class=""): VNode =
let url = if "http" notin text: "http://" & text else: text
buildHtml():
@ -91,3 +87,12 @@ proc genDate*(pref, state: string): VNode =
else:
verbatim &"<input name={pref} type=\"date\"/>"
icon "calendar"
proc genImg*(url: string; class=""): VNode =
buildHtml():
img(src=getPicUrl(url), class=class, alt="Image")
proc getTabClass*(query: Query; tab: QueryKind): string =
result = "tab-item"
if query.kind == tab:
result &= " active"

26
src/views/rss.nimf
View file

@ -71,3 +71,29 @@
</channel>
</rss>
#end proc
#
#proc renderListRss*(tweets: seq[Tweet]; name, list: string): string =
#let prefs = Prefs(replaceTwitter: hostname)
#result = ""
<?xml version="1.0" encoding="UTF-8"?>
<rss xmlns:atom="http://www.w3.org/2005/Atom" xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0">
<channel>
<atom:link href="https://${hostname}/${name}/lists/${list}/rss" rel="self" type="application/rss+xml" />
<title>${list} / @${name}</title>
<link>https://${hostname}/${name}/lists/${list}</link>
<description>Twitter feed for: ${list} by @${name}. Generated by ${hostname}</description>
<language>en-us</language>
<ttl>40</ttl>
#for tweet in tweets:
<item>
<title>${getTitle(tweet, prefs)}</title>
<dc:creator>@${tweet.profile.username}</dc:creator>
<description><![CDATA[${renderRssTweet(tweet, prefs).strip(chars={'\n'})}]]></description>
<pubDate>${getRfc822Time(tweet)}</pubDate>
<guid>https://${hostname}${getLink(tweet)}</guid>
<link>https://${hostname}${getLink(tweet)}</link>
</item>
#end for
</channel>
</rss>
#end proc

13
src/views/search.nim
View file

@ -23,15 +23,10 @@ proc renderSearch*(): VNode =
buildHtml(tdiv(class="panel-container")):
tdiv(class="search-bar"):
form(`method`="get", action="/search"):
hiddenField("kind", "users")
hiddenField("kind", "userSearch")
input(`type`="text", name="text", autofocus="", placeholder="Enter username...")
button(`type`="submit"): icon "search"
proc getTabClass(query: Query; tab: QueryKind): string =
result = "tab-item"
if query.kind == tab:
result &= " active"
proc renderProfileTabs*(query: Query; username: string): VNode =
let link = "/" & username
buildHtml(ul(class="tab")):
@ -50,8 +45,8 @@ proc renderSearchTabs*(query: Query): VNode =
li(class=query.getTabClass(custom)):
q.kind = custom
a(href=("?" & genQueryUrl(q))): text "Tweets"
li(class=query.getTabClass(users)):
q.kind = users
li(class=query.getTabClass(userSearch)):
q.kind = userSearch
a(href=("?" & genQueryUrl(q))): text "Users"
proc isPanelOpen(q: Query): bool =
@ -114,7 +109,7 @@ proc renderUserSearch*(users: Result[Profile]; prefs: Prefs): VNode =
buildHtml(tdiv(class="timeline-container")):
tdiv(class="timeline-header"):
form(`method`="get", action="/search", class="search-field"):
hiddenField("kind", "users")
hiddenField("kind", "userSearch")
genInput("text", "", users.query.text, "Enter username...", class="pref-inline")
button(`type`="submit"): icon "search"

3
src/views/timeline.nim
View file

@ -64,7 +64,8 @@ proc renderTimelineUsers*(results: Result[Profile]; prefs: Prefs; path=""): VNod
if results.content.len > 0:
for user in results.content:
renderUser(user, prefs)
renderMore(results.query, results.minId)
if results.minId != "0":
renderMore(results.query, results.minId)
elif results.beginning:
renderNoneFound()
else: