Use seperate table for videos pulled from RSS

This commit is contained in:
Omar Roth 2018-03-28 22:29:54 -05:00
parent e89f15a65c
commit 239a6c892c
6 changed files with 81 additions and 54 deletions

View file

@ -0,0 +1,20 @@
-- Table: public.channel_videos
-- DROP TABLE public.channel_videos;
CREATE TABLE public.channel_videos
(
id text COLLATE pg_catalog."default" NOT NULL,
title text COLLATE pg_catalog."default",
published timestamp with time zone,
updated timestamp with time zone,
ucid text COLLATE pg_catalog."default",
author text COLLATE pg_catalog."default",
CONSTRAINT channel_videos_id_key UNIQUE (id)
)
WITH (
OIDS = FALSE
)
TABLESPACE pg_default;
GRANT ALL ON TABLE public.channel_videos TO kemal;

View file

@ -5,9 +5,8 @@
CREATE TABLE public.channels CREATE TABLE public.channels
( (
id text COLLATE pg_catalog."default" NOT NULL, id text COLLATE pg_catalog."default" NOT NULL,
rss text COLLATE pg_catalog."default", author text COLLATE pg_catalog."default",
updated timestamp with time zone, updated timestamp with time zone
author text COLLATE pg_catalog."default"
) )
WITH ( WITH (
OIDS = FALSE OIDS = FALSE
@ -15,12 +14,3 @@ WITH (
TABLESPACE pg_default; TABLESPACE pg_default;
GRANT ALL ON TABLE public.channels TO kemal; GRANT ALL ON TABLE public.channels TO kemal;
-- Index: channel_id_idx
-- DROP INDEX public.channel_id_idx;
CREATE UNIQUE INDEX channel_id_idx
ON public.channels USING btree
(id COLLATE pg_catalog."default")
TABLESPACE pg_default;

View file

@ -4,3 +4,4 @@ createdb invidious
createuser kemal createuser kemal
psql invidious < config/sql/channels.sql psql invidious < config/sql/channels.sql
psql invidious < config/sql/videos.sql psql invidious < config/sql/videos.sql
psql invidious < config/sql/channel_videos.sql

View file

@ -63,13 +63,18 @@ class InvidiousChannel
add_mapping({ add_mapping({
id: String, id: String,
rss: { author: String,
type: XML::Node,
default: XML.parse_html(""),
converter: InvidiousChannel::XMLConverter,
},
updated: Time, updated: Time,
})
end
class ChannelVideo
add_mapping({
id: String,
title: String,
published: Time,
updated: Time,
ucid: String,
author: String, author: String,
}) })
end end
@ -203,10 +208,16 @@ def get_video(id, client, db, refresh = true)
# If record was last updated over an hour ago, refresh (expire param in response lasts for 6 hours) # If record was last updated over an hour ago, refresh (expire param in response lasts for 6 hours)
if refresh && Time.now - video.updated > 1.hours if refresh && Time.now - video.updated > 1.hours
db.exec("DELETE FROM videos * WHERE id = $1", id) begin
video = fetch_video(id, client) video = fetch_video(id, client)
args = arg_array(video.to_a) video_array = video.to_a[1..-1]
db.exec("INSERT INTO videos VALUES (#{args})", video.to_a) args = arg_array(video_array)
db.exec("UPDATE videos SET (id,info,updated,title,views,likes,dislikes,wilson_score,published,description,language)\
= (#{args}) WHERE id = '#{video.id}'", video_array)
rescue ex
db.exec("DELETE FROM videos * WHERE id = $1", id)
end
end end
else else
video = fetch_video(id, client) video = fetch_video(id, client)
@ -490,14 +501,14 @@ def get_channel(id, client, db)
channel = db.query_one("SELECT * FROM channels WHERE id = $1", id, as: InvidiousChannel) channel = db.query_one("SELECT * FROM channels WHERE id = $1", id, as: InvidiousChannel)
if Time.now - channel.updated > 1.minutes if Time.now - channel.updated > 1.minutes
channel = fetch_channel(id, client) channel = fetch_channel(id, client, db)
channel_array = channel.to_a[1..-1] channel_array = channel.to_a[1..-1]
args = arg_array(channel_array) args = arg_array(channel_array)
db.exec("UPDATE channels SET (rss,updated,author) = (#{args}) WHERE id = '#{channel.id}'", channel_array) db.exec("UPDATE channels SET (author,updated) = (#{args}) WHERE id = '#{channel.id}'", channel_array)
end end
else else
channel = fetch_channel(id, client) channel = fetch_channel(id, client, db)
args = arg_array(channel.to_a) args = arg_array(channel.to_a)
db.exec("INSERT INTO channels VALUES (#{args})", channel.to_a) db.exec("INSERT INTO channels VALUES (#{args})", channel.to_a)
end end
@ -505,13 +516,31 @@ def get_channel(id, client, db)
return channel return channel
end end
def fetch_channel(id, client) def fetch_channel(id, client, db)
rss = client.get("/feeds/videos.xml?channel_id=#{id}").body rss = client.get("/feeds/videos.xml?channel_id=#{id}").body
rss = XML.parse_html(rss) rss = XML.parse_html(rss)
rss.xpath_nodes("//feed/entry").each do |entry|
video_id = entry.xpath_node("videoid").not_nil!.content
title = entry.xpath_node("title").not_nil!.content
published = Time.parse(entry.xpath_node("published").not_nil!.content, "%FT%X%z")
updated = Time.parse(entry.xpath_node("updated").not_nil!.content, "%FT%X%z")
author = entry.xpath_node("author/name").not_nil!.content
ucid = entry.xpath_node("channelid").not_nil!.content
video = ChannelVideo.new(video_id, title, published, updated, ucid, author)
video_array = video.to_a[1..-1]
args = arg_array(video_array)
# TODO: Update record on conflict
db.exec("INSERT INTO channel_videos VALUES (#{arg_array(video.to_a)})\
ON CONFLICT (id) DO NOTHING", video.to_a)
end
author = rss.xpath_node("//feed/author/name").not_nil!.content author = rss.xpath_node("//feed/author/name").not_nil!.content
channel = InvidiousChannel.new(id, rss, Time.now, author) channel = InvidiousChannel.new(id, author, Time.now)
return channel return channel
end end

View file

@ -574,35 +574,22 @@ get "/feed/subscriptions" do |env|
feed = client.get("/subscription_manager?action_takeout=1", headers).body feed = client.get("/subscription_manager?action_takeout=1", headers).body
videos = Array(Hash(String, String | Time)).new channels = [] of String
feed = XML.parse_html(feed) feed = XML.parse_html(feed)
feed.xpath_nodes("//opml/outline/outline").each do |channel| feed.xpath_nodes("//opml/outline/outline").each do |channel|
id = channel["xmlurl"][-24..-1] id = channel["xmlurl"][-24..-1]
rss = get_channel(id, client, PG_DB).rss get_channel(id, client, PG_DB)
rss.xpath_nodes("//feed/entry").each do |entry| channels << id
video = {} of String => String | Time
video["id"] = entry.xpath_node("videoid").not_nil!.content
video["title"] = entry.xpath_node("title").not_nil!.content
video["published"] = Time.parse(entry.xpath_node("published").not_nil!.content, "%FT%X%z")
video["author"] = entry.xpath_node("author/name").not_nil!.content
video["ucid"] = entry.xpath_node("channelid").not_nil!.content
video["thumbnail"] = entry.xpath_node("group/thumbnail").not_nil!["url"].gsub(/hqdefault\.jpg$/, "mqdefault.jpg")
videos << video
end end
end
youtube_pool << client youtube_pool << client
videos.sort_by! { |video| video["published"].as(Time).epoch } time = Time.now
videos.reverse! args = arg_array(channels)
offset = (page - 1) * max_results
start = (page - 1)*max_results videos = PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid IN (#{args})\
stop = start + max_results - 1 ORDER BY published DESC LIMIT #{max_results} OFFSET #{offset}", channels, as: ChannelVideo)
videos = videos[start..stop]
templated "subscriptions" templated "subscriptions"
else else

View file

@ -7,15 +7,15 @@
<% slice.each do |video| %> <% slice.each do |video| %>
<div class="pure-u-1 pure-u-md-1-4"> <div class="pure-u-1 pure-u-md-1-4">
<div style="overflow-wrap:break-word; word-wrap:break-word;" class="h-box"> <div style="overflow-wrap:break-word; word-wrap:break-word;" class="h-box">
<a style="width:100%;" href="/watch?v=<%= video["id"] %>"> <a style="width:100%;" href="/watch?v=<%= video.id %>">
<img style="width:100%;" src="<%= video["thumbnail"] %>"/> <img style="width:100%;" src="https://i.ytimg.com/vi/<%= video.id %>/mqdefault.jpg"/>
<p style="height:100%"><%= video["title"] %></p> <p style="height:100%"><%= video.title %></p>
</a> </a>
<p> <p>
<b><a style="width:100%;" href="https://youtube.com/channel/<%= video["ucid"] %>"><%= video["author"] %></a></b> <b><a style="width:100%;" href="https://youtube.com/channel/<%= video.author %>"><%= video.author %></a></b>
</p> </p>
<p> <p>
<h5>Shared <%= video["published"].as(Time).to_s("%B %-d, %Y at %r") %></h5> <h5>Shared <%= video.published.to_s("%B %-d, %Y at %r") %></h5>
</p> </p>
</div> </div>
</div> </div>