mirror of
https://gitea.invidious.io/iv-org/invidious-copy-2022-08-14.git
synced 2024-08-15 00:53:20 +00:00
Use seperate table for videos pulled from RSS
This commit is contained in:
parent
e89f15a65c
commit
239a6c892c
6 changed files with 81 additions and 54 deletions
20
config/sql/channel_videos.sql
Normal file
20
config/sql/channel_videos.sql
Normal file
|
@ -0,0 +1,20 @@
|
|||
-- Table: public.channel_videos
|
||||
|
||||
-- DROP TABLE public.channel_videos;
|
||||
|
||||
CREATE TABLE public.channel_videos
|
||||
(
|
||||
id text COLLATE pg_catalog."default" NOT NULL,
|
||||
title text COLLATE pg_catalog."default",
|
||||
published timestamp with time zone,
|
||||
updated timestamp with time zone,
|
||||
ucid text COLLATE pg_catalog."default",
|
||||
author text COLLATE pg_catalog."default",
|
||||
CONSTRAINT channel_videos_id_key UNIQUE (id)
|
||||
)
|
||||
WITH (
|
||||
OIDS = FALSE
|
||||
)
|
||||
TABLESPACE pg_default;
|
||||
|
||||
GRANT ALL ON TABLE public.channel_videos TO kemal;
|
|
@ -5,9 +5,8 @@
|
|||
CREATE TABLE public.channels
|
||||
(
|
||||
id text COLLATE pg_catalog."default" NOT NULL,
|
||||
rss text COLLATE pg_catalog."default",
|
||||
updated timestamp with time zone,
|
||||
author text COLLATE pg_catalog."default"
|
||||
author text COLLATE pg_catalog."default",
|
||||
updated timestamp with time zone
|
||||
)
|
||||
WITH (
|
||||
OIDS = FALSE
|
||||
|
@ -15,12 +14,3 @@ WITH (
|
|||
TABLESPACE pg_default;
|
||||
|
||||
GRANT ALL ON TABLE public.channels TO kemal;
|
||||
|
||||
-- Index: channel_id_idx
|
||||
|
||||
-- DROP INDEX public.channel_id_idx;
|
||||
|
||||
CREATE UNIQUE INDEX channel_id_idx
|
||||
ON public.channels USING btree
|
||||
(id COLLATE pg_catalog."default")
|
||||
TABLESPACE pg_default;
|
1
setup.sh
1
setup.sh
|
@ -4,3 +4,4 @@ createdb invidious
|
|||
createuser kemal
|
||||
psql invidious < config/sql/channels.sql
|
||||
psql invidious < config/sql/videos.sql
|
||||
psql invidious < config/sql/channel_videos.sql
|
||||
|
|
|
@ -62,15 +62,20 @@ class InvidiousChannel
|
|||
end
|
||||
|
||||
add_mapping({
|
||||
id: String,
|
||||
rss: {
|
||||
type: XML::Node,
|
||||
default: XML.parse_html(""),
|
||||
converter: InvidiousChannel::XMLConverter,
|
||||
|
||||
},
|
||||
updated: Time,
|
||||
id: String,
|
||||
author: String,
|
||||
updated: Time,
|
||||
})
|
||||
end
|
||||
|
||||
class ChannelVideo
|
||||
add_mapping({
|
||||
id: String,
|
||||
title: String,
|
||||
published: Time,
|
||||
updated: Time,
|
||||
ucid: String,
|
||||
author: String,
|
||||
})
|
||||
end
|
||||
|
||||
|
@ -203,10 +208,16 @@ def get_video(id, client, db, refresh = true)
|
|||
|
||||
# If record was last updated over an hour ago, refresh (expire param in response lasts for 6 hours)
|
||||
if refresh && Time.now - video.updated > 1.hours
|
||||
db.exec("DELETE FROM videos * WHERE id = $1", id)
|
||||
begin
|
||||
video = fetch_video(id, client)
|
||||
args = arg_array(video.to_a)
|
||||
db.exec("INSERT INTO videos VALUES (#{args})", video.to_a)
|
||||
video_array = video.to_a[1..-1]
|
||||
args = arg_array(video_array)
|
||||
|
||||
db.exec("UPDATE videos SET (id,info,updated,title,views,likes,dislikes,wilson_score,published,description,language)\
|
||||
= (#{args}) WHERE id = '#{video.id}'", video_array)
|
||||
rescue ex
|
||||
db.exec("DELETE FROM videos * WHERE id = $1", id)
|
||||
end
|
||||
end
|
||||
else
|
||||
video = fetch_video(id, client)
|
||||
|
@ -490,14 +501,14 @@ def get_channel(id, client, db)
|
|||
channel = db.query_one("SELECT * FROM channels WHERE id = $1", id, as: InvidiousChannel)
|
||||
|
||||
if Time.now - channel.updated > 1.minutes
|
||||
channel = fetch_channel(id, client)
|
||||
channel = fetch_channel(id, client, db)
|
||||
channel_array = channel.to_a[1..-1]
|
||||
args = arg_array(channel_array)
|
||||
|
||||
db.exec("UPDATE channels SET (rss,updated,author) = (#{args}) WHERE id = '#{channel.id}'", channel_array)
|
||||
db.exec("UPDATE channels SET (author,updated) = (#{args}) WHERE id = '#{channel.id}'", channel_array)
|
||||
end
|
||||
else
|
||||
channel = fetch_channel(id, client)
|
||||
channel = fetch_channel(id, client, db)
|
||||
args = arg_array(channel.to_a)
|
||||
db.exec("INSERT INTO channels VALUES (#{args})", channel.to_a)
|
||||
end
|
||||
|
@ -505,13 +516,31 @@ def get_channel(id, client, db)
|
|||
return channel
|
||||
end
|
||||
|
||||
def fetch_channel(id, client)
|
||||
def fetch_channel(id, client, db)
|
||||
rss = client.get("/feeds/videos.xml?channel_id=#{id}").body
|
||||
rss = XML.parse_html(rss)
|
||||
|
||||
rss.xpath_nodes("//feed/entry").each do |entry|
|
||||
video_id = entry.xpath_node("videoid").not_nil!.content
|
||||
title = entry.xpath_node("title").not_nil!.content
|
||||
published = Time.parse(entry.xpath_node("published").not_nil!.content, "%FT%X%z")
|
||||
updated = Time.parse(entry.xpath_node("updated").not_nil!.content, "%FT%X%z")
|
||||
author = entry.xpath_node("author/name").not_nil!.content
|
||||
ucid = entry.xpath_node("channelid").not_nil!.content
|
||||
|
||||
video = ChannelVideo.new(video_id, title, published, updated, ucid, author)
|
||||
|
||||
video_array = video.to_a[1..-1]
|
||||
args = arg_array(video_array)
|
||||
|
||||
# TODO: Update record on conflict
|
||||
db.exec("INSERT INTO channel_videos VALUES (#{arg_array(video.to_a)})\
|
||||
ON CONFLICT (id) DO NOTHING", video.to_a)
|
||||
end
|
||||
|
||||
author = rss.xpath_node("//feed/author/name").not_nil!.content
|
||||
|
||||
channel = InvidiousChannel.new(id, rss, Time.now, author)
|
||||
channel = InvidiousChannel.new(id, author, Time.now)
|
||||
|
||||
return channel
|
||||
end
|
||||
|
|
|
@ -574,35 +574,22 @@ get "/feed/subscriptions" do |env|
|
|||
|
||||
feed = client.get("/subscription_manager?action_takeout=1", headers).body
|
||||
|
||||
videos = Array(Hash(String, String | Time)).new
|
||||
channels = [] of String
|
||||
|
||||
feed = XML.parse_html(feed)
|
||||
feed.xpath_nodes("//opml/outline/outline").each do |channel|
|
||||
id = channel["xmlurl"][-24..-1]
|
||||
rss = get_channel(id, client, PG_DB).rss
|
||||
get_channel(id, client, PG_DB)
|
||||
|
||||
rss.xpath_nodes("//feed/entry").each do |entry|
|
||||
video = {} of String => String | Time
|
||||
|
||||
video["id"] = entry.xpath_node("videoid").not_nil!.content
|
||||
video["title"] = entry.xpath_node("title").not_nil!.content
|
||||
video["published"] = Time.parse(entry.xpath_node("published").not_nil!.content, "%FT%X%z")
|
||||
video["author"] = entry.xpath_node("author/name").not_nil!.content
|
||||
video["ucid"] = entry.xpath_node("channelid").not_nil!.content
|
||||
video["thumbnail"] = entry.xpath_node("group/thumbnail").not_nil!["url"].gsub(/hqdefault\.jpg$/, "mqdefault.jpg")
|
||||
|
||||
videos << video
|
||||
end
|
||||
channels << id
|
||||
end
|
||||
|
||||
youtube_pool << client
|
||||
|
||||
videos.sort_by! { |video| video["published"].as(Time).epoch }
|
||||
videos.reverse!
|
||||
|
||||
start = (page - 1)*max_results
|
||||
stop = start + max_results - 1
|
||||
videos = videos[start..stop]
|
||||
time = Time.now
|
||||
args = arg_array(channels)
|
||||
offset = (page - 1) * max_results
|
||||
videos = PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid IN (#{args})\
|
||||
ORDER BY published DESC LIMIT #{max_results} OFFSET #{offset}", channels, as: ChannelVideo)
|
||||
|
||||
templated "subscriptions"
|
||||
else
|
||||
|
|
|
@ -7,15 +7,15 @@
|
|||
<% slice.each do |video| %>
|
||||
<div class="pure-u-1 pure-u-md-1-4">
|
||||
<div style="overflow-wrap:break-word; word-wrap:break-word;" class="h-box">
|
||||
<a style="width:100%;" href="/watch?v=<%= video["id"] %>">
|
||||
<img style="width:100%;" src="<%= video["thumbnail"] %>"/>
|
||||
<p style="height:100%"><%= video["title"] %></p>
|
||||
<a style="width:100%;" href="/watch?v=<%= video.id %>">
|
||||
<img style="width:100%;" src="https://i.ytimg.com/vi/<%= video.id %>/mqdefault.jpg"/>
|
||||
<p style="height:100%"><%= video.title %></p>
|
||||
</a>
|
||||
<p>
|
||||
<b><a style="width:100%;" href="https://youtube.com/channel/<%= video["ucid"] %>"><%= video["author"] %></a></b>
|
||||
<b><a style="width:100%;" href="https://youtube.com/channel/<%= video.author %>"><%= video.author %></a></b>
|
||||
</p>
|
||||
<p>
|
||||
<h5>Shared <%= video["published"].as(Time).to_s("%B %-d, %Y at %r") %></h5>
|
||||
<h5>Shared <%= video.published.to_s("%B %-d, %Y at %r") %></h5>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
|
Loading…
Reference in a new issue