mirror of
https://gitea.invidious.io/iv-org/invidious-copy-2022-03-16.git
synced 2024-08-15 00:53:18 +00:00
Fix full channel refresh
This commit is contained in:
parent
b6adeb80e6
commit
f38aac851e
1 changed files with 13 additions and 41 deletions
|
@ -69,61 +69,33 @@ def fetch_channel(ucid, client, db, pull_all_videos = true)
|
||||||
updated = $4, ucid = $5, author = $6", video_array)
|
updated = $4, ucid = $5, author = $6", video_array)
|
||||||
end
|
end
|
||||||
else
|
else
|
||||||
videos = [] of ChannelVideo
|
|
||||||
page = 1
|
page = 1
|
||||||
|
ids = [] of String
|
||||||
|
|
||||||
loop do
|
loop do
|
||||||
url = produce_channel_videos_url(ucid, page)
|
videos = extract_playlist(ucid, page)
|
||||||
response = client.get(url)
|
videos = videos.map { |video| ChannelVideo.new(video.id, video.title, video.published, Time.now, video.ucid, video.author) }
|
||||||
|
count = videos.size
|
||||||
|
|
||||||
json = JSON.parse(response.body)
|
|
||||||
content_html = json["content_html"].as_s
|
|
||||||
if content_html.empty?
|
|
||||||
# If we don't get anything, move on
|
|
||||||
break
|
|
||||||
end
|
|
||||||
document = XML.parse_html(content_html)
|
|
||||||
|
|
||||||
document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")])).each do |item|
|
|
||||||
anchor = item.xpath_node(%q(.//h3[contains(@class,"yt-lockup-title")]/a))
|
|
||||||
if !anchor
|
|
||||||
raise "could not find anchor"
|
|
||||||
end
|
|
||||||
|
|
||||||
title = anchor.content.strip
|
|
||||||
video_id = anchor["href"].lchop("/watch?v=")
|
|
||||||
|
|
||||||
published = item.xpath_node(%q(.//div[@class="yt-lockup-meta"]/ul/li[1]))
|
|
||||||
if !published
|
|
||||||
# This happens on Youtube red videos, here we just skip them
|
|
||||||
next
|
|
||||||
end
|
|
||||||
published = published.content
|
|
||||||
published = decode_date(published)
|
|
||||||
|
|
||||||
videos << ChannelVideo.new(video_id, title, published, Time.now, ucid, author)
|
|
||||||
end
|
|
||||||
|
|
||||||
if document.xpath_nodes(%q(//li[contains(@class, "channels-content-item")])).size < 30
|
|
||||||
break
|
|
||||||
end
|
|
||||||
|
|
||||||
page += 1
|
|
||||||
end
|
|
||||||
|
|
||||||
video_ids = [] of String
|
|
||||||
videos.each do |video|
|
videos.each do |video|
|
||||||
|
ids << video.id
|
||||||
db.exec("UPDATE users SET notifications = notifications || $1 \
|
db.exec("UPDATE users SET notifications = notifications || $1 \
|
||||||
WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)", video.id, video.published, ucid)
|
WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)", video.id, video.published, ucid)
|
||||||
video_ids << video.id
|
|
||||||
|
|
||||||
video_array = video.to_a
|
video_array = video.to_a
|
||||||
args = arg_array(video_array)
|
args = arg_array(video_array)
|
||||||
db.exec("INSERT INTO channel_videos VALUES (#{args}) ON CONFLICT (id) DO NOTHING", video_array)
|
db.exec("INSERT INTO channel_videos VALUES (#{args}) ON CONFLICT (id) DO NOTHING", video_array)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
if count < 100
|
||||||
|
break
|
||||||
|
end
|
||||||
|
|
||||||
|
page += 1
|
||||||
|
end
|
||||||
|
|
||||||
# When a video is deleted from a channel, we find and remove it here
|
# When a video is deleted from a channel, we find and remove it here
|
||||||
db.exec("DELETE FROM channel_videos * WHERE NOT id = ANY ('{#{video_ids.map { |a| %("#{a}") }.join(",")}}') AND ucid = $1", ucid)
|
db.exec("DELETE FROM channel_videos * WHERE NOT id = ANY ('{#{ids.map { |id| %("#{id}") }.join(",")}}') AND ucid = $1", ucid)
|
||||||
end
|
end
|
||||||
|
|
||||||
channel = InvidiousChannel.new(ucid, author, Time.now)
|
channel = InvidiousChannel.new(ucid, author, Time.now)
|
||||||
|
|
Loading…
Reference in a new issue