mirror of
https://gitea.invidious.io/iv-org/invidious-copy-2022-08-14.git
synced 2024-08-15 00:53:20 +00:00
Add related videos and clean up video class
This commit is contained in:
Omar Roth
parent
d1eb81b653
commit
060c4da96d
5 changed files with 180 additions and 164 deletions
244
src/invidious.cr
244
src/invidious.cr
|
|
@ -5,85 +5,53 @@ require "pg"
|
|||
require "xml"
|
||||
require "time"
|
||||
|
||||
PG_DB = DB.open "postgres://kemal:kemal@localhost:5432/invidious"
|
||||
CONTEXT = OpenSSL::SSL::Context::Client.insecure
|
||||
|
||||
macro templated(filename)
|
||||
render "src/views/#{{{filename}}}.ecr", "src/views/layout.ecr"
|
||||
end
|
||||
|
||||
class Video
|
||||
getter last_updated : Time
|
||||
getter video_id : String
|
||||
getter video_info : String
|
||||
getter video_html : String
|
||||
getter views : String
|
||||
getter likes : Int32
|
||||
getter dislikes : Int32
|
||||
getter rating : Float64
|
||||
getter description : String
|
||||
module HTTPParamConverter
|
||||
def self.from_rs(rs)
|
||||
HTTP::Params.parse(rs.read(String))
|
||||
end
|
||||
end
|
||||
|
||||
def initialize(last_updated, video_id, video_info, video_html, views, likes, dislikes, rating, description)
|
||||
@last_updated = last_updated
|
||||
@video_id = video_id
|
||||
@video_info = video_info
|
||||
@video_html = video_html
|
||||
@views = views
|
||||
@likes = likes
|
||||
@dislikes = dislikes
|
||||
@rating = rating
|
||||
@description = description
|
||||
module XMLConverter
|
||||
def self.from_rs(rs)
|
||||
XML.parse(rs.read(String))
|
||||
end
|
||||
end
|
||||
|
||||
def initialize(id, info, html, updated)
|
||||
@id = id
|
||||
@info = info
|
||||
@html = html
|
||||
@updated = updated
|
||||
end
|
||||
|
||||
def to_a
|
||||
return [@last_updated, @video_id, @video_info, @video_html, @views, @likes, @dislikes, @rating, @description]
|
||||
return [@id, @info, @html, @updated]
|
||||
end
|
||||
|
||||
DB.mapping({
|
||||
last_updated: Time,
|
||||
video_id: String,
|
||||
video_info: String,
|
||||
video_html: String,
|
||||
views: Int64,
|
||||
likes: Int32,
|
||||
dislikes: Int32,
|
||||
rating: Float64,
|
||||
description: String,
|
||||
id: String,
|
||||
info: {
|
||||
type: HTTP::Params,
|
||||
default: HTTP::Params.parse(""),
|
||||
converter: Video::HTTPParamConverter,
|
||||
},
|
||||
html: {
|
||||
type: XML::Node,
|
||||
default: XML.parse(""),
|
||||
converter: Video::XMLConverter,
|
||||
},
|
||||
updated: Time,
|
||||
})
|
||||
end
|
||||
|
||||
def get_video(video_id, context)
|
||||
client = HTTP::Client.new("www.youtube.com", 443, context)
|
||||
video_info = client.get("/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en").body
|
||||
info = HTTP::Params.parse(video_info)
|
||||
video_html = client.get("/watch?v=#{video_id}").body
|
||||
html = XML.parse(video_html)
|
||||
views = info["view_count"].to_i64
|
||||
rating = info["avg_rating"].to_f64
|
||||
|
||||
likes = html.xpath_node(%q(//button[@title="I like this"]/span))
|
||||
if likes
|
||||
likes = likes.content.delete(",").to_i
|
||||
else
|
||||
likes = 1
|
||||
end
|
||||
|
||||
dislikes = html.xpath_node(%q(//button[@title="I dislike this"]/span))
|
||||
if dislikes
|
||||
dislikes = dislikes.content.delete(",").to_i
|
||||
else
|
||||
dislikes = 1
|
||||
end
|
||||
|
||||
description = html.xpath_node(%q(//p[@id="eow-description"]))
|
||||
if description
|
||||
description = description.to_xml
|
||||
else
|
||||
description = ""
|
||||
end
|
||||
|
||||
video_record = Video.new(Time.now, video_id, video_info, video_html, views, likes, dislikes, rating, description)
|
||||
|
||||
return video_record
|
||||
end
|
||||
|
||||
# See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html
|
||||
def ci_lower_bound(pos, n)
|
||||
if n == 0
|
||||
|
|
@ -97,15 +65,52 @@ def ci_lower_bound(pos, n)
|
|||
return (phat + z*z/(2*n) - z * Math.sqrt((phat*(1 - phat) + z*z/(4*n))/n))/(1 + z*z/n)
|
||||
end
|
||||
|
||||
def fetch_video(id)
|
||||
client = HTTP::Client.new("www.youtube.com", 443, CONTEXT)
|
||||
info = client.get("/get_video_info?video_id=#{id}&el=info&ps=default&eurl=&gl=US&hl=en").body
|
||||
info = HTTP::Params.parse(info)
|
||||
|
||||
html = client.get("/watch?v=#{id}").body
|
||||
html = XML.parse(html)
|
||||
|
||||
if info["reason"]?
|
||||
raise info["reason"]
|
||||
end
|
||||
|
||||
video = Video.new(id, info, html, Time.now)
|
||||
|
||||
return video
|
||||
end
|
||||
|
||||
def get_video(id)
|
||||
if PG_DB.query_one?("SELECT EXISTS ( SELECT true FROM videos WHERE id = $1)", id, as: Bool)
|
||||
video = PG_DB.query_one("SELECT * FROM videos WHERE id = $1", id, as: Video)
|
||||
|
||||
# If record was last updated more than 5 hours ago, refresh (expire param in response lasts for 6 hours)
|
||||
if Time.now - video.updated > Time::Span.new(0, 5, 0, 0)
|
||||
video = fetch_video(id)
|
||||
end
|
||||
else
|
||||
video = fetch_video(id)
|
||||
PG_DB.exec("INSERT INTO videos VALUES ($1, $2, $3, $4)", video.to_a)
|
||||
end
|
||||
|
||||
return video
|
||||
end
|
||||
|
||||
get "/" do |env|
|
||||
templated "index"
|
||||
end
|
||||
|
||||
pg = DB.open "postgres://kemal:kemal@localhost:5432/invidious"
|
||||
context = OpenSSL::SSL::Context::Client.insecure
|
||||
|
||||
get "/watch" do |env|
|
||||
video_id = env.params.query["v"]
|
||||
id = env.params.query["v"]
|
||||
|
||||
begin
|
||||
video = get_video(id)
|
||||
rescue ex
|
||||
error_message = ex.message
|
||||
next templated "error"
|
||||
end
|
||||
|
||||
if env.params.query["listen"]? && env.params.query["listen"] == "true"
|
||||
env.request.query_params.delete_all("listen")
|
||||
|
|
@ -115,55 +120,60 @@ get "/watch" do |env|
|
|||
listen = false
|
||||
end
|
||||
|
||||
if pg.query_one?("select exists (select true from videos where video_id = $1)", video_id, as: Bool)
|
||||
video_record = pg.query_one("select * from videos where video_id = $1", video_id, as: Video)
|
||||
|
||||
# If record was last updated more than 5 hours ago, refresh (expire param in response lasts for 6 hours)
|
||||
if Time.now - video_record.last_updated > Time::Span.new(0, 5, 0, 0)
|
||||
video_record = get_video(video_id, context)
|
||||
pg.exec("update videos set last_updated = $1, video_info = $3, video_html = $4,\
|
||||
views = $5, likes = $6, dislikes = $7, rating = $8, description = $9 where video_id = $2",
|
||||
video_record.to_a)
|
||||
end
|
||||
else
|
||||
client = HTTP::Client.new("www.youtube.com", 443, context)
|
||||
video_info = client.get("/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en").body
|
||||
info = HTTP::Params.parse(video_info)
|
||||
|
||||
if info["reason"]?
|
||||
error_message = info["reason"]
|
||||
next templated "error"
|
||||
end
|
||||
|
||||
video_record = get_video(video_id, context)
|
||||
pg.exec("insert into videos values ($1,$2,$3,$4,$5,$6,$7,$8, $9)", video_record.to_a)
|
||||
end
|
||||
|
||||
# last_updated, video_id, video_info, video_html, views, likes, dislikes, rating
|
||||
video_info = HTTP::Params.parse(video_record.video_info)
|
||||
video_html = XML.parse(video_record.video_html)
|
||||
|
||||
fmt_stream = [] of HTTP::Params
|
||||
video_info["url_encoded_fmt_stream_map"].split(",") do |string|
|
||||
video.info["url_encoded_fmt_stream_map"].split(",") do |string|
|
||||
fmt_stream << HTTP::Params.parse(string)
|
||||
end
|
||||
|
||||
adaptive_fmts = [] of HTTP::Params
|
||||
video_info["adaptive_fmts"].split(",") do |string|
|
||||
adaptive_fmts << HTTP::Params.parse(string)
|
||||
end
|
||||
|
||||
fmt_stream.reverse! # We want lowest quality first
|
||||
|
||||
related_videos = video_html.xpath_nodes(%q(//li/div/a[contains(@class,"content-link")]/@href))
|
||||
|
||||
if related_videos.empty?
|
||||
related_videos = video_html.xpath_nodes(%q(//ytd-compact-video-renderer/div/a/@href))
|
||||
adaptive_fmts = [] of HTTP::Params
|
||||
video.info["adaptive_fmts"].split(",") do |string|
|
||||
adaptive_fmts << HTTP::Params.parse(string)
|
||||
end
|
||||
|
||||
likes = video_record.likes.to_f
|
||||
dislikes = video_record.dislikes.to_f
|
||||
views = video_record.views.to_f
|
||||
related_videos = video.html.xpath_nodes(%q(//li/div/a[contains(@class,"content-link")]/@href))
|
||||
if related_videos.empty?
|
||||
related_videos = video.html.xpath_nodes(%q(//ytd-compact-video-renderer/div/a/@href))
|
||||
end
|
||||
|
||||
related_videos_list = [] of Video
|
||||
related_videos.each do |related_video|
|
||||
related_id = related_video.content.split("=")[1]
|
||||
begin
|
||||
related_videos_list << get_video(related_id)
|
||||
rescue ex
|
||||
p "#{related_id}: #{ex.message}"
|
||||
end
|
||||
end
|
||||
|
||||
likes = video.html.xpath_node(%q(//button[@title="I like this"]/span))
|
||||
if likes
|
||||
likes = likes.content.delete(",").to_i
|
||||
else
|
||||
likes = 1
|
||||
end
|
||||
|
||||
dislikes = video.html.xpath_node(%q(//button[@title="I dislike this"]/span))
|
||||
if dislikes
|
||||
dislikes = dislikes.content.delete(",").to_i
|
||||
else
|
||||
dislikes = 1
|
||||
end
|
||||
|
||||
description = video.html.xpath_node(%q(//p[@id="eow-description"]))
|
||||
if description
|
||||
description = description.to_xml
|
||||
else
|
||||
description = ""
|
||||
end
|
||||
|
||||
views = video.info["view_count"].to_i64
|
||||
rating = video.info["avg_rating"].to_f64
|
||||
|
||||
likes = likes.to_f
|
||||
dislikes = dislikes.to_f
|
||||
views = views.to_f
|
||||
|
||||
engagement = ((dislikes + likes)/views * 100)
|
||||
calculated_rating = (likes/(likes + dislikes) * 4 + 1)
|
||||
|
|
@ -173,9 +183,9 @@ end
|
|||
|
||||
get "/search" do |env|
|
||||
query = URI.escape(env.params.query["q"])
|
||||
client = HTTP::Client.new("www.youtube.com", 443, context)
|
||||
results_html = client.get("https://www.youtube.com/results?q=#{query}&page=1").body
|
||||
html = XML.parse(results_html)
|
||||
client = HTTP::Client.new("www.youtube.com", 443, CONTEXT)
|
||||
html = client.get("https://www.youtube.com/results?q=#{query}&page=1").body
|
||||
html = XML.parse(html)
|
||||
|
||||
videos = html.xpath_nodes(%q(//div[@class="style-scope ytd-item-section-renderer"]/ytd-video-renderer))
|
||||
channels = html.xpath_nodes(%q(//div[@class="style-scope ytd-item-section-renderer"]/ytd-channel-renderer))
|
||||
|
|
@ -185,6 +195,16 @@ get "/search" do |env|
|
|||
channels = html.xpath_nodes(%q(//div[contains(@class,"yt-lockup-channel")]/div/div[contains(@class,"yt-lockup-thumbnail")]/a/@href))
|
||||
end
|
||||
|
||||
videos_list = [] of Video
|
||||
videos.each do |video|
|
||||
id = video.content.split("=")[1]
|
||||
begin
|
||||
videos_list << get_video(id)
|
||||
rescue ex
|
||||
p "#{id}: #{ex.message}"
|
||||
end
|
||||
end
|
||||
|
||||
templated "search"
|
||||
end
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue