From 38cff5a752aa855497346a6da4654670f82e43a6 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sun, 14 Jan 2018 21:16:09 -0600 Subject: [PATCH] Freshen up for the big dance --- assets/css/custom.css | 2 +- src/invidious.cr | 120 ++++++++++++++++++------------------------ src/views/index.ecr | 2 + src/views/search.ecr | 22 +++----- src/views/watch.ecr | 49 +++++++++++------ 5 files changed, 92 insertions(+), 103 deletions(-) diff --git a/assets/css/custom.css b/assets/css/custom.css index 2aa91778..70f0c0a6 100644 --- a/assets/css/custom.css +++ b/assets/css/custom.css @@ -1,5 +1,5 @@ a.link { - color: #222; + color: #333; text-decoration: none; } diff --git a/src/invidious.cr b/src/invidious.cr index 9ed41865..74cf8c17 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -25,7 +25,7 @@ class Video module XMLConverter def self.from_rs(rs) - XML.parse(rs.read(String)) + XML.parse_html(rs.read(String)) end end @@ -49,7 +49,7 @@ class Video }, html: { type: XML::Node, - default: XML.parse(""), + default: XML.parse_html(""), converter: Video::XMLConverter, }, updated: Time, @@ -69,20 +69,24 @@ def ci_lower_bound(pos, n) return (phat + z*z/(2*n) - z * Math.sqrt((phat*(1 - phat) + z*z/(4*n))/n))/(1 + z*z/n) end -def fetch_video(id) - # Grab connection from pool +def get_client while POOL.empty? sleep rand(0..10).milliseconds end - client = POOL.pop + return POOL.pop +end + +def fetch_video(id) + # Grab connection from pool + client = get_client # client = HTTP::Client.new("www.youtube.com", 443, CONTEXT) - info = client.get("/get_video_info?video_id=#{id}&el=info&ps=default&eurl=&gl=US&hl=en").body + info = client.get("/get_video_info?video_id=#{id}&el=detailpage&ps=default&eurl=&gl=US&hl=en").body info = HTTP::Params.parse(info) html = client.get("/watch?v=#{id}").body - html = XML.parse(html) + html = XML.parse_html(html) if info["reason"]? raise info["reason"] @@ -119,6 +123,7 @@ end get "/watch" do |env| id = env.params.query["v"] + listen = env.params.query["listen"]? || "false" begin video = get_video(id) @@ -127,14 +132,7 @@ get "/watch" do |env| next templated "error" end - query = HTTP::Params.parse(env.request.query.not_nil!) - if query["listen"]? && query["listen"] == "true" - query.delete_all("listen") - listen = true - else - query["listen"] = "true" - listen = false - end + player_response = JSON.parse(video.info["player_response"]) fmt_stream = [] of HTTP::Params video.info["url_encoded_fmt_stream_map"].split(",") do |string| @@ -148,83 +146,65 @@ get "/watch" do |env| adaptive_fmts << HTTP::Params.parse(string) end - related_videos = video.html.xpath_nodes(%q(//li/div/a[contains(@class,"content-link")]/@href)) - if related_videos.empty? - related_videos = video.html.xpath_nodes(%q(//ytd-compact-video-renderer/div/a/@href)) - end - - related_videos_list = [] of Video - related_videos.each do |related_video| - related_id = related_video.content.split("=")[1] - begin - related_videos_list << get_video(related_id, false) - rescue ex - p "#{related_id}: #{ex.message}" - end - end - likes = video.html.xpath_node(%q(//button[@title="I like this"]/span)) - if likes - likes = likes.content.delete(",").to_i - else - likes = 1 - end + likes = likes ? likes.content.delete(",").to_i : 1 dislikes = video.html.xpath_node(%q(//button[@title="I dislike this"]/span)) - if dislikes - dislikes = dislikes.content.delete(",").to_i - else - dislikes = 1 - end + dislikes = dislikes ? dislikes.content.delete(",").to_i : 1 description = video.html.xpath_node(%q(//p[@id="eow-description"])) - if description - description = description.to_xml - else - description = "" - end + description = description ? description.to_xml : "Could not load description" views = video.info["view_count"].to_i64 rating = video.info["avg_rating"].to_f64 - likes = likes.to_f - dislikes = dislikes.to_f - views = views.to_f + engagement = ((dislikes.to_f + likes.to_f)/views * 100) + calculated_rating = (likes.to_f/(likes.to_f + dislikes.to_f) * 4 + 1) - engagement = ((dislikes + likes)/views * 100) - calculated_rating = (likes/(likes + dislikes) * 4 + 1) + rvs = [] of Hash(String, String) + video.info["rvs"].split(",").each do |rv| + rvs << HTTP::Params.parse(rv).to_h + end templated "watch" end get "/search" do |env| query = env.params.query["q"] + page = env.params.query["page"]? && env.params.query["page"].to_i? ? env.params.query["page"].to_i : 1 + speed = env.params.query["speed"]? && env.params.query["speed"].to_i? ? env.params.query["speed"].to_i : 1 - while POOL.empty? - sleep rand(0..10).milliseconds - end + client = get_client - client = POOL.pop + html = client.get("https://www.youtube.com/results?q=#{URI.escape(query)}&page=#{page}").body + html = XML.parse_html(html) - html = client.get("https://www.youtube.com/results?q=#{URI.escape(query)}&page=1").body - html = XML.parse(html) + videos = Array(Hash(String, String)).new - videos = html.xpath_nodes(%q(//div[contains(@class,"yt-lockup-video")]/div/div[contains(@class,"yt-lockup-thumbnail")]/a/@href)) - channels = html.xpath_nodes(%q(//div[contains(@class,"yt-lockup-channel")]/div/div[contains(@class,"yt-lockup-thumbnail")]/a/@href)) + html.xpath_nodes(%q(//div[contains(@class,"yt-lockup-video")]/div)).each do |item| + video = {} of String => String - if videos.empty? - videos = html.xpath_nodes(%q(//div[contains(@class,"yt-lockup-video")]/div/div[@class="yt-lockup-content"]/h3/a/@href)) - channels = html.xpath_nodes(%q(//div[contains(@class,"yt-lockup-channel")]/div[@class="yt-lockup-content"]/h3/a/@href)) - end - - videos_list = [] of Video - videos.each do |video| - id = video.content.split("=")[1] - begin - videos_list << get_video(id, false) - rescue ex - p "#{id}: #{ex.message}" + link = item.xpath_node(%q(div/div[@class="yt-lockup-content"]/h3/a/@href)) + if link + video["link"] = link.content + else + link = item.xpath_node(%q(div[@class="yt-lockup-content"]/h3/a/@href)) + if link + video["link"] = link.content + end end + + title = item.xpath_node(%q(div/div[@class="yt-lockup-content"]/h3/a)) + if title + video["title"] = title.content + else + title = item.xpath_node(%q(div[@class="yt-lockup-content"]/h3/a)) + if title + video["title"] = title.content + end + end + + videos << video end POOL << client diff --git a/src/views/index.ecr b/src/views/index.ecr index e69de29b..483a0c51 100644 --- a/src/views/index.ecr +++ b/src/views/index.ecr @@ -0,0 +1,2 @@ +<% content_for "header" do %> +<% end %> \ No newline at end of file diff --git a/src/views/search.ecr b/src/views/search.ecr index c48ac3e5..45eb4caf 100644 --- a/src/views/search.ecr +++ b/src/views/search.ecr @@ -1,17 +1,9 @@ <% content_for "header" do %> - <%= query.size > 30 ? query[0,30] + "... - " : query %> + <%= query.size > 30 ? query[0,30].chomp("...") + "... - " : query + " - " %> <% end %> -<% videos_list.each do |video| %> -
-
- - "> - -
-
- - <%= video.info["title"] %> - -
-
-<% end %> \ No newline at end of file + +<% videos.each do |item| %> +

"><%= item["title"] %>

+<% end %> + +

Next page

\ No newline at end of file diff --git a/src/views/watch.ecr b/src/views/watch.ecr index 6766ddfa..7a9e2efc 100644 --- a/src/views/watch.ecr +++ b/src/views/watch.ecr @@ -1,8 +1,9 @@ <% content_for "header" do %> <%= video.info["title"] + " - " %> <% end %> -