From 31a3e1bb3a7f9f753efef68bd76b3a9358b847e9 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Thu, 23 Nov 2017 22:06:43 -0600 Subject: [PATCH] Add view counter, likes/dislikes, rating, and HTML parser --- shard.yml | 2 + src/proxy.cr | 0 src/visor.cr | 110 +++++++++++++++++++++++++++++------------------ views/listen.ecr | 21 +++++---- views/watch.ecr | 29 +++++++------ 5 files changed, 99 insertions(+), 63 deletions(-) delete mode 100644 src/proxy.cr diff --git a/shard.yml b/shard.yml index 5180d51d..dc5cb756 100644 --- a/shard.yml +++ b/shard.yml @@ -11,6 +11,8 @@ targets: dependencies: kemal: github: kemalcr/kemal + pg: + github: will/crystal-pg crystal: 0.23.1 diff --git a/src/proxy.cr b/src/proxy.cr deleted file mode 100644 index e69de29b..00000000 diff --git a/src/visor.cr b/src/visor.cr index 76988b29..77e7e170 100644 --- a/src/visor.cr +++ b/src/visor.cr @@ -1,25 +1,56 @@ -require "kemal" -require "xml" require "http/client" -require "base64" +require "json" +require "kemal" +require "pg" +require "xml" macro templated(filename) render "views/#{{{filename}}}.ecr", "views/layout.ecr" end +# pg = DB.open("postgres://kemal@visor/dev") + +alias Type = String | Hash(String, Type) + +def object_to_hash(value) + object = {} of String => Type + items = value.split("&") + items.each do |item| + key, value = item.split("=") + value = URI.unescape(value) + object[key] = parse_uri(value) + end + return object +end + +def array_to_hash(value) + array = {} of String => Type + items = value.split(",") + count = 0 + items.each do |item| + array[count.to_s] = parse_uri(item) + count += 1 + end + return array +end + +def parse_uri(value) + if value.starts_with?("http") || value.starts_with?("[") + return value + else + if value.includes?(",") + return array_to_hash(value) + elsif value.includes?("&") + return object_to_hash(value) + else + return value + end + end +end + context = OpenSSL::SSL::Context::Client.insecure client = HTTP::Client.new("www.youtube.com", 443, context) -def params_to_hash(params) - pairs = params.split("&") - hash = Hash(String, String).new - pairs.each do |pair| - key, value = pair.split("=") - hash[key] = URI.unescape(value) - end - return hash -end - get "/" do |env| templated "index" end @@ -27,42 +58,37 @@ end get "/watch/:video_id" do |env| video_id = env.params.url["video_id"] - if File.exists?("video_info/#{video_id}") - video_info = JSON.parse(File.open("video_info/#{video_id}")) + video_info_encoded = HTTP::Client.get("https://www.youtube.com/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en", nil, nil, tls = context).body + video_info = object_to_hash(video_info_encoded) + body = client.get("/watch?v=#{video_id}").body + doc = XML.parse(body) + + likes = doc.xpath_node(%q(//button[@title="I like this"]/span)) + if likes + likes = likes.content else - video_info_encoded = HTTP::Client.get("https://www.youtube.com/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en", nil, nil, tls = context).body - video_info = params_to_hash(video_info_encoded) - - File.write("video_info/#{video_id}", video_info.to_json) + likes = "n/a" + end + + dislikes = doc.xpath_node(%q(//button[@title="I dislike this"]/span)) + if dislikes + dislikes.content + else + dislikes = "n/a" end - fmt_stream_map = video_info["url_encoded_fmt_stream_map"].to_s.split(",") - fmt_stream = Array(Hash(String, String)).new - fmt_stream_map.each do |fmt| - fmt_stream << params_to_hash(fmt.to_s) - end - fmt_stream.reverse! + File.write("video_info/#{video_id}", video_info.to_json) templated "watch" end -get "/listen/:video_id" do |env| - video_id = env.params.url["video_id"] +# get "/listen/:video_id" do |env| +# video_id = env.params.url["video_id"] - if File.exists?("video_info/#{video_id}") - video_info = JSON.parse(File.open("video_info/#{video_id}")) - else - video_info_encoded = HTTP::Client.get("https://www.youtube.com/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en", nil, nil, tls = context).body - video_info = params_to_hash(video_info_encoded) - - File.write("video_info/#{video_id}", video_info.to_json) - end - - adaptive_fmt = Array(Hash(String, String)).new - video_info["adaptive_fmts"].to_s.split(",") do |fmt| - adaptive_fmt << params_to_hash(video_info["adaptive_fmts"].to_s) - end - templated "listen" -end +# video_info_encoded = HTTP::Client.get("https://www.youtube.com/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en", nil, nil, tls = context).body +# video_info = object_to_hash(video_info_encoded) +# File.write("video_info/#{video_id}", video_info.to_json) +# templated "listen" +# end public_folder "assets" diff --git a/views/listen.ecr b/views/listen.ecr index 13f03805..c8040f14 100644 --- a/views/listen.ecr +++ b/views/listen.ecr @@ -1,16 +1,19 @@ -

<%= URI.unescape(video_info["title"].to_s,true) %>

-