From 7828cd97672d068e5d4f255aeea676d755e27c0f Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Mon, 26 Feb 2018 18:58:45 -0600 Subject: [PATCH] Remove html from DB --- src/helpers.cr | 22 +++++++++++----------- src/invidious.cr | 3 --- src/views/watch.ecr | 2 +- videos.sql | 2 +- 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/helpers.cr b/src/helpers.cr index d3c89e03..30e544e5 100644 --- a/src/helpers.cr +++ b/src/helpers.cr @@ -11,10 +11,9 @@ class Video end end - def initialize(id, info, html, updated, title, views, likes, dislikes, wilson_score, published) + def initialize(id, info, updated, title, views, likes, dislikes, wilson_score, published, description) @id = id @info = info - @html = html @updated = updated @title = title @views = views @@ -22,10 +21,11 @@ class Video @dislikes = dislikes @wilson_score = wilson_score @published = published + @description = description end def to_a - return [@id, @info, @html, @updated, @title, @views, @likes, @dislikes, @wilson_score, @published] + return [@id, @info, @updated, @title, @views, @likes, @dislikes, @wilson_score, @published, @description] end DB.mapping({ @@ -35,11 +35,6 @@ class Video default: HTTP::Params.parse(""), converter: Video::HTTPParamConverter, }, - html: { - type: XML::Node, - default: XML.parse_html(""), - converter: Video::XMLConverter, - }, updated: Time, title: String, views: Int64, @@ -47,6 +42,7 @@ class Video dislikes: Int32, wilson_score: Float64, published: Time, + description: String, }) end @@ -105,6 +101,9 @@ def fetch_video(id, client) dislikes = html.xpath_node(%q(//button[@title="I dislike this"]/span)) dislikes = dislikes ? dislikes.content.delete(",").to_i : 0 + description = html.xpath_node(%q(//p[@id="eow-description"])) + description = description ? description.to_xml : "" + wilson_score = ci_lower_bound(likes, likes + dislikes) published = html.xpath_node(%q(//strong[contains(@class,"watch-time-text")])) @@ -134,7 +133,7 @@ def fetch_video(id, client) end end - video = Video.new(id, info, html, Time.now, title, views, likes, dislikes, wilson_score, published) + video = Video.new(id, info, Time.now, title, views, likes, dislikes, wilson_score, published, description) return video end @@ -146,8 +145,9 @@ def get_video(id, client, db, refresh = true) # If record was last updated over an hour ago, refresh (expire param in response lasts for 6 hours) if refresh && Time.now - video.updated > 1.hours video = fetch_video(id, client) - db.exec("UPDATE videos SET info = $2, html = $3, updated = $4,\ - title = $5, views = $6, likes = $7, dislikes = $8, wilson_score = $9, published = $10 WHERE id = $1", video.to_a) + db.exec("UPDATE videos SET info = $2, updated = $3,\ + title = $4, views = $5, likes = $6, dislikes = $7, wilson_score = $8,\ + published = $9, description = $10 WHERE id = $1", video.to_a) end else video = fetch_video(id, client) diff --git a/src/invidious.cr b/src/invidious.cr index ae9e0978..3a0c91e1 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -211,9 +211,6 @@ get "/watch" do |env| player_response = JSON.parse(video.info["player_response"]) - description = video.html.xpath_node(%q(//p[@id="eow-description"])) - description = description ? description.to_xml : "Could not load description" - rating = video.info["avg_rating"].to_f64 engagement = ((video.dislikes.to_f + video.likes.to_f)/video.views * 100) diff --git a/src/views/watch.ecr b/src/views/watch.ecr index 4d428f6e..ee2b0345 100644 --- a/src/views/watch.ecr +++ b/src/views/watch.ecr @@ -92,7 +92,7 @@ var player = videojs('player', options, function() {

- <%= description %> + <%= video.description %>
diff --git a/videos.sql b/videos.sql index 9a9b0782..9a434096 100644 --- a/videos.sql +++ b/videos.sql @@ -4,7 +4,6 @@ CREATE TABLE public.videos ( id text COLLATE pg_catalog."default" NOT NULL, info text COLLATE pg_catalog."default", - html text COLLATE pg_catalog."default", updated timestamp with time zone, title text COLLATE pg_catalog."default", views bigint, @@ -12,6 +11,7 @@ CREATE TABLE public.videos dislikes integer, wilson_score double precision, published timestamp with time zone, + description text COLLATE pg_catalog."default", CONSTRAINT videos_pkey PRIMARY KEY (id) ) WITH (