Remove html from DB

This commit is contained in:
Omar Roth 2018-02-26 18:58:45 -06:00
parent 13ef4440d0
commit 7828cd9767
4 changed files with 13 additions and 16 deletions

View File

@ -11,10 +11,9 @@ class Video
end end
end end
def initialize(id, info, html, updated, title, views, likes, dislikes, wilson_score, published) def initialize(id, info, updated, title, views, likes, dislikes, wilson_score, published, description)
@id = id @id = id
@info = info @info = info
@html = html
@updated = updated @updated = updated
@title = title @title = title
@views = views @views = views
@ -22,10 +21,11 @@ class Video
@dislikes = dislikes @dislikes = dislikes
@wilson_score = wilson_score @wilson_score = wilson_score
@published = published @published = published
@description = description
end end
def to_a def to_a
return [@id, @info, @html, @updated, @title, @views, @likes, @dislikes, @wilson_score, @published] return [@id, @info, @updated, @title, @views, @likes, @dislikes, @wilson_score, @published, @description]
end end
DB.mapping({ DB.mapping({
@ -35,11 +35,6 @@ class Video
default: HTTP::Params.parse(""), default: HTTP::Params.parse(""),
converter: Video::HTTPParamConverter, converter: Video::HTTPParamConverter,
}, },
html: {
type: XML::Node,
default: XML.parse_html(""),
converter: Video::XMLConverter,
},
updated: Time, updated: Time,
title: String, title: String,
views: Int64, views: Int64,
@ -47,6 +42,7 @@ class Video
dislikes: Int32, dislikes: Int32,
wilson_score: Float64, wilson_score: Float64,
published: Time, published: Time,
description: String,
}) })
end end
@ -105,6 +101,9 @@ def fetch_video(id, client)
dislikes = html.xpath_node(%q(//button[@title="I dislike this"]/span)) dislikes = html.xpath_node(%q(//button[@title="I dislike this"]/span))
dislikes = dislikes ? dislikes.content.delete(",").to_i : 0 dislikes = dislikes ? dislikes.content.delete(",").to_i : 0
description = html.xpath_node(%q(//p[@id="eow-description"]))
description = description ? description.to_xml : ""
wilson_score = ci_lower_bound(likes, likes + dislikes) wilson_score = ci_lower_bound(likes, likes + dislikes)
published = html.xpath_node(%q(//strong[contains(@class,"watch-time-text")])) published = html.xpath_node(%q(//strong[contains(@class,"watch-time-text")]))
@ -134,7 +133,7 @@ def fetch_video(id, client)
end end
end end
video = Video.new(id, info, html, Time.now, title, views, likes, dislikes, wilson_score, published) video = Video.new(id, info, Time.now, title, views, likes, dislikes, wilson_score, published, description)
return video return video
end end
@ -146,8 +145,9 @@ def get_video(id, client, db, refresh = true)
# If record was last updated over an hour ago, refresh (expire param in response lasts for 6 hours) # If record was last updated over an hour ago, refresh (expire param in response lasts for 6 hours)
if refresh && Time.now - video.updated > 1.hours if refresh && Time.now - video.updated > 1.hours
video = fetch_video(id, client) video = fetch_video(id, client)
db.exec("UPDATE videos SET info = $2, html = $3, updated = $4,\ db.exec("UPDATE videos SET info = $2, updated = $3,\
title = $5, views = $6, likes = $7, dislikes = $8, wilson_score = $9, published = $10 WHERE id = $1", video.to_a) title = $4, views = $5, likes = $6, dislikes = $7, wilson_score = $8,\
published = $9, description = $10 WHERE id = $1", video.to_a)
end end
else else
video = fetch_video(id, client) video = fetch_video(id, client)

View File

@ -211,9 +211,6 @@ get "/watch" do |env|
player_response = JSON.parse(video.info["player_response"]) player_response = JSON.parse(video.info["player_response"])
description = video.html.xpath_node(%q(//p[@id="eow-description"]))
description = description ? description.to_xml : "Could not load description"
rating = video.info["avg_rating"].to_f64 rating = video.info["avg_rating"].to_f64
engagement = ((video.dislikes.to_f + video.likes.to_f)/video.views * 100) engagement = ((video.dislikes.to_f + video.likes.to_f)/video.views * 100)

View File

@ -92,7 +92,7 @@ var player = videojs('player', options, function() {
</a> </a>
</p> </p>
<div style="margin-right:1em;"> <div style="margin-right:1em;">
<%= description %> <%= video.description %>
</div> </div>
</div> </div>

View File

@ -4,7 +4,6 @@ CREATE TABLE public.videos
( (
id text COLLATE pg_catalog."default" NOT NULL, id text COLLATE pg_catalog."default" NOT NULL,
info text COLLATE pg_catalog."default", info text COLLATE pg_catalog."default",
html text COLLATE pg_catalog."default",
updated timestamp with time zone, updated timestamp with time zone,
title text COLLATE pg_catalog."default", title text COLLATE pg_catalog."default",
views bigint, views bigint,
@ -12,6 +11,7 @@ CREATE TABLE public.videos
dislikes integer, dislikes integer,
wilson_score double precision, wilson_score double precision,
published timestamp with time zone, published timestamp with time zone,
description text COLLATE pg_catalog."default",
CONSTRAINT videos_pkey PRIMARY KEY (id) CONSTRAINT videos_pkey PRIMARY KEY (id)
) )
WITH ( WITH (