Remove "detect_language" from dependencies

This commit is contained in:
Omar Roth 2019-03-03 11:51:28 -06:00
parent 2c9b148627
commit 66605196ad
4 changed files with 5 additions and 52 deletions

View file

@ -9,8 +9,6 @@ targets:
main: src/invidious.cr main: src/invidious.cr
dependencies: dependencies:
detect_language:
github: detectlanguage/detectlanguage-crystal
kemal: kemal:
github: kemalcr/kemal github: kemalcr/kemal
pg: pg:
@ -18,6 +16,6 @@ dependencies:
sqlite3: sqlite3:
github: crystal-lang/crystal-sqlite3 github: crystal-lang/crystal-sqlite3
crystal: 0.27.1 crystal: 0.27.2
license: AGPLv3 license: AGPLv3

View file

@ -14,7 +14,6 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
require "detect_language"
require "digest/md5" require "digest/md5"
require "file_utils" require "file_utils"
require "kemal" require "kemal"

View file

@ -14,8 +14,7 @@ user: String,
full_refresh: Bool, # Used for crawling channels: threads should check all videos uploaded by a channel full_refresh: Bool, # Used for crawling channels: threads should check all videos uploaded by a channel
https_only: Bool?, # Used to tell Invidious it is behind a proxy, so links to resources should be https:// https_only: Bool?, # Used to tell Invidious it is behind a proxy, so links to resources should be https://
hmac_key: String?, # HMAC signing key for CSRF tokens hmac_key: String?, # HMAC signing key for CSRF tokens
domain: String, # Domain to be used for links to resources on the site where an absolute URL is required domain: String?, # Domain to be used for links to resources on the site where an absolute URL is required
dl_api_key: String?, # DetectLanguage API Key (used to filter non-English results from "top" page), mostly non-functional
default_home: {type: String, default: "Top"}, default_home: {type: String, default: "Top"},
feed_menu: {type: Array(String), default: ["Popular", "Top", "Trending"]}, feed_menu: {type: Array(String), default: ["Popular", "Top", "Trending"]},
top_enabled: {type: Bool, default: true}, top_enabled: {type: Bool, default: true},
@ -74,7 +73,7 @@ class DenyFrame < Kemal::Handler
end end
end end
def rank_videos(db, n, filter, url) def rank_videos(db, n)
top = [] of {Float64, String} top = [] of {Float64, String}
db.query("SELECT id, wilson_score, published FROM videos WHERE views > 5000 ORDER BY published DESC LIMIT 1000") do |rs| db.query("SELECT id, wilson_score, published FROM videos WHERE views > 5000 ORDER BY published DESC LIMIT 1000") do |rs|
@ -95,41 +94,7 @@ def rank_videos(db, n, filter, url)
top.reverse! top.reverse!
top = top.map { |a, b| b } top = top.map { |a, b| b }
if filter return top[0..n - 1]
language_list = [] of String
top.each do |id|
if language_list.size == n
break
else
client = make_client(url)
begin
video = get_video(id, db)
rescue ex
next
end
if video.language
language = video.language
else
description = XML.parse(video.description)
content = [video.title, description.content].join(" ")
content = content[0, 10000]
results = DetectLanguage.detect(content)
language = results[0].language
db.exec("UPDATE videos SET language = $1 WHERE id = $2", language, id)
end
if language == "en"
language_list << id
end
end
end
return language_list
else
return top[0..n - 1]
end
end end
def login_req(login_form, f_req) def login_req(login_form, f_req)

View file

@ -154,18 +154,9 @@ def refresh_feeds(db, logger, max_threads = 1)
end end
def pull_top_videos(config, db) def pull_top_videos(config, db)
if config.dl_api_key
DetectLanguage.configure do |dl_config|
dl_config.api_key = config.dl_api_key.not_nil!
end
filter = true
end
filter ||= false
loop do loop do
begin begin
top = rank_videos(db, 40, filter, YT_URL) top = rank_videos(db, 40)
rescue ex rescue ex
next next
end end