From 0fba601e91b6c02d456c5a0710d445d7dfcebfe0 Mon Sep 17 00:00:00 2001
From: Omar Roth <omarroth@hotmail.com>
Date: Thu, 26 Jul 2018 08:41:23 -0500
Subject: [PATCH] Fix crash in video crawler

---
 src/invidious.cr         | 2 +-
 src/invidious/helpers.cr | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/invidious.cr b/src/invidious.cr
index 72259f87..2cc34d61 100644
--- a/src/invidious.cr
+++ b/src/invidious.cr
@@ -86,6 +86,7 @@ crawl_threads.times do
     end
 
     loop do
+      client = make_client(YT_URL)
       if ids.empty?
         search(random.base64(3), client) do |id|
           ids << id
@@ -97,7 +98,6 @@ crawl_threads.times do
         video = get_video(id, client, PG_DB)
       rescue ex
         STDOUT << id << " : " << ex.message << "\n"
-        client = make_client(YT_URL)
         next
       ensure
         ids.delete(id)
diff --git a/src/invidious/helpers.cr b/src/invidious/helpers.cr
index 71b462d6..3448245b 100644
--- a/src/invidious/helpers.cr
+++ b/src/invidious/helpers.cr
@@ -307,9 +307,8 @@ def get_video(id, client, db, refresh = true)
   return video
 end
 
-def search(query, client)
-  html = client.get("https://www.youtube.com/results?q=#{query}&sp=EgIQAVAU&disable_polymer=1").body
-
+def search(query, client, &block)
+  html = client.get("/results?q=#{query}&sp=EgIQAVAU&disable_polymer=1").body
   html = XML.parse_html(html)
 
   html.xpath_nodes(%q(//ol[@class="item-section"]/li)).each do |item|