diff --git a/src/invidious.cr b/src/invidious.cr index 72259f87..2cc34d61 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -86,6 +86,7 @@ crawl_threads.times do end loop do + client = make_client(YT_URL) if ids.empty? search(random.base64(3), client) do |id| ids << id @@ -97,7 +98,6 @@ crawl_threads.times do video = get_video(id, client, PG_DB) rescue ex STDOUT << id << " : " << ex.message << "\n" - client = make_client(YT_URL) next ensure ids.delete(id) diff --git a/src/invidious/helpers.cr b/src/invidious/helpers.cr index 71b462d6..3448245b 100644 --- a/src/invidious/helpers.cr +++ b/src/invidious/helpers.cr @@ -307,9 +307,8 @@ def get_video(id, client, db, refresh = true) return video end -def search(query, client) - html = client.get("https://www.youtube.com/results?q=#{query}&sp=EgIQAVAU&disable_polymer=1").body - +def search(query, client, &block) + html = client.get("/results?q=#{query}&sp=EgIQAVAU&disable_polymer=1").body html = XML.parse_html(html) html.xpath_nodes(%q(//ol[@class="item-section"]/li)).each do |item|