mirror of
https://gitea.invidious.io/iv-org/invidious-copy-2022-04-11.git
synced 2024-08-15 00:43:26 +00:00
Add wait time option and add handling for reddit_pool
This commit is contained in:
parent
84c8fb9b99
commit
323d27a5f6
2 changed files with 20 additions and 4 deletions
|
@ -252,8 +252,8 @@ end
|
|||
|
||||
def make_client(url, context)
|
||||
client = HTTP::Client.new(url, context)
|
||||
client.read_timeout = 10.seconds
|
||||
client.connect_timeout = 10.seconds
|
||||
client.read_timeout = 30.seconds
|
||||
client.connect_timeout = 30.seconds
|
||||
return client
|
||||
end
|
||||
|
||||
|
|
|
@ -21,7 +21,7 @@ require "xml"
|
|||
require "./helpers"
|
||||
|
||||
threads = 10
|
||||
|
||||
sleep_time = 1.0
|
||||
Kemal.config.extra_options do |parser|
|
||||
parser.banner = "Usage: invidious [arguments]"
|
||||
parser.on("-t THREADS", "--threads=THREADS", "Number of threads for crawling (default: 10)") do |number|
|
||||
|
@ -32,6 +32,14 @@ Kemal.config.extra_options do |parser|
|
|||
exit
|
||||
end
|
||||
end
|
||||
parser.on("-w TIME", "--wait=TIME", "Time to wait between server requests in seconds (default: 1 second)") do |number|
|
||||
begin
|
||||
sleep_time = number.to_f64
|
||||
rescue ex
|
||||
puts "TIME must be integer or float"
|
||||
exit
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Kemal::CLI.new
|
||||
|
@ -108,6 +116,8 @@ threads.times do
|
|||
end
|
||||
end
|
||||
|
||||
sleep sleep_time.seconds
|
||||
|
||||
youtube_pool << client
|
||||
end
|
||||
end
|
||||
|
@ -118,8 +128,14 @@ threads.times do
|
|||
loop do
|
||||
client = get_client(reddit_pool)
|
||||
|
||||
begin
|
||||
client.get("/")
|
||||
sleep 10.seconds
|
||||
rescue ex
|
||||
reddit_pool << make_client(URI.parse("https://api.reddit.com"), CONTEXT)
|
||||
next
|
||||
end
|
||||
|
||||
sleep sleep_time.seconds
|
||||
|
||||
reddit_pool << client
|
||||
end
|
||||
|
|
Loading…
Reference in a new issue