mirror of
https://gitea.invidious.io/iv-org/invidious-copy-2022-04-11.git
synced 2024-08-15 00:43:26 +00:00
Add wait time option and add handling for reddit_pool
This commit is contained in:
parent
84c8fb9b99
commit
323d27a5f6
2 changed files with 20 additions and 4 deletions
|
@ -252,8 +252,8 @@ end
|
||||||
|
|
||||||
def make_client(url, context)
|
def make_client(url, context)
|
||||||
client = HTTP::Client.new(url, context)
|
client = HTTP::Client.new(url, context)
|
||||||
client.read_timeout = 10.seconds
|
client.read_timeout = 30.seconds
|
||||||
client.connect_timeout = 10.seconds
|
client.connect_timeout = 30.seconds
|
||||||
return client
|
return client
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
@ -21,7 +21,7 @@ require "xml"
|
||||||
require "./helpers"
|
require "./helpers"
|
||||||
|
|
||||||
threads = 10
|
threads = 10
|
||||||
|
sleep_time = 1.0
|
||||||
Kemal.config.extra_options do |parser|
|
Kemal.config.extra_options do |parser|
|
||||||
parser.banner = "Usage: invidious [arguments]"
|
parser.banner = "Usage: invidious [arguments]"
|
||||||
parser.on("-t THREADS", "--threads=THREADS", "Number of threads for crawling (default: 10)") do |number|
|
parser.on("-t THREADS", "--threads=THREADS", "Number of threads for crawling (default: 10)") do |number|
|
||||||
|
@ -32,6 +32,14 @@ Kemal.config.extra_options do |parser|
|
||||||
exit
|
exit
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
parser.on("-w TIME", "--wait=TIME", "Time to wait between server requests in seconds (default: 1 second)") do |number|
|
||||||
|
begin
|
||||||
|
sleep_time = number.to_f64
|
||||||
|
rescue ex
|
||||||
|
puts "TIME must be integer or float"
|
||||||
|
exit
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
Kemal::CLI.new
|
Kemal::CLI.new
|
||||||
|
@ -108,6 +116,8 @@ threads.times do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
sleep sleep_time.seconds
|
||||||
|
|
||||||
youtube_pool << client
|
youtube_pool << client
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
@ -118,8 +128,14 @@ threads.times do
|
||||||
loop do
|
loop do
|
||||||
client = get_client(reddit_pool)
|
client = get_client(reddit_pool)
|
||||||
|
|
||||||
|
begin
|
||||||
client.get("/")
|
client.get("/")
|
||||||
sleep 10.seconds
|
rescue ex
|
||||||
|
reddit_pool << make_client(URI.parse("https://api.reddit.com"), CONTEXT)
|
||||||
|
next
|
||||||
|
end
|
||||||
|
|
||||||
|
sleep sleep_time.seconds
|
||||||
|
|
||||||
reddit_pool << client
|
reddit_pool << client
|
||||||
end
|
end
|
||||||
|
|
Loading…
Reference in a new issue