mirror of
https://gitea.invidious.io/iv-org/invidious-copy-2022-04-11.git
synced 2024-08-15 00:43:26 +00:00
Add ability to specify number of threads
This commit is contained in:
parent
1e2139d78c
commit
03aeb27b40
1 changed files with 20 additions and 1 deletions
|
@ -15,10 +15,29 @@
|
|||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
require "kemal"
|
||||
require "option_parser"
|
||||
require "pg"
|
||||
require "xml"
|
||||
require "./helpers"
|
||||
|
||||
threads = 10
|
||||
|
||||
OptionParser.parse! do |parser|
|
||||
parser.banner = "Usage: invidious [arguments]"
|
||||
parser.on("-t THREADS", "--threads=THREADS", "Number of threads for crawling") do |number|
|
||||
begin
|
||||
threads = number.to_i32
|
||||
rescue ex
|
||||
puts "THREADS must be integer"
|
||||
exit
|
||||
end
|
||||
end
|
||||
parser.on("-h", "--help", "Show this help") do
|
||||
puts parser
|
||||
exit
|
||||
end
|
||||
end
|
||||
|
||||
PG_DB = DB.open "postgres://kemal:kemal@localhost:5432/invidious"
|
||||
URL = URI.parse("https://www.youtube.com")
|
||||
CONTEXT = OpenSSL::SSL::Context::Client.new
|
||||
|
@ -36,7 +55,7 @@ POOL = Deque.new(30) do
|
|||
end
|
||||
|
||||
# Refresh pool by crawling YT
|
||||
10.times do
|
||||
threads.times do
|
||||
spawn do
|
||||
io = STDOUT
|
||||
ids = Deque(String).new
|
||||
|
|
Loading…
Reference in a new issue