mirror of
https://gitea.invidious.io/iv-org/invidious-copy-2022-03-16.git
synced 2024-08-15 00:53:18 +00:00
Add ability to specify number of threads
This commit is contained in:
parent
1e2139d78c
commit
03aeb27b40
1 changed files with 20 additions and 1 deletions
|
@ -15,10 +15,29 @@
|
||||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
require "kemal"
|
require "kemal"
|
||||||
|
require "option_parser"
|
||||||
require "pg"
|
require "pg"
|
||||||
require "xml"
|
require "xml"
|
||||||
require "./helpers"
|
require "./helpers"
|
||||||
|
|
||||||
|
threads = 10
|
||||||
|
|
||||||
|
OptionParser.parse! do |parser|
|
||||||
|
parser.banner = "Usage: invidious [arguments]"
|
||||||
|
parser.on("-t THREADS", "--threads=THREADS", "Number of threads for crawling") do |number|
|
||||||
|
begin
|
||||||
|
threads = number.to_i32
|
||||||
|
rescue ex
|
||||||
|
puts "THREADS must be integer"
|
||||||
|
exit
|
||||||
|
end
|
||||||
|
end
|
||||||
|
parser.on("-h", "--help", "Show this help") do
|
||||||
|
puts parser
|
||||||
|
exit
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
PG_DB = DB.open "postgres://kemal:kemal@localhost:5432/invidious"
|
PG_DB = DB.open "postgres://kemal:kemal@localhost:5432/invidious"
|
||||||
URL = URI.parse("https://www.youtube.com")
|
URL = URI.parse("https://www.youtube.com")
|
||||||
CONTEXT = OpenSSL::SSL::Context::Client.new
|
CONTEXT = OpenSSL::SSL::Context::Client.new
|
||||||
|
@ -36,7 +55,7 @@ POOL = Deque.new(30) do
|
||||||
end
|
end
|
||||||
|
|
||||||
# Refresh pool by crawling YT
|
# Refresh pool by crawling YT
|
||||||
10.times do
|
threads.times do
|
||||||
spawn do
|
spawn do
|
||||||
io = STDOUT
|
io = STDOUT
|
||||||
ids = Deque(String).new
|
ids = Deque(String).new
|
||||||
|
|
Loading…
Reference in a new issue