mirror of
https://gitea.invidious.io/iv-org/invidious-copy-2023-06-08.git
synced 2024-08-15 00:53:38 +00:00
Try to speed up find_working_proxies
This commit is contained in:
parent
f83e9e6eb9
commit
3830604e42
4 changed files with 51 additions and 41 deletions
|
@ -107,14 +107,14 @@ end
|
|||
|
||||
proxies = {} of String => Array({ip: String, port: Int32})
|
||||
if CONFIG.geo_bypass
|
||||
spawn do
|
||||
find_working_proxies(BYPASS_REGIONS) do |region, list|
|
||||
if !list.empty?
|
||||
proxies[region] = list
|
||||
spawn do
|
||||
find_working_proxies(BYPASS_REGIONS) do |region, list|
|
||||
if !list.empty?
|
||||
proxies[region] = list
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
before_all do |env|
|
||||
env.response.headers["X-XSS-Protection"] = "1; mode=block;"
|
||||
|
@ -1876,6 +1876,8 @@ get "/api/v1/comments/:id" do |env|
|
|||
|
||||
proxies.each do |region, list|
|
||||
spawn do
|
||||
proxy_html = %(<meta itemprop="regionsAllowed" content="">)
|
||||
|
||||
list.each do |proxy|
|
||||
begin
|
||||
proxy_client = HTTPClient.new(YT_URL)
|
||||
|
@ -1886,10 +1888,10 @@ get "/api/v1/comments/:id" do |env|
|
|||
proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
|
||||
proxy_client.set_proxy(proxy)
|
||||
|
||||
proxy_html = proxy_client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1")
|
||||
response = proxy_client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1")
|
||||
proxy_headers = HTTP::Headers.new
|
||||
proxy_headers["cookie"] = proxy_html.cookies.add_request_headers(headers)["cookie"]
|
||||
proxy_html = proxy_html.body
|
||||
proxy_headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"]
|
||||
proxy_html = response.body
|
||||
|
||||
if proxy_html.match(/<meta itemprop="regionsAllowed" content="">/)
|
||||
bypass_channel.send(nil)
|
||||
|
@ -1901,6 +1903,11 @@ get "/api/v1/comments/:id" do |env|
|
|||
rescue ex
|
||||
end
|
||||
end
|
||||
|
||||
# If none of the proxies we tried returned a valid response
|
||||
if proxy_html.match(/<meta itemprop="regionsAllowed" content="">/)
|
||||
bypass_channel.send(nil)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
@ -93,6 +93,25 @@ def get_proxies(country_code = "US")
|
|||
return get_nova_proxies(country_code)
|
||||
end
|
||||
|
||||
def filter_proxies(proxies)
|
||||
proxies.select! do |proxy|
|
||||
begin
|
||||
client = HTTPClient.new(YT_URL)
|
||||
client.read_timeout = 10.seconds
|
||||
client.connect_timeout = 10.seconds
|
||||
|
||||
proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
|
||||
client.set_proxy(proxy)
|
||||
|
||||
client.head("/").status_code == 200
|
||||
rescue ex
|
||||
false
|
||||
end
|
||||
end
|
||||
|
||||
return proxies
|
||||
end
|
||||
|
||||
def get_nova_proxies(country_code = "US")
|
||||
country_code = country_code.downcase
|
||||
client = HTTP::Client.new(URI.parse("https://www.proxynova.com"))
|
||||
|
@ -127,7 +146,7 @@ def get_nova_proxies(country_code = "US")
|
|||
proxies << {ip: ip, port: port, score: score}
|
||||
end
|
||||
|
||||
proxies = proxies.sort_by { |proxy| proxy[:score] }.reverse
|
||||
# proxies = proxies.sort_by { |proxy| proxy[:score] }.reverse
|
||||
return proxies
|
||||
end
|
||||
|
||||
|
|
|
@ -156,39 +156,14 @@ def update_decrypt_function
|
|||
end
|
||||
|
||||
def find_working_proxies(regions)
|
||||
proxy_channel = Channel({String, Array({ip: String, port: Int32})}).new
|
||||
loop do
|
||||
regions.each do |region|
|
||||
proxies = get_proxies(region).first(20)
|
||||
proxies = proxies.map { |proxy| {ip: proxy[:ip], port: proxy[:port]} }
|
||||
# proxies = filter_proxies(proxies)
|
||||
|
||||
regions.each do |region|
|
||||
spawn do
|
||||
loop do
|
||||
begin
|
||||
proxies = get_proxies(region).first(20)
|
||||
rescue ex
|
||||
next proxy_channel.send({region, Array({ip: String, port: Int32}).new})
|
||||
end
|
||||
|
||||
proxies.select! do |proxy|
|
||||
begin
|
||||
client = HTTPClient.new(YT_URL)
|
||||
client.read_timeout = 10.seconds
|
||||
client.connect_timeout = 10.seconds
|
||||
|
||||
proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
|
||||
client.set_proxy(proxy)
|
||||
|
||||
client.get("/").status_code == 200
|
||||
rescue ex
|
||||
false
|
||||
end
|
||||
end
|
||||
proxies = proxies.map { |proxy| {ip: proxy[:ip], port: proxy[:port]} }
|
||||
|
||||
proxy_channel.send({region, proxies})
|
||||
end
|
||||
yield region, proxies
|
||||
Fiber.yield
|
||||
end
|
||||
end
|
||||
|
||||
loop do
|
||||
yield proxy_channel.receive
|
||||
end
|
||||
end
|
||||
|
|
|
@ -543,6 +543,10 @@ def fetch_video(id, proxies)
|
|||
|
||||
proxies.each do |region, list|
|
||||
spawn do
|
||||
info = HTTP::Params.new({
|
||||
"reason" => [info["reason"]],
|
||||
})
|
||||
|
||||
list.each do |proxy|
|
||||
begin
|
||||
client = HTTPClient.new(YT_URL)
|
||||
|
@ -563,6 +567,11 @@ def fetch_video(id, proxies)
|
|||
rescue ex
|
||||
end
|
||||
end
|
||||
|
||||
# If none of the proxies we tried returned a valid response
|
||||
if info["reason"]?
|
||||
bypass_channel.send(nil)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
|
Loading…
Reference in a new issue