Attempt to bypass channel region locks

This commit is contained in:
Omar Roth 2018-10-14 09:53:40 -05:00
parent 245d0b571f
commit 95c6747a3e
3 changed files with 52 additions and 16 deletions

View file

@ -1487,7 +1487,7 @@ get "/feed/channel/:ucid" do |env|
end end
page = 1 page = 1
videos, count = get_60_videos(ucid, page, auto_generated) videos, count = get_60_videos(ucid, page, auto_generated, proxies)
host_url = make_host_url(Kemal.config.ssl || CONFIG.https_only, env.request.headers["Host"]?) host_url = make_host_url(Kemal.config.ssl || CONFIG.https_only, env.request.headers["Host"]?)
path = env.request.path path = env.request.path
@ -1732,7 +1732,7 @@ get "/channel/:ucid" do |env|
end end
end end
videos, count = get_60_videos(ucid, page, auto_generated) videos, count = get_60_videos(ucid, page, auto_generated, proxies)
templated "channel" templated "channel"
end end
@ -1885,7 +1885,6 @@ get "/api/v1/comments/:id" do |env|
proxy_client.read_timeout = 10.seconds proxy_client.read_timeout = 10.seconds
proxy_client.connect_timeout = 10.seconds proxy_client.connect_timeout = 10.seconds
proxy = list.sample(1)[0]
proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
proxy_client.set_proxy(proxy) proxy_client.set_proxy(proxy)
@ -1894,13 +1893,10 @@ get "/api/v1/comments/:id" do |env|
proxy_headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"] proxy_headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"]
proxy_html = response.body proxy_html = response.body
if proxy_html.match(/<meta itemprop="regionsAllowed" content="">/) if !proxy_html.match(/<meta itemprop="regionsAllowed" content="">/)
bypass_channel.send(nil)
else
bypass_channel.send({proxy_html, proxy_client, proxy_headers}) bypass_channel.send({proxy_html, proxy_client, proxy_headers})
end
break break
end
rescue ex rescue ex
end end
end end
@ -2493,7 +2489,7 @@ get "/api/v1/channels/:ucid" do |env|
end end
page = 1 page = 1
videos, count = get_60_videos(ucid, page, auto_generated) videos, count = get_60_videos(ucid, page, auto_generated, proxies)
client = make_client(YT_URL) client = make_client(YT_URL)
channel_html = client.get("/channel/#{ucid}/about?disable_polymer=1").body channel_html = client.get("/channel/#{ucid}/about?disable_polymer=1").body
@ -2631,7 +2627,7 @@ end
halt env, status_code: 404, response: error_message halt env, status_code: 404, response: error_message
end end
videos, count = get_60_videos(ucid, page, auto_generated) videos, count = get_60_videos(ucid, page, auto_generated, proxies)
result = JSON.build do |json| result = JSON.build do |json|
json.array do json.array do

View file

@ -176,7 +176,7 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil)
continuation = Base64.urlsafe_encode(continuation) continuation = Base64.urlsafe_encode(continuation)
continuation = URI.escape(continuation) continuation = URI.escape(continuation)
url = "/browse_ajax?continuation=#{continuation}" url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
return url return url
end end
@ -216,7 +216,7 @@ def get_about_info(ucid)
return {author, ucid, auto_generated, sub_count} return {author, ucid, auto_generated, sub_count}
end end
def get_60_videos(ucid, page, auto_generated) def get_60_videos(ucid, page, auto_generated, proxies)
count = 0 count = 0
videos = [] of SearchVideo videos = [] of SearchVideo
@ -235,6 +235,49 @@ def get_60_videos(ucid, page, auto_generated)
count += 30 count += 30
end end
if !json["load_more_widget_html"]?.try &.as_s.empty? && nodeset.size < 30
bypass_channel = Channel(XML::NodeSet | Nil).new
proxies.each do |region, list|
spawn do
list.each do |proxy|
begin
proxy_client = HTTPClient.new(YT_URL)
proxy_client.read_timeout = 10.seconds
proxy_client.connect_timeout = 10.seconds
proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
proxy_client.set_proxy(proxy)
proxy_response = proxy_client.get(url)
json = JSON.parse(proxy_response.body)
document = XML.parse_html(json["content_html"].as_s)
nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
if nodeset.size == 30
bypass_channel.send(nodeset)
break
end
rescue ex
end
end
if nodeset.size != 30
bypass_channel.send(nil)
end
end
end
proxies.size.times do
response = bypass_channel.receive
if response
nodeset = response
break
end
end
end
if auto_generated if auto_generated
videos += extract_videos(nodeset) videos += extract_videos(nodeset)
else else

View file

@ -573,11 +573,8 @@ def fetch_video(id, proxies)
info = HTTP::Params.parse(client.get("/get_video_info?video_id=#{id}&ps=default&eurl=&gl=US&hl=en&disable_polymer=1").body) info = HTTP::Params.parse(client.get("/get_video_info?video_id=#{id}&ps=default&eurl=&gl=US&hl=en&disable_polymer=1").body)
if !info["reason"]? if !info["reason"]?
bypass_channel.send(proxy) bypass_channel.send(proxy)
else
bypass_channel.send(nil)
end
break break
end
rescue ex rescue ex
end end
end end