iv-org-mirror/invidious-copy-2022-04-11
1
0
Fork 0
mirror of https://gitea.invidious.io/iv-org/invidious-copy-2022-04-11.git synced 2024-08-15 00:43:26 +00:00
Code Issues Projects Releases Packages Wiki Activity
invidious-copy-2022-04-11/src/invidious/videos.cr

679 lines
26 KiB
Crystal
Raw Normal View History

Convert array constants to tuples
2018-08-13 01:04:13 +00:00
CAPTION_LANGUAGES = {
Add preferred captions
2018-08-06 18:23:36 +00:00
"",
"English",
"English (auto-generated)",
"Afrikaans",
"Albanian",
"Amharic",
"Arabic",
"Armenian",
"Azerbaijani",
"Bangla",
"Basque",
"Belarusian",
"Bosnian",
"Bulgarian",
"Burmese",
"Catalan",
"Cebuano",
"Chinese (Simplified)",
"Chinese (Traditional)",
"Corsican",
"Croatian",
"Czech",
"Danish",
"Dutch",
"Esperanto",
"Estonian",
"Filipino",
"Finnish",
"French",
"Galician",
"Georgian",
"German",
"Greek",
"Gujarati",
"Haitian Creole",
"Hausa",
"Hawaiian",
"Hebrew",
"Hindi",
"Hmong",
"Hungarian",
"Icelandic",
"Igbo",
"Indonesian",
"Irish",
"Italian",
"Japanese",
"Javanese",
"Kannada",
"Kazakh",
"Khmer",
"Korean",
"Kurdish",
"Kyrgyz",
"Lao",
"Latin",
"Latvian",
"Lithuanian",
"Luxembourgish",
"Macedonian",
"Malagasy",
"Malay",
"Malayalam",
"Maltese",
"Maori",
"Marathi",
"Mongolian",
"Nepali",
"Norwegian",
"Nyanja",
"Pashto",
"Persian",
"Polish",
"Portuguese",
"Punjabi",
"Romanian",
"Russian",
"Samoan",
"Scottish Gaelic",
"Serbian",
"Shona",
"Sindhi",
"Sinhala",
"Slovak",
"Slovenian",
"Somali",
"Southern Sotho",
"Spanish",
Add JSON mapping for captions
2018-08-06 23:25:25 +00:00
"Spanish (Latin America)",
Add preferred captions
2018-08-06 18:23:36 +00:00
"Sundanese",
"Swahili",
"Swedish",
"Tajik",
"Tamil",
"Telugu",
"Thai",
"Turkish",
"Ukrainian",
"Urdu",
"Uzbek",
"Vietnamese",
"Welsh",
"Western Frisian",
"Xhosa",
"Yiddish",
"Yoruba",
"Zulu",
Convert array constants to tuples
2018-08-13 01:04:13 +00:00
}
Add preferred captions
2018-08-06 18:23:36 +00:00
Add geo-bypass for video info
2018-08-13 14:17:28 +00:00
REGIONS = {"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW"}
BYPASS_REGIONS = {"CA", "DE", "FR", "JP", "RU", "UK"}
Convert array constants to tuples
2018-08-13 01:04:13 +00:00
VIDEO_THUMBNAILS = {
Proxy thumbnails
2018-09-15 02:24:28 +00:00
{name: "maxresdefault", url: "maxresdefault", height: 720, width: 1280},
Add 'maxresdefault', 'sddefault', to thumbnails
2018-08-12 14:46:47 +00:00
{name: "sddefault", url: "sddefault", height: 480, width: 640},
Proxy thumbnails
2018-09-15 02:24:28 +00:00
{name: "high", url: "hqdefault", height: 360, width: 480},
{name: "medium", url: "mqdefault", height: 180, width: 320},
{name: "default", url: "default", height: 90, width: 120},
Add 'maxresdefault', 'sddefault', to thumbnails
2018-08-12 14:46:47 +00:00
{name: "start", url: "1", height: 90, width: 120},
{name: "middle", url: "2", height: 90, width: 120},
{name: "end", url: "3", height: 90, width: 120},
Convert array constants to tuples
2018-08-13 01:04:13 +00:00
}
Add 'maxresdefault', 'sddefault', to thumbnails
2018-08-12 14:46:47 +00:00
Make 'video_formats' constant
2018-08-12 14:34:26 +00:00
# See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476
VIDEO_FORMATS = {
"5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
"6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
"13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"},
"17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"},
"18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"},
"22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
"34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"},
"37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
"38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
"43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
"44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
"45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
"46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
"59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
# 3D videos
"82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
"85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
"100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
"101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
"102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
# Apple HTTP Live Streaming
"91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
"92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
"93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
"96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
"132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
"151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"},
# DASH mp4 video
"133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"},
"134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"},
"135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
"136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"},
"137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"},
"138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https=>//github.com/rg3/youtube-dl/issues/4559)
"160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"},
"212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
"264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"},
"298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
"299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
"266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"},
# Dash mp4 audio
"139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"},
"140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"},
"141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"},
"256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
"258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
"325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"},
"328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"},
# Dash webm
"167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
"168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
"169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
"170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
"218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
"219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
"278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"},
"242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"},
"243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"},
"244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
"245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
"246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
"247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"},
"248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"},
"271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"},
# itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
"272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
"302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
"315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
# Dash webm audio
"171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128},
"172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256},
# Dash webm audio with opus inside
"249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50},
"250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70},
"251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160},
}
Split helpers.cr into multiple files
2018-08-04 20:30:44 +00:00
class Video
Add note for livestreams that haven't started
2018-08-18 16:47:16 +00:00
property player_json : JSON::Any?
Split helpers.cr into multiple files
2018-08-04 20:30:44 +00:00
module HTTPParamConverter
def self.from_rs(rs)
HTTP::Params.parse(rs.read(String))
end
end
Major cleanup
2018-08-05 04:07:38 +00:00
def fmt_stream(decrypt_function)
streams = [] of HTTP::Params
self.info["url_encoded_fmt_stream_map"].split(",") do |string|
if !string.empty?
streams << HTTP::Params.parse(string)
end
end
streams.each { |s| s.add("label", "#{s["quality"]} - #{s["type"].split(";")[0].split("/")[1]}") }
streams = streams.uniq { |s| s["label"] }
if streams[0]? && streams[0]["s"]?
streams.each do |fmt|
fmt["url"] += "&signature=" + decrypt_signature(fmt["s"], decrypt_function)
end
end
return streams
end
def adaptive_fmts(decrypt_function)
adaptive_fmts = [] of HTTP::Params
if self.info.has_key?("adaptive_fmts")
self.info["adaptive_fmts"].split(",") do |string|
adaptive_fmts << HTTP::Params.parse(string)
end
Add support for dashmpd within video info
2018-09-13 03:31:47 +00:00
elsif self.info.has_key?("dashmpd")
client = make_client(YT_URL)
response = client.get(self.info["dashmpd"])
document = XML.parse_html(response.body)
document.xpath_nodes(%q(//adaptationset)).each do |adaptation_set|
mime_type = adaptation_set["mimetype"]
document.xpath_nodes(%q(.//representation)).each do |representation|
codecs = representation["codecs"]
itag = representation["id"]
bandwidth = representation["bandwidth"]
url = representation.xpath_node(%q(.//baseurl)).not_nil!.content
clen = url.match(/clen\/(?<clen>\d+)/).try &.["clen"]
clen ||= "0"
Fix dash parsing for video info
2018-09-15 13:56:47 +00:00
lmt = url.match(/lmt\/(?<lmt>\d+)/).try &.["lmt"]
lmt ||= "#{((Time.now + 1.hour).epoch_f.to_f64 * 1000000).to_i64}"
Add support for dashmpd within video info
2018-09-13 03:31:47 +00:00
segment_list = representation.xpath_node(%q(.//segmentlist)).not_nil!
Fix dash parsing for video info
2018-09-15 13:56:47 +00:00
init = segment_list.xpath_node(%q(.//initialization))
Add support for dashmpd within video info
2018-09-13 03:31:47 +00:00
# TODO: Replace with sane defaults when byteranges are absent
Fix dash parsing for video info
2018-09-15 13:56:47 +00:00
if init
init = init["sourceurl"].lchop("range/")
index = segment_list.xpath_node(%q(.//segmenturl)).not_nil!["media"]
Add support for dashmpd within video info
2018-09-13 03:31:47 +00:00
index = index.lchop("range/")
index = "#{init.split("-")[1].to_i + 1}-#{index.split("-")[0].to_i}"
Fix dash parsing for video info
2018-09-15 13:56:47 +00:00
else
init = "0-0"
index = "1-1"
Add support for dashmpd within video info
2018-09-13 03:31:47 +00:00
end
params = {
"type" => ["#{mime_type}; codecs=\"#{codecs}\""],
"url" => [url],
"projection_type" => ["1"],
"index" => [index],
"init" => [init],
"xtags" => [] of String,
"lmt" => [lmt],
"clen" => [clen],
"bitrate" => [bandwidth],
"itag" => [itag],
}
if mime_type == "video/mp4"
width = representation["width"]?
height = representation["height"]?
fps = representation["framerate"]?
metadata = itag_to_metadata?(itag)
if metadata
width ||= metadata["width"]?
height ||= metadata["height"]?
fps ||= metadata["fps"]?
end
if width && height
params["size"] = ["#{width}x#{height}"]
end
if width
params["quality_label"] = ["#{height}p"]
end
end
adaptive_fmts << HTTP::Params.new(params)
end
end
Major cleanup
2018-08-05 04:07:38 +00:00
end
if adaptive_fmts[0]? && adaptive_fmts[0]["s"]?
adaptive_fmts.each do |fmt|
fmt["url"] += "&signature=" + decrypt_signature(fmt["s"], decrypt_function)
end
end
return adaptive_fmts
end
Add '/videoplayback' redirect
2018-08-07 16:39:56 +00:00
def video_streams(adaptive_fmts)
video_streams = adaptive_fmts.compact_map { |s| s["type"].starts_with?("video") ? s : nil }
return video_streams
end
Major cleanup
2018-08-05 04:07:38 +00:00
def audio_streams(adaptive_fmts)
audio_streams = adaptive_fmts.compact_map { |s| s["type"].starts_with?("audio") ? s : nil }
audio_streams.sort_by! { |s| s["bitrate"].to_i }.reverse!
audio_streams.each do |stream|
stream["bitrate"] = (stream["bitrate"].to_f64/1000).to_i.to_s
end
return audio_streams
end
Add note for livestreams that haven't started
2018-08-18 16:47:16 +00:00
def player_response
if !@player_json
@player_json = JSON.parse(@info["player_response"])
end
Major cleanup
2018-08-05 04:07:38 +00:00
Add note for livestreams that haven't started
2018-08-18 16:47:16 +00:00
return @player_json.not_nil!
end
def captions
Add JSON mapping for captions
2018-08-06 23:25:25 +00:00
captions = [] of Caption
Major cleanup
2018-08-05 04:07:38 +00:00
if player_response["captions"]?
Fix for videos that don't have 'captionTracks'
2018-08-07 01:03:10 +00:00
caption_list = player_response["captions"]["playerCaptionsTracklistRenderer"]["captionTracks"]?.try &.as_a
caption_list ||= [] of JSON::Any
Add JSON mapping for captions
2018-08-06 23:25:25 +00:00
caption_list.each do |caption|
caption = Caption.from_json(caption.to_json)
caption.name.simpleText = caption.name.simpleText.split(" - ")[0]
captions << caption
end
Major cleanup
2018-08-05 04:07:38 +00:00
end
return captions
end
def short_description
description = self.description.gsub("<br>", " ")
description = description.gsub("<br/>", " ")
description = XML.parse_html(description).content[0..200].gsub('"', "&quot;").gsub("\n", " ").strip(" ")
if description.empty?
description = " "
end
return description
end
Split helpers.cr into multiple files
2018-08-04 20:30:44 +00:00
add_mapping({
id: String,
info: {
type: HTTP::Params,
default: HTTP::Params.parse(""),
converter: Video::HTTPParamConverter,
},
updated: Time,
title: String,
views: Int64,
likes: Int32,
dislikes: Int32,
wilson_score: Float64,
published: Time,
description: String,
language: String?,
author: String,
ucid: String,
allowed_regions: Array(String),
is_family_friendly: Bool,
genre: String,
Remove migration point
2018-09-09 19:34:16 +00:00
genre_url: String,
Add 'license'
2018-09-09 19:47:26 +00:00
license: {
type: String,
default: "",
},
Split helpers.cr into multiple files
2018-08-04 20:30:44 +00:00
})
end
Add JSON mapping for captions
2018-08-06 23:25:25 +00:00
class Caption
JSON.mapping(
name: CaptionName,
baseUrl: String,
languageCode: String
)
end
class CaptionName
JSON.mapping(
simpleText: String,
)
end
Split helpers.cr into multiple files
2018-08-04 20:30:44 +00:00
def get_video(id, db, refresh = true)
if db.query_one?("SELECT EXISTS (SELECT true FROM videos WHERE id = $1)", id, as: Bool)
video = db.query_one("SELECT * FROM videos WHERE id = $1", id, as: Video)
Shrink size between video requests
2018-09-09 19:41:29 +00:00
# If record was last updated over 10 minutes ago, refresh (expire param in response lasts for 6 hours)
if refresh && Time.now - video.updated > 10.minutes
Split helpers.cr into multiple files
2018-08-04 20:30:44 +00:00
begin
video = fetch_video(id)
video_array = video.to_a
Add genre URLs
2018-09-04 14:50:19 +00:00
Split helpers.cr into multiple files
2018-08-04 20:30:44 +00:00
args = arg_array(video_array[1..-1], 2)
db.exec("UPDATE videos SET (info,updated,title,views,likes,dislikes,wilson_score,\
Update SQL and remove migration points
2018-09-04 14:57:40 +00:00
published,description,language,author,ucid, allowed_regions, is_family_friendly,\
Remove migration points
2018-09-10 02:58:22 +00:00
genre, genre_url, license)\
Split helpers.cr into multiple files
2018-08-04 20:30:44 +00:00
= (#{args}) WHERE id = $1", video_array)
rescue ex
db.exec("DELETE FROM videos * WHERE id = $1", id)
raise ex
end
end
else
video = fetch_video(id)
video_array = video.to_a
Add genre URLs
2018-09-04 14:50:19 +00:00
Split helpers.cr into multiple files
2018-08-04 20:30:44 +00:00
args = arg_array(video_array)
db.exec("INSERT INTO videos VALUES (#{args}) ON CONFLICT (id) DO NOTHING", video_array)
end
return video
end
def fetch_video(id)
html_channel = Channel(XML::Node).new
info_channel = Channel(HTTP::Params).new
spawn do
client = make_client(YT_URL)
html = client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&disable_polymer=1")
html = XML.parse_html(html.body)
html_channel.send(html)
end
spawn do
client = make_client(YT_URL)
info = client.get("/get_video_info?video_id=#{id}&el=detailpage&ps=default&eurl=&gl=US&hl=en&disable_polymer=1")
info = HTTP::Params.parse(info.body)
if info["reason"]?
info = client.get("/get_video_info?video_id=#{id}&ps=default&eurl=&gl=US&hl=en&disable_polymer=1")
info = HTTP::Params.parse(info.body)
end
info_channel.send(info)
end
html = html_channel.receive
info = info_channel.receive
Add geo-bypass for video info
2018-08-13 14:17:28 +00:00
if info["reason"]? && info["reason"].includes? "your country"
bypass_channel = Channel({HTTP::Params | Nil, XML::Node | Nil}).new
BYPASS_REGIONS.each do |country_code|
spawn do
begin
proxies = get_proxies(country_code)
# Try not to overload single proxy
proxy = proxies[0, 5].sample(1)[0]
proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
client = HTTPClient.new(URI.parse("https://www.youtube.com"))
client.read_timeout = 10.seconds
client.connect_timeout = 10.seconds
client.set_proxy(proxy)
proxy_info = client.get("/get_video_info?video_id=#{id}&el=detailpage&ps=default&eurl=&gl=US&hl=en&disable_polymer=1")
proxy_info = HTTP::Params.parse(proxy_info.body)
if !proxy_info["reason"]?
proxy_html = client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1")
proxy_html = XML.parse_html(proxy_html.body)
bypass_channel.send({proxy_info, proxy_html})
else
bypass_channel.send({nil, nil})
end
rescue ex
bypass_channel.send({nil, nil})
end
end
end
BYPASS_REGIONS.size.times do
response = bypass_channel.receive
if response[0] || response[1]
info = response[0].not_nil!
html = response[1].not_nil!
end
end
end
Split helpers.cr into multiple files
2018-08-04 20:30:44 +00:00
if info["reason"]?
raise info["reason"]
end
title = info["title"]
views = info["view_count"].to_i64
author = info["author"]
ucid = info["ucid"]
likes = html.xpath_node(%q(//button[@title="I like this"]/span))
likes = likes.try &.content.delete(",").try &.to_i
likes ||= 0
dislikes = html.xpath_node(%q(//button[@title="I dislike this"]/span))
dislikes = dislikes.try &.content.delete(",").try &.to_i
dislikes ||= 0
description = html.xpath_node(%q(//p[@id="eow-description"]))
description = description ? description.to_xml : ""
wilson_score = ci_lower_bound(likes, likes + dislikes)
published = html.xpath_node(%q(//meta[@itemprop="datePublished"])).not_nil!["content"]
published = Time.parse(published, "%Y-%m-%d", Time::Location.local)
allowed_regions = html.xpath_node(%q(//meta[@itemprop="regionsAllowed"])).not_nil!["content"].split(",")
is_family_friendly = html.xpath_node(%q(//meta[@itemprop="isFamilyFriendly"])).not_nil!["content"] == "True"
Add genre URLs
2018-09-04 14:50:19 +00:00
Split helpers.cr into multiple files
2018-08-04 20:30:44 +00:00
genre = html.xpath_node(%q(//meta[@itemprop="genre"])).not_nil!["content"]
Add genre URLs
2018-09-04 14:50:19 +00:00
genre_url = html.xpath_node(%(//a[text()="#{genre}"])).not_nil!["href"]
Split helpers.cr into multiple files
2018-08-04 20:30:44 +00:00
Add 'license'
2018-09-09 19:47:26 +00:00
license = html.xpath_node(%q(//h4[contains(text(),"License")]/parent::*/ul/li))
if license
license = license.content
else
license ||= ""
end
Split helpers.cr into multiple files
2018-08-04 20:30:44 +00:00
video = Video.new(id, info, Time.now, title, views, likes, dislikes, wilson_score, published, description,
Add 'license'
2018-09-09 19:47:26 +00:00
nil, author, ucid, allowed_regions, is_family_friendly, genre, genre_url, license)
Split helpers.cr into multiple files
2018-08-04 20:30:44 +00:00
return video
end
Add 'fps' to '/videos/:id' endpoint
2018-08-12 14:24:59 +00:00
def itag_to_metadata?(itag : String)
Make 'video_formats' constant
2018-08-12 14:34:26 +00:00
return VIDEO_FORMATS[itag]?
Split helpers.cr into multiple files
2018-08-04 20:30:44 +00:00
end
Major cleanup
2018-08-05 04:07:38 +00:00
def process_video_params(query, preferences)
autoplay = query["autoplay"]?.try &.to_i?
Add support for preferences as query params
2018-08-26 01:05:51 +00:00
preferred_captions = query["subtitles"]?.try &.split(",").map { |a| a.downcase }
quality = query["quality"]?
speed = query["speed"]?.try &.to_f?
Major cleanup
2018-08-05 04:07:38 +00:00
video_loop = query["loop"]?.try &.to_i?
Add support for preferences as query params
2018-08-26 01:05:51 +00:00
volume = query["volume"]?.try &.to_i?
Major cleanup
2018-08-05 04:07:38 +00:00
if preferences
autoplay ||= preferences.autoplay.to_unsafe
Add support for preferences as query params
2018-08-26 01:05:51 +00:00
preferred_captions ||= preferences.captions
quality ||= preferences.quality
speed ||= preferences.speed
Major cleanup
2018-08-05 04:07:38 +00:00
video_loop ||= preferences.video_loop.to_unsafe
Add support for preferences as query params
2018-08-26 01:05:51 +00:00
volume ||= preferences.volume
Major cleanup
2018-08-05 04:07:38 +00:00
end
Add support for preferences as query params
2018-08-26 01:05:51 +00:00
autoplay ||= 0
preferred_captions ||= [] of String
quality ||= "hd720"
speed ||= 1
Major cleanup
2018-08-05 04:07:38 +00:00
video_loop ||= 0
Add support for preferences as query params
2018-08-26 01:05:51 +00:00
volume ||= 100
autoplay = autoplay == 1
Major cleanup
2018-08-05 04:07:38 +00:00
video_loop = video_loop == 1
if query["t"]?
video_start = decode_time(query["t"])
end
video_start ||= 0
Fix typo
2018-08-31 01:25:43 +00:00
if query["time_continue"]?
Fix typo in video params
2018-08-31 02:04:41 +00:00
video_start = decode_time(query["time_continue"])
Add 'time_continue'
2018-08-05 17:35:33 +00:00
end
video_start ||= 0
if query["start"]?
video_start = decode_time(query["start"])
end
Major cleanup
2018-08-05 04:07:38 +00:00
if query["end"]?
video_end = decode_time(query["end"])
end
video_end ||= -1
if query["listen"]? && (query["listen"] == "true" || query["listen"] == "1")
listen = true
end
listen ||= false
raw = query["raw"]?.try &.to_i?
raw ||= 0
raw = raw == 1
controls = query["controls"]?.try &.to_i?
controls ||= 1
controls = controls == 1
Add support for preferences as query params
2018-08-26 01:05:51 +00:00
params = {
autoplay: autoplay,
controls: controls,
listen: listen,
preferred_captions: preferred_captions,
quality: quality,
raw: raw,
speed: speed,
video_end: video_end,
video_loop: video_loop,
video_start: video_start,
volume: volume,
}
return params
Major cleanup
2018-08-05 04:07:38 +00:00
end
Pull thumbnails into seperate function
2018-08-10 13:50:25 +00:00
def generate_thumbnails(json, id)
json.array do
Add 'maxresdefault', 'sddefault', to thumbnails
2018-08-12 14:46:47 +00:00
VIDEO_THUMBNAILS.each do |thumbnail|
Pull thumbnails into seperate function
2018-08-10 13:50:25 +00:00
json.object do
Add 'maxresdefault', 'sddefault', to thumbnails
2018-08-12 14:46:47 +00:00
json.field "quality", thumbnail[:name]
json.field "url", "https://i.ytimg.com/vi/#{id}/#{thumbnail["url"]}.jpg"
json.field "width", thumbnail[:width]
json.field "height", thumbnail[:height]
Pull thumbnails into seperate function
2018-08-10 13:50:25 +00:00
end
end
end
end
Reference in a new issue Copy permalink