diff --git a/spec/invidious/helpers_spec.cr b/spec/invidious/helpers_spec.cr index b2436989..5ecebef3 100644 --- a/spec/invidious/helpers_spec.cr +++ b/spec/invidious/helpers_spec.cr @@ -29,20 +29,6 @@ Spectator.describe "Helper" do end end - describe "#produce_search_params" do - it "correctly produces token for searching with specified filters" do - expect(produce_search_params).to eq("CAASAhABSAA%3D") - - expect(produce_search_params(sort: "upload_date", content_type: "video")).to eq("CAISAhABSAA%3D") - - expect(produce_search_params(content_type: "playlist")).to eq("CAASAhADSAA%3D") - - expect(produce_search_params(sort: "date", content_type: "video", features: ["hd", "cc", "purchased", "hdr"])).to eq("CAISCxABIAEwAUgByAEBSAA%3D") - - expect(produce_search_params(content_type: "channel")).to eq("CAASAhACSAA%3D") - end - end - describe "#produce_comment_continuation" do it "correctly produces a continuation token for comments" do expect(produce_comment_continuation("_cE8xSu6swE", "ADSJ_i2qvJeFtL0htmS5_K5Ctj3eGFVBMWL9Wd42o3kmUL6_mAzdLp85-liQZL0mYr_16BhaggUqX652Sv9JqV6VXinShSP-ZT6rL4NolPBaPXVtJsO5_rA_qE3GubAuLFw9uzIIXU2-HnpXbdgPLWTFavfX206hqWmmpHwUOrmxQV_OX6tYkM3ux3rPAKCDrT8eWL7MU3bLiNcnbgkW8o0h8KYLL_8BPa8LcHbTv8pAoNkjerlX1x7K4pqxaXPoyz89qNlnh6rRx6AXgAzzoHH1dmcyQ8CIBeOHg-m4i8ZxdX4dP88XWrIFg-jJGhpGP8JUMDgZgavxVx225hUEYZMyrLGler5em4FgbG62YWC51moLDLeYEA")).to eq("EkMSC19jRTh4U3U2c3dFyAEA4AEBogINKP___________wFAAMICHQgEGhdodHRwczovL3d3dy55b3V0dWJlLmNvbSIAGAYyjAMK9gJBRFNKX2kycXZKZUZ0TDBodG1TNV9LNUN0ajNlR0ZWQk1XTDlXZDQybzNrbVVMNl9tQXpkTHA4NS1saVFaTDBtWXJfMTZCaGFnZ1VxWDY1MlN2OUpxVjZWWGluU2hTUC1aVDZyTDROb2xQQmFQWFZ0SnNPNV9yQV9xRTNHdWJBdUxGdzl1eklJWFUyLUhucFhiZGdQTFdURmF2ZlgyMDZocVdtbXBId1VPcm14UVZfT1g2dFlrTTN1eDNyUEFLQ0RyVDhlV0w3TVUzYkxpTmNuYmdrVzhvMGg4S1lMTF84QlBhOExjSGJUdjhwQW9Oa2plcmxYMXg3SzRwcXhhWFBveXo4OXFObG5oNnJSeDZBWGdBenpvSEgxZG1jeVE4Q0lCZU9IZy1tNGk4WnhkWDRkUDg4WFdySUZnLWpKR2hwR1A4SlVNRGdaZ2F2eFZ4MjI1aFVFWVpNeXJMR2xlcjVlbTRGZ2JHNjJZV0M1MW1vTERMZVlFQSIPIgtfY0U4eFN1NnN3RTAAKBQ%3D") diff --git a/spec/invidious/search/iv_filters_spec.cr b/spec/invidious/search/iv_filters_spec.cr new file mode 100644 index 00000000..b0897a63 --- /dev/null +++ b/spec/invidious/search/iv_filters_spec.cr @@ -0,0 +1,371 @@ +require "../../../src/invidious/search/filters" + +require "http/params" +require "spectator" + +Spectator.configure do |config| + config.fail_blank + config.randomize +end + +FEATURES_TEXT = { + Invidious::Search::Filters::Features::Live => "live", + Invidious::Search::Filters::Features::FourK => "4k", + Invidious::Search::Filters::Features::HD => "hd", + Invidious::Search::Filters::Features::Subtitles => "subtitles", + Invidious::Search::Filters::Features::CCommons => "commons", + Invidious::Search::Filters::Features::ThreeSixty => "360", + Invidious::Search::Filters::Features::VR180 => "vr180", + Invidious::Search::Filters::Features::ThreeD => "3d", + Invidious::Search::Filters::Features::HDR => "hdr", + Invidious::Search::Filters::Features::Location => "location", + Invidious::Search::Filters::Features::Purchased => "purchased", +} + +Spectator.describe Invidious::Search::Filters do + # ------------------- + # Decode (legacy) + # ------------------- + + describe "#from_legacy_filters" do + it "Decodes channel: filter" do + query = "test channel:UC123456 request" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new) + expect(chan).to eq("UC123456") + expect(qury).to eq("test request") + expect(subs).to be_false + end + + it "Decodes user: filter" do + query = "user:LinusTechTips broke something (again)" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new) + expect(chan).to eq("LinusTechTips") + expect(qury).to eq("broke something (again)") + expect(subs).to be_false + end + + it "Decodes type: filter" do + Invidious::Search::Filters::Type.each do |value| + query = "Eiffel 65 - Blue [1 Hour] type:#{value}" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new(type: value)) + expect(chan).to eq("") + expect(qury).to eq("Eiffel 65 - Blue [1 Hour]") + expect(subs).to be_false + end + end + + it "Decodes content_type: filter" do + Invidious::Search::Filters::Type.each do |value| + query = "I like to watch content_type:#{value}" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new(type: value)) + expect(chan).to eq("") + expect(qury).to eq("I like to watch") + expect(subs).to be_false + end + end + + it "Decodes date: filter" do + Invidious::Search::Filters::Date.each do |value| + query = "This date:#{value} is old!" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new(date: value)) + expect(chan).to eq("") + expect(qury).to eq("This is old!") + expect(subs).to be_false + end + end + + it "Decodes duration: filter" do + Invidious::Search::Filters::Duration.each do |value| + query = "This duration:#{value} is old!" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new(duration: value)) + expect(chan).to eq("") + expect(qury).to eq("This is old!") + expect(subs).to be_false + end + end + + it "Decodes feature: filter" do + Invidious::Search::Filters::Features.each do |value| + string = FEATURES_TEXT[value] + query = "I like my precious feature:#{string} ^^" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new(features: value)) + expect(chan).to eq("") + expect(qury).to eq("I like my precious ^^") + expect(subs).to be_false + end + end + + it "Decodes features: filter" do + query = "This search has many features:vr180,cc,hdr :o" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + features = Invidious::Search::Filters::Features.flags(HDR, VR180, CCommons) + + expect(fltr).to eq(described_class.new(features: features)) + expect(chan).to eq("") + expect(qury).to eq("This search has many :o") + expect(subs).to be_false + end + + it "Decodes sort: filter" do + Invidious::Search::Filters::Sort.each do |value| + query = "Computer? sort:#{value} my files!" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new(sort: value)) + expect(chan).to eq("") + expect(qury).to eq("Computer? my files!") + expect(subs).to be_false + end + end + + it "Decodes subscriptions: filter" do + query = "enable subscriptions:true" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new) + expect(chan).to eq("") + expect(qury).to eq("enable") + expect(subs).to be_true + end + + it "Ignores junk data" do + query = "duration:I sort:like type:cleaning features:stuff date:up!" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new) + expect(chan).to eq("") + expect(qury).to eq("") + expect(subs).to be_false + end + + it "Keeps unknown keys" do + query = "to:be or:not to:be" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new) + expect(chan).to eq("") + expect(qury).to eq("to:be or:not to:be") + expect(subs).to be_false + end + end + + # ------------------- + # Decode (URL) + # ------------------- + + describe "#from_iv_params" do + it "Decodes type= filter" do + Invidious::Search::Filters::Type.each do |value| + params = HTTP::Params.parse("type=#{value}") + + expect(described_class.from_iv_params(params)) + .to eq(described_class.new(type: value)) + end + end + + it "Decodes date= filter" do + Invidious::Search::Filters::Date.each do |value| + params = HTTP::Params.parse("date=#{value}") + + expect(described_class.from_iv_params(params)) + .to eq(described_class.new(date: value)) + end + end + + it "Decodes duration= filter" do + Invidious::Search::Filters::Duration.each do |value| + params = HTTP::Params.parse("duration=#{value}") + + expect(described_class.from_iv_params(params)) + .to eq(described_class.new(duration: value)) + end + end + + it "Decodes features= filter (single)" do + Invidious::Search::Filters::Features.each do |value| + string = described_class.format_features(value) + params = HTTP::Params.parse("features=#{string}") + + expect(described_class.from_iv_params(params)) + .to eq(described_class.new(features: value)) + end + end + + it "Decodes features= filter (multiple - comma separated)" do + features = Invidious::Search::Filters::Features.flags(HDR, VR180, CCommons) + params = HTTP::Params.parse("features=vr180%2Ccc%2Chdr") # %2C is a comma + + expect(described_class.from_iv_params(params)) + .to eq(described_class.new(features: features)) + end + + it "Decodes features= filter (multiple - URL parameters)" do + features = Invidious::Search::Filters::Features.flags(ThreeSixty, HD, FourK) + params = HTTP::Params.parse("features=4k&features=360&features=hd") + + expect(described_class.from_iv_params(params)) + .to eq(described_class.new(features: features)) + end + + it "Decodes sort= filter" do + Invidious::Search::Filters::Sort.each do |value| + params = HTTP::Params.parse("sort=#{value}") + + expect(described_class.from_iv_params(params)) + .to eq(described_class.new(sort: value)) + end + end + + it "Ignores junk data" do + params = HTTP::Params.parse("foo=bar&sort=views&answer=42&type=channel") + + expect(described_class.from_iv_params(params)).to eq( + described_class.new( + sort: Invidious::Search::Filters::Sort::Views, + type: Invidious::Search::Filters::Type::Channel + ) + ) + end + end + + # ------------------- + # Encode (URL) + # ------------------- + + describe "#to_iv_params" do + it "Encodes date filter" do + Invidious::Search::Filters::Date.each do |value| + filters = described_class.new(date: value) + params = filters.to_iv_params + + if value.none? + expect("#{params}").to eq("") + else + expect("#{params}").to eq("date=#{value.to_s.underscore}") + end + end + end + + it "Encodes type filter" do + Invidious::Search::Filters::Type.each do |value| + filters = described_class.new(type: value) + params = filters.to_iv_params + + if value.all? + expect("#{params}").to eq("") + else + expect("#{params}").to eq("type=#{value.to_s.underscore}") + end + end + end + + it "Encodes duration filter" do + Invidious::Search::Filters::Duration.each do |value| + filters = described_class.new(duration: value) + params = filters.to_iv_params + + if value.none? + expect("#{params}").to eq("") + else + expect("#{params}").to eq("duration=#{value.to_s.underscore}") + end + end + end + + it "Encodes features filter (single)" do + Invidious::Search::Filters::Features.each do |value| + string = described_class.format_features(value) + filters = described_class.new(features: value) + + expect("#{filters.to_iv_params}") + .to eq("features=" + FEATURES_TEXT[value]) + end + end + + it "Encodes features filter (multiple)" do + features = Invidious::Search::Filters::Features.flags(Subtitles, Live, ThreeSixty) + filters = described_class.new(features: features) + + expect("#{filters.to_iv_params}") + .to eq("features=live%2Csubtitles%2C360") # %2C is a comma + end + + it "Encodes sort filter" do + Invidious::Search::Filters::Sort.each do |value| + filters = described_class.new(sort: value) + params = filters.to_iv_params + + if value.relevance? + expect("#{params}").to eq("") + else + expect("#{params}").to eq("sort=#{value.to_s.underscore}") + end + end + end + + it "Encodes multiple filters" do + filters = described_class.new( + date: Invidious::Search::Filters::Date::Today, + duration: Invidious::Search::Filters::Duration::Medium, + features: Invidious::Search::Filters::Features.flags(Location, Purchased), + sort: Invidious::Search::Filters::Sort::Relevance + ) + + params = filters.to_iv_params + + # Check the `date` param + expect(params).to have_key("date") + expect(params.fetch_all("date")).to contain_exactly("today") + + # Check the `type` param + expect(params).to_not have_key("type") + expect(params["type"]?).to be_nil + + # Check the `duration` param + expect(params).to have_key("duration") + expect(params.fetch_all("duration")).to contain_exactly("medium") + + # Check the `features` param + expect(params).to have_key("features") + expect(params.fetch_all("features")).to contain_exactly("location,purchased") + + # Check the `sort` param + expect(params).to_not have_key("sort") + expect(params["sort"]?).to be_nil + + # Check if there aren't other parameters + params.delete("date") + params.delete("duration") + params.delete("features") + + expect(params).to be_empty + end + end +end diff --git a/spec/invidious/search/yt_filters_spec.cr b/spec/invidious/search/yt_filters_spec.cr new file mode 100644 index 00000000..2c0d61eb --- /dev/null +++ b/spec/invidious/search/yt_filters_spec.cr @@ -0,0 +1,143 @@ +require "../../../src/invidious/search/filters" + +require "http/params" +require "spectator" + +Spectator.configure do |config| + config.fail_blank + config.randomize +end + +# Encoded filter values are extracted from the search +# page of Youtube with any browser devtools HTML inspector. + +DATE_FILTERS = { + Invidious::Search::Filters::Date::Hour => "EgIIAQ%3D%3D", + Invidious::Search::Filters::Date::Today => "EgIIAg%3D%3D", + Invidious::Search::Filters::Date::Week => "EgIIAw%3D%3D", + Invidious::Search::Filters::Date::Month => "EgIIBA%3D%3D", + Invidious::Search::Filters::Date::Year => "EgIIBQ%3D%3D", +} + +TYPE_FILTERS = { + Invidious::Search::Filters::Type::Video => "EgIQAQ%3D%3D", + Invidious::Search::Filters::Type::Channel => "EgIQAg%3D%3D", + Invidious::Search::Filters::Type::Playlist => "EgIQAw%3D%3D", + Invidious::Search::Filters::Type::Movie => "EgIQBA%3D%3D", +} + +DURATION_FILTERS = { + Invidious::Search::Filters::Duration::Short => "EgIYAQ%3D%3D", + Invidious::Search::Filters::Duration::Medium => "EgIYAw%3D%3D", + Invidious::Search::Filters::Duration::Long => "EgIYAg%3D%3D", +} + +FEATURE_FILTERS = { + Invidious::Search::Filters::Features::Live => "EgJAAQ%3D%3D", + Invidious::Search::Filters::Features::FourK => "EgJwAQ%3D%3D", + Invidious::Search::Filters::Features::HD => "EgIgAQ%3D%3D", + Invidious::Search::Filters::Features::Subtitles => "EgIoAQ%3D%3D", + Invidious::Search::Filters::Features::CCommons => "EgIwAQ%3D%3D", + Invidious::Search::Filters::Features::ThreeSixty => "EgJ4AQ%3D%3D", + Invidious::Search::Filters::Features::VR180 => "EgPQAQE%3D", + Invidious::Search::Filters::Features::ThreeD => "EgI4AQ%3D%3D", + Invidious::Search::Filters::Features::HDR => "EgPIAQE%3D", + Invidious::Search::Filters::Features::Location => "EgO4AQE%3D", + Invidious::Search::Filters::Features::Purchased => "EgJIAQ%3D%3D", +} + +SORT_FILTERS = { + Invidious::Search::Filters::Sort::Relevance => "", + Invidious::Search::Filters::Sort::UploadDate => "CAI%3D", + Invidious::Search::Filters::Sort::Views => "CAM%3D", + Invidious::Search::Filters::Sort::Rating => "CAE%3D", +} + +Spectator.describe Invidious::Search::Filters do + # ------------------- + # Encode YT params + # ------------------- + + describe "#to_yt_params" do + sample DATE_FILTERS do |value, result| + it "Encodes upload date filter '#{value}'" do + expect(described_class.new(date: value).to_yt_params).to eq(result) + end + end + + sample TYPE_FILTERS do |value, result| + it "Encodes content type filter '#{value}'" do + expect(described_class.new(type: value).to_yt_params).to eq(result) + end + end + + sample DURATION_FILTERS do |value, result| + it "Encodes duration filter '#{value}'" do + expect(described_class.new(duration: value).to_yt_params).to eq(result) + end + end + + sample FEATURE_FILTERS do |value, result| + it "Encodes feature filter '#{value}'" do + expect(described_class.new(features: value).to_yt_params).to eq(result) + end + end + + sample SORT_FILTERS do |value, result| + it "Encodes sort filter '#{value}'" do + expect(described_class.new(sort: value).to_yt_params).to eq(result) + end + end + end + + # ------------------- + # Decode YT params + # ------------------- + + describe "#from_yt_params" do + sample DATE_FILTERS do |value, encoded| + it "Decodes upload date filter '#{value}'" do + params = HTTP::Params.parse("sp=#{encoded}") + + expect(described_class.from_yt_params(params)) + .to eq(described_class.new(date: value)) + end + end + + sample TYPE_FILTERS do |value, encoded| + it "Decodes content type filter '#{value}'" do + params = HTTP::Params.parse("sp=#{encoded}") + + expect(described_class.from_yt_params(params)) + .to eq(described_class.new(type: value)) + end + end + + sample DURATION_FILTERS do |value, encoded| + it "Decodes duration filter '#{value}'" do + params = HTTP::Params.parse("sp=#{encoded}") + + expect(described_class.from_yt_params(params)) + .to eq(described_class.new(duration: value)) + end + end + + sample FEATURE_FILTERS do |value, encoded| + it "Decodes feature filter '#{value}'" do + params = HTTP::Params.parse("sp=#{encoded}") + + expect(described_class.from_yt_params(params)) + .to eq(described_class.new(features: value)) + end + end + + sample SORT_FILTERS do |value, encoded| + it "Decodes sort filter '#{value}'" do + params = HTTP::Params.parse("sp=#{encoded}") + + expect(described_class.from_yt_params(params)) + .to eq(described_class.new(sort: value)) + end + end + end +end diff --git a/src/invidious.cr b/src/invidious.cr index a470c6b6..9f3d5d10 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -35,6 +35,7 @@ require "./invidious/frontend/*" require "./invidious/*" require "./invidious/channels/*" require "./invidious/user/*" +require "./invidious/search/*" require "./invidious/routes/**" require "./invidious/jobs/**" diff --git a/src/invidious/frontend/search_filters.cr b/src/invidious/frontend/search_filters.cr new file mode 100644 index 00000000..75bea5c2 --- /dev/null +++ b/src/invidious/frontend/search_filters.cr @@ -0,0 +1,128 @@ +module Invidious::Frontend::SearchFilters + extend self + + # Generate the search filters collapsable widget. + def generate(filters : Search::Filters, query : String, page : Int, locale : String) : String + return String.build(8000) do |str| + str << "
\n" + str << "\t
" + str << "\t\t

" << translate(locale, "search_filters_title") << "

\n" + + str << "\t\t
\n" + str << "\t\t\t
\n" + + str << "\t\t\t\t\n" + str << "\t\t\t\t\n" + + str << filter_wrapper(date) + str << filter_wrapper(type) + str << filter_wrapper(duration) + str << filter_wrapper(features) + str << filter_wrapper(sort) + + str << "\t\t\t\t
" + str << "
\n" + + str << "\t\t\t
\n" + str << "\t\t
\n" + + str << "\t
\n" + str << "
\n" + end + end + + # Generate wrapper HTML (`
`, filter name, etc...) around the + # `` elements of a search filter + macro filter_wrapper(name) + str << "\t\t\t\t
\n" + + str << "\t\t\t\t
" + str << translate(locale, "search_filters_{{name}}_label") + str << "
\n" + + str << "\t\t\t\t
" + str << make_{{name}}_filter_options(str, filters.{{name}}, locale) + str << "
\n" + + str << "\t\t\t\t
\n" + end + + # Generates the HTML for the list of radio buttons of the "date" search filter + def make_date_filter_options(str : String::Builder, value : Search::Filters::Date, locale : String) + {% for value in Invidious::Search::Filters::Date.constants %} + {% date = value.underscore %} + + str << "\t\t\t\t\n" + + str << "\t\t\t\t
\n" + {% end %} + end + + # Generates the HTML for the list of radio buttons of the "type" search filter + def make_type_filter_options(str : String::Builder, value : Search::Filters::Type, locale : String) + {% for value in Invidious::Search::Filters::Type.constants %} + {% type = value.underscore %} + + str << "\t\t\t\t\n" + + str << "\t\t\t\t
\n" + {% end %} + end + + # Generates the HTML for the list of radio buttons of the "duration" search filter + def make_duration_filter_options(str : String::Builder, value : Search::Filters::Duration, locale : String) + {% for value in Invidious::Search::Filters::Duration.constants %} + {% duration = value.underscore %} + + str << "\t\t\t\t\n" + + str << "\t\t\t\t
\n" + {% end %} + end + + # Generates the HTML for the list of checkboxes of the "features" search filter + def make_features_filter_options(str : String::Builder, value : Search::Filters::Features, locale : String) + {% for value in Invidious::Search::Filters::Features.constants %} + {% if value.stringify != "All" && value.stringify != "None" %} + {% feature = value.underscore %} + + str << "\t\t\t\t\n" + + str << "\t\t\t\t
\n" + {% end %} + {% end %} + end + + # Generates the HTML for the list of radio buttons of the "sort" search filter + def make_sort_filter_options(str : String::Builder, value : Search::Filters::Sort, locale : String) + {% for value in Invidious::Search::Filters::Sort.constants %} + {% sort = value.underscore %} + + str << "\t\t\t\t\n" + + str << "\t\t\t\t
\n" + {% end %} + end +end diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index c4d6643a..c4395353 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -262,7 +262,7 @@ module Invidious::Routes::API::V1::Channels page = env.params.query["page"]?.try &.to_i? page ||= 1 - search_results = channel_search(query, page, ucid) + search_results = Invidious::Search::Processors.channel(query, page, ucid) JSON.build do |json| json.array do search_results.each do |item| diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index 5666460d..ddf6dff4 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -11,28 +11,15 @@ module Invidious::Routes::API::V1::Search page = env.params.query["page"]?.try &.to_i? page ||= 1 - sort_by = env.params.query["sort_by"]?.try &.downcase - sort_by ||= "relevance" - - date = env.params.query["date"]?.try &.downcase - date ||= "" - - duration = env.params.query["duration"]?.try &.downcase - duration ||= "" - - features = env.params.query["features"]?.try &.split(",").map(&.downcase) - features ||= [] of String - - content_type = env.params.query["type"]?.try &.downcase - content_type ||= "video" + filters = Invidious::Search::Filters.from_iv_params(env.params.query) + search_params = filters.to_yt_params begin - search_params = produce_search_params(page, sort_by, date, content_type, duration, features) + search_results = search(query, search_params, region) rescue ex return error_json(400, ex) end - search_results = search(query, search_params, region) JSON.build do |json| json.array do search_results.each do |item| diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index dbeb4f97..fa8cfb68 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -239,7 +239,7 @@ module Invidious::Routes::Playlists query = env.params.query["q"]? if query begin - search_query, items, operators = process_search_query(query, page, user, region: nil) + search_query, items, _ = process_search_query(query, page, user, region: nil) videos = items.select(SearchVideo).map(&.as(SearchVideo)) rescue ex videos = [] of SearchVideo diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index 3f4c7e5e..92585a00 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -54,19 +54,13 @@ module Invidious::Routes::Search user = env.get? "user" begin - search_query, videos, operators = process_search_query(query, page, user, region: region) + search_query, videos, filters = process_search_query(query, page, user, region: region) rescue ex : ChannelSearchException return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.") rescue ex return error_template(500, ex) end - operator_hash = {} of String => String - operators.each do |operator| - key, value = operator.downcase.split(":") - operator_hash[key] = value - end - env.set "search", query templated "search" end diff --git a/src/invidious/search.cr b/src/invidious/search.cr index ae106bf6..8f4f2fca 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -5,36 +5,7 @@ class ChannelSearchException < InfoException end end -def channel_search(query, page, channel) : Array(SearchItem) - response = YT_POOL.client &.get("/channel/#{channel}") - - if response.status_code == 404 - response = YT_POOL.client &.get("/user/#{channel}") - response = YT_POOL.client &.get("/c/#{channel}") if response.status_code == 404 - initial_data = extract_initial_data(response.body) - ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?) - raise ChannelSearchException.new(channel) if !ucid - else - ucid = channel - end - - continuation = produce_channel_search_continuation(ucid, query, page) - response_json = YoutubeAPI.browse(continuation) - - continuation_items = response_json["onResponseReceivedActions"]? - .try &.[0]["appendContinuationItemsAction"]["continuationItems"] - - return [] of SearchItem if !continuation_items - - items = [] of SearchItem - continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item| - extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t } - end - - return items -end - -def search(query, search_params = produce_search_params(content_type: "all"), region = nil) : Array(SearchItem) +def search(query, search_params, region = nil) : Array(SearchItem) return [] of SearchItem if query.empty? client_config = YoutubeAPI::ClientConfig.new(region: region) @@ -43,104 +14,6 @@ def search(query, search_params = produce_search_params(content_type: "all"), re return extract_items(initial_data) end -def produce_search_params(page = 1, sort : String = "relevance", date : String = "", content_type : String = "", - duration : String = "", features : Array(String) = [] of String) - object = { - "1:varint" => 0_i64, - "2:embedded" => {} of String => Int64, - "9:varint" => ((page - 1) * 20).to_i64, - } - - case sort - when "relevance" - object["1:varint"] = 0_i64 - when "rating" - object["1:varint"] = 1_i64 - when "upload_date", "date" - object["1:varint"] = 2_i64 - when "view_count", "views" - object["1:varint"] = 3_i64 - else - raise "No sort #{sort}" - end - - case date - when "hour" - object["2:embedded"].as(Hash)["1:varint"] = 1_i64 - when "today" - object["2:embedded"].as(Hash)["1:varint"] = 2_i64 - when "week" - object["2:embedded"].as(Hash)["1:varint"] = 3_i64 - when "month" - object["2:embedded"].as(Hash)["1:varint"] = 4_i64 - when "year" - object["2:embedded"].as(Hash)["1:varint"] = 5_i64 - else nil # Ignore - end - - case content_type - when "video" - object["2:embedded"].as(Hash)["2:varint"] = 1_i64 - when "channel" - object["2:embedded"].as(Hash)["2:varint"] = 2_i64 - when "playlist" - object["2:embedded"].as(Hash)["2:varint"] = 3_i64 - when "movie" - object["2:embedded"].as(Hash)["2:varint"] = 4_i64 - when "show" - object["2:embedded"].as(Hash)["2:varint"] = 5_i64 - when "all" - # - else - object["2:embedded"].as(Hash)["2:varint"] = 1_i64 - end - - case duration - when "short" - object["2:embedded"].as(Hash)["3:varint"] = 1_i64 - when "long" - object["2:embedded"].as(Hash)["3:varint"] = 2_i64 - else nil # Ignore - end - - features.each do |feature| - case feature - when "hd" - object["2:embedded"].as(Hash)["4:varint"] = 1_i64 - when "subtitles" - object["2:embedded"].as(Hash)["5:varint"] = 1_i64 - when "creative_commons", "cc" - object["2:embedded"].as(Hash)["6:varint"] = 1_i64 - when "3d" - object["2:embedded"].as(Hash)["7:varint"] = 1_i64 - when "live", "livestream" - object["2:embedded"].as(Hash)["8:varint"] = 1_i64 - when "purchased" - object["2:embedded"].as(Hash)["9:varint"] = 1_i64 - when "4k" - object["2:embedded"].as(Hash)["14:varint"] = 1_i64 - when "360" - object["2:embedded"].as(Hash)["15:varint"] = 1_i64 - when "location" - object["2:embedded"].as(Hash)["23:varint"] = 1_i64 - when "hdr" - object["2:embedded"].as(Hash)["25:varint"] = 1_i64 - else nil # Ignore - end - end - - if object["2:embedded"].as(Hash).empty? - object.delete("2:embedded") - end - - params = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return params -end - def produce_channel_search_continuation(ucid, query, page) if page <= 1 idx = 0_i64 @@ -175,63 +48,20 @@ def produce_channel_search_continuation(ucid, query, page) end def process_search_query(query, page, user, region) - if user - user = user.as(Invidious::User) - view_name = "subscriptions_#{sha256(user.email)}" - end - - channel = nil - content_type = "all" - date = "" - duration = "" - features = [] of String - sort = "relevance" - subscriptions = nil - - operators = query.split(" ").select(&.match(/\w+:[\w,]+/)) - operators.each do |operator| - key, value = operator.downcase.split(":") - - case key - when "channel", "user" - channel = operator.split(":")[-1] - when "content_type", "type" - content_type = value - when "date" - date = value - when "duration" - duration = value - when "feature", "features" - features = value.split(",") - when "sort" - sort = value - when "subscriptions" - subscriptions = value == "true" - else - operators.delete(operator) - end - end - - search_query = (query.split(" ") - operators).join(" ") + # Parse legacy query + filters, channel, search_query, subscriptions = Invidious::Search::Filters.from_legacy_filters(query) if channel - items = channel_search(search_query, page, channel) + items = Invidious::Search::Processors.channel(search_query, page, channel) elsif subscriptions - if view_name - items = PG_DB.query_all("SELECT id,title,published,updated,ucid,author,length_seconds FROM ( - SELECT *, - to_tsvector(#{view_name}.title) || - to_tsvector(#{view_name}.author) - as document - FROM #{view_name} - ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", search_query, (page - 1) * 20, as: ChannelVideo) + if user + user = user.as(Invidious::User) + items = Invidious::Search::Processors.subscriptions(query, page, user) else items = [] of ChannelVideo end else - search_params = produce_search_params(page: page, sort: sort, date: date, content_type: content_type, - duration: duration, features: features) - + search_params = filters.to_yt_params(page: page) items = search(search_query, search_params, region) end @@ -250,5 +80,5 @@ def process_search_query(query, page, user, region) end end - {search_query, items_without_category, operators} + {search_query, items_without_category, filters} end diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr new file mode 100644 index 00000000..bc9d4b38 --- /dev/null +++ b/src/invidious/search/filters.cr @@ -0,0 +1,374 @@ +require "protodec/utils" +require "http/params" + +module Invidious::Search + struct Filters + # Values correspond to { "2:embedded": { "1:varint": }} + # except for "None" which is only used by us (= nothing selected) + enum Date + None = 0 + Hour = 1 + Today = 2 + Week = 3 + Month = 4 + Year = 5 + end + + # Values correspond to { "2:embedded": { "2:varint": }} + # except for "All" which is only used by us (= nothing selected) + enum Type + All = 0 + Video = 1 + Channel = 2 + Playlist = 3 + Movie = 4 + + # Has it been removed? + # (Not available on youtube's UI) + Show = 5 + end + + # Values correspond to { "2:embedded": { "3:varint": }} + # except for "None" which is only used by us (= nothing selected) + enum Duration + None = 0 + Short = 1 # "Under 4 minutes" + Long = 2 # "Over 20 minutes" + Medium = 3 # "4 - 20 minutes" + end + + # Note: flag enums automatically generate + # "none" and "all" members + @[Flags] + enum Features + Live + FourK # "4K" + HD + Subtitles # "Subtitles/CC" + CCommons # "Creative Commons" + ThreeSixty # "360°" + VR180 + ThreeD # "3D" + HDR + Location + Purchased + end + + # Values correspond to { "1:varint": } + enum Sort + Relevance = 0 + Rating = 1 + Date = 2 + Views = 3 + + # Alias for `Date` + UploadDate = 2 + end + + # Parameters are sorted as on Youtube + property date : Date + property type : Type + property duration : Duration + property features : Features + property sort : Sort + + def initialize( + *, # All parameters must be named + @date : Date = Date::None, + @type : Type = Type::All, + @duration : Duration = Duration::None, + @features : Features = Features::None, + @sort : Sort = Sort::Relevance + ) + end + + # ------------------- + # Invidious params + # ------------------- + + def self.parse_features(raw : Array(String)) : Features + # Initialize return variable + features = Features.new(0) + + raw.each do |ft| + case ft.downcase + when "live", "livestream" + features = features | Features::Live + when "4k" then features = features | Features::FourK + when "hd" then features = features | Features::HD + when "subtitles" then features = features | Features::Subtitles + when "creative_commons", "commons", "cc" + features = features | Features::CCommons + when "360" then features = features | Features::ThreeSixty + when "vr180" then features = features | Features::VR180 + when "3d" then features = features | Features::ThreeD + when "hdr" then features = features | Features::HDR + when "location" then features = features | Features::Location + when "purchased" then features = features | Features::Purchased + end + end + + return features + end + + def self.format_features(features : Features) : String + # Directly return an empty string if there are no features + return "" if features.none? + + # Initialize return variable + str = [] of String + + str << "live" if features.live? + str << "4k" if features.four_k? + str << "hd" if features.hd? + str << "subtitles" if features.subtitles? + str << "commons" if features.c_commons? + str << "360" if features.three_sixty? + str << "vr180" if features.vr180? + str << "3d" if features.three_d? + str << "hdr" if features.hdr? + str << "location" if features.location? + str << "purchased" if features.purchased? + + return str.join(',') + end + + def self.from_legacy_filters(str : String) : {Filters, String, String, Bool} + # Split search query on spaces + members = str.split(' ') + + # Output variables + channel = "" + filters = Filters.new + subscriptions = false + + # Array to hold the non-filter members + query = [] of String + + # Parse! + members.each do |substr| + # Separator operators + operators = substr.split(':') + + case operators[0] + when "user", "channel" + next if operators.size != 2 + channel = operators[1] + # + when "type", "content_type" + next if operators.size != 2 + type = Type.parse?(operators[1]) + filters.type = type if !type.nil? + # + when "date" + next if operators.size != 2 + date = Date.parse?(operators[1]) + filters.date = date if !date.nil? + # + when "duration" + next if operators.size != 2 + duration = Duration.parse?(operators[1]) + filters.duration = duration if !duration.nil? + # + when "feature", "features" + next if operators.size != 2 + features = parse_features(operators[1].split(',')) + filters.features = features if !features.nil? + # + when "sort" + next if operators.size != 2 + sort = Sort.parse?(operators[1]) + filters.sort = sort if !sort.nil? + # + when "subscriptions" + next if operators.size != 2 + subscriptions = {"true", "on", "yes", "1"}.any?(&.== operators[1]) + # + else + query << substr + end + end + + # Re-assemble query (without filters) + cleaned_query = query.join(' ') + + return {filters, channel, cleaned_query, subscriptions} + end + + def self.from_iv_params(params : HTTP::Params) : Filters + # Temporary variables + filters = Filters.new + + if type = params["type"]? + filters.type = Type.parse?(type) || Type::All + params.delete("type") + end + + if date = params["date"]? + filters.date = Date.parse?(date) || Date::None + params.delete("date") + end + + if duration = params["duration"]? + filters.duration = Duration.parse?(duration) || Duration::None + params.delete("duration") + end + + features = params.fetch_all("features") + if !features.empty? + # Un-array input so it can be treated as a comma-separated list + features = features[0].split(',') if features.size == 1 + + filters.features = parse_features(features) || Features::None + params.delete_all("features") + end + + if sort = params["sort"]? + filters.sort = Sort.parse?(sort) || Sort::Relevance + params.delete("sort") + end + + return filters + end + + def to_iv_params : HTTP::Params + # Temporary variables + raw_params = {} of String => Array(String) + + raw_params["date"] = [@date.to_s.underscore] if !@date.none? + raw_params["type"] = [@type.to_s.underscore] if !@type.all? + raw_params["sort"] = [@sort.to_s.underscore] if !@sort.relevance? + + if !@duration.none? + raw_params["duration"] = [@duration.to_s.underscore] + end + + if !@features.none? + raw_params["features"] = [Filters.format_features(@features)] + end + + return HTTP::Params.new(raw_params) + end + + # ------------------- + # Youtube params + # ------------------- + + # Produce the youtube search parameters for the + # innertube API (base64-encoded protobuf object). + def to_yt_params(page : Int = 1) : String + # Initialize the embedded protobuf object + embedded = {} of String => Int64 + + # Add these field only if associated parameter is selected + embedded["1:varint"] = @date.to_i64 if !@date.none? + embedded["2:varint"] = @type.to_i64 if !@type.all? + embedded["3:varint"] = @duration.to_i64 if !@duration.none? + + if !@features.none? + # All features have a value of "1" when enabled, and + # the field is omitted when the feature is no selected. + embedded["4:varint"] = 1_i64 if @features.includes?(Features::HD) + embedded["5:varint"] = 1_i64 if @features.includes?(Features::Subtitles) + embedded["6:varint"] = 1_i64 if @features.includes?(Features::CCommons) + embedded["7:varint"] = 1_i64 if @features.includes?(Features::ThreeD) + embedded["8:varint"] = 1_i64 if @features.includes?(Features::Live) + embedded["9:varint"] = 1_i64 if @features.includes?(Features::Purchased) + embedded["14:varint"] = 1_i64 if @features.includes?(Features::FourK) + embedded["15:varint"] = 1_i64 if @features.includes?(Features::ThreeSixty) + embedded["23:varint"] = 1_i64 if @features.includes?(Features::Location) + embedded["25:varint"] = 1_i64 if @features.includes?(Features::HDR) + embedded["26:varint"] = 1_i64 if @features.includes?(Features::VR180) + end + + # Initialize an empty protobuf object + object = {} of String => (Int64 | String | Hash(String, Int64)) + + # As usual, everything can be omitted if it has no value + object["2:embedded"] = embedded if !embedded.empty? + + # Default sort is "relevance", so when this option is selected, + # the associated field can be omitted. + if !@sort.relevance? + object["1:varint"] = @sort.to_i64 + end + + # Add page number (if provided) + if page > 1 + object["9:varint"] = ((page - 1) * 20).to_i64 + end + + # If the object is empty, return an empty string, + # otherwise encode to protobuf then to base64 + return "" if object.empty? + + return object + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + end + + # Function to parse the `sp` URL parameter from Youtube + # search page. It's a base64-encoded protobuf object. + def self.from_yt_params(params : HTTP::Params) : Filters + # Initialize output variable + filters = Filters.new + + # Get parameter, and check emptyness + search_params = params["sp"]? + + if search_params.nil? || search_params.empty? + return filters + end + + # Decode protobuf object + object = search_params + .try { |i| URI.decode_www_form(i) } + .try { |i| Base64.decode(i) } + .try { |i| IO::Memory.new(i) } + .try { |i| Protodec::Any.parse(i) } + + # Parse items from embedded object + if embedded = object["2:0:embedded"]? + # All the following fields (date, type, duration) are optional. + if date = embedded["1:0:varint"]? + filters.date = Date.from_value?(date.as_i) || Date::None + end + + if type = embedded["2:0:varint"]? + filters.type = Type.from_value?(type.as_i) || Type::All + end + + if duration = embedded["3:0:varint"]? + filters.duration = Duration.from_value?(duration.as_i) || Duration::None + end + + # All features should have a value of "1" when enabled, and + # the field should be omitted when the feature is no selected. + features = 0 + features += (embedded["4:0:varint"]?.try &.as_i == 1_i64) ? Features::HD.value : 0 + features += (embedded["5:0:varint"]?.try &.as_i == 1_i64) ? Features::Subtitles.value : 0 + features += (embedded["6:0:varint"]?.try &.as_i == 1_i64) ? Features::CCommons.value : 0 + features += (embedded["7:0:varint"]?.try &.as_i == 1_i64) ? Features::ThreeD.value : 0 + features += (embedded["8:0:varint"]?.try &.as_i == 1_i64) ? Features::Live.value : 0 + features += (embedded["9:0:varint"]?.try &.as_i == 1_i64) ? Features::Purchased.value : 0 + features += (embedded["14:0:varint"]?.try &.as_i == 1_i64) ? Features::FourK.value : 0 + features += (embedded["15:0:varint"]?.try &.as_i == 1_i64) ? Features::ThreeSixty.value : 0 + features += (embedded["23:0:varint"]?.try &.as_i == 1_i64) ? Features::Location.value : 0 + features += (embedded["25:0:varint"]?.try &.as_i == 1_i64) ? Features::HDR.value : 0 + features += (embedded["26:0:varint"]?.try &.as_i == 1_i64) ? Features::VR180.value : 0 + + filters.features = Features.from_value?(features) || Features::None + end + + if sort = object["1:0:varint"]? + filters.sort = Sort.from_value?(sort.as_i) || Sort::Relevance + end + + # Remove URL parameter and return result + params.delete("sp") + return filters + end + end +end diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr new file mode 100644 index 00000000..c5327f34 --- /dev/null +++ b/src/invidious/search/processors.cr @@ -0,0 +1,54 @@ +module Invidious::Search + module Processors + extend self + + # Search a youtube channel + # TODO: clean code, and rely more on YoutubeAPI + def channel(query, page, channel) : Array(SearchItem) + response = YT_POOL.client &.get("/channel/#{channel}") + + if response.status_code == 404 + response = YT_POOL.client &.get("/user/#{channel}") + response = YT_POOL.client &.get("/c/#{channel}") if response.status_code == 404 + initial_data = extract_initial_data(response.body) + ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?) + raise ChannelSearchException.new(channel) if !ucid + else + ucid = channel + end + + continuation = produce_channel_search_continuation(ucid, query, page) + response_json = YoutubeAPI.browse(continuation) + + continuation_items = response_json["onResponseReceivedActions"]? + .try &.[0]["appendContinuationItemsAction"]["continuationItems"] + + return [] of SearchItem if !continuation_items + + items = [] of SearchItem + continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item| + extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t } + end + + return items + end + + # Search inside of user subscriptions + def subscriptions(query, page, user : Invidious::User) : Array(ChannelVideo) + view_name = "subscriptions_#{sha256(user.email)}" + + return PG_DB.query_all(" + SELECT id,title,published,updated,ucid,author,length_seconds + FROM ( + SELECT *, + to_tsvector(#{view_name}.title) || + to_tsvector(#{view_name}.author) + as document + FROM #{view_name} + ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", + query, (page - 1) * 20, + as: ChannelVideo + ) + end + end +end diff --git a/src/invidious/views/search.ecr b/src/invidious/views/search.ecr index 45bbdefc..89ab4e32 100644 --- a/src/invidious/views/search.ecr +++ b/src/invidious/views/search.ecr @@ -9,94 +9,11 @@

"><%= translate(locale, "Broken? Try another Invidious Instance!") %>

-<% else %> -
- -

<%= translate(locale, "filter") %>

-
-
-
- <%= translate(locale, "date") %> -
- <% ["hour", "today", "week", "month", "year"].each do |date| %> -
- <% if operator_hash.fetch("date", "all") == date %> - <%= translate(locale, date) %> - <% else %> - &page=<%= page %>"> - <%= translate(locale, date) %> - - <% end %> -
- <% end %> -
-
- <%= translate(locale, "content_type") %> -
- <% ["video", "channel", "playlist", "movie", "show"].each do |content_type| %> -
- <% if operator_hash.fetch("content_type", "all") == content_type %> - <%= translate(locale, content_type) %> - <% else %> - &page=<%= page %>"> - <%= translate(locale, content_type) %> - - <% end %> -
- <% end %> -
-
- <%= translate(locale, "duration") %> -
- <% ["short", "long"].each do |duration| %> -
- <% if operator_hash.fetch("duration", "all") == duration %> - <%= translate(locale, duration) %> - <% else %> - &page=<%= page %>"> - <%= translate(locale, duration) %> - - <% end %> -
- <% end %> -
-
- <%= translate(locale, "features") %> -
- <% ["hd", "subtitles", "creative_commons", "3d", "live", "purchased", "4k", "360", "location", "hdr"].each do |feature| %> -
- <% if operator_hash.fetch("features", "all").includes?(feature) %> - <%= translate(locale, feature) %> - <% elsif operator_hash.has_key?("features") %> - &page=<%= page %>"> - <%= translate(locale, feature) %> - - <% else %> - &page=<%= page %>"> - <%= translate(locale, feature) %> - - <% end %> -
- <% end %> -
-
- <%= translate(locale, "sort") %> -
- <% ["relevance", "rating", "date", "views"].each do |sort| %> -
- <% if operator_hash.fetch("sort", "relevance") == sort %> - <%= translate(locale, sort) %> - <% else %> - &page=<%= page %>"> - <%= translate(locale, sort) %> - - <% end %> -
- <% end %> -
-
-
-<% end %> +<% + else + Invidious::Frontend::SearchFilters.generate(filters, search_query, page, locale) + end +%> <% if videos.size == 0 %>
@@ -107,7 +24,7 @@
<% if page > 1 %> - + <%= translate(locale, "Previous page") %> <% end %> @@ -115,7 +32,7 @@
<% if videos.size >= 20 %> - + <%= translate(locale, "Next page") %> <% end %> @@ -131,7 +48,7 @@
<% if page > 1 %> - + <%= translate(locale, "Previous page") %> <% end %> @@ -139,7 +56,7 @@
<% if videos.size >= 20 %> - + <%= translate(locale, "Next page") %> <% end %>