Make use of Search::Query/Filters and associated HTML generator

src/invidious/routes/api/v1/channels.cr

@@ -251,18 +251,22 @@ module Invidious::Routes::API::V1::Channels
 
   def self.search(env)
     locale = env.get("preferences").as(Preferences).locale
+    region = env.params.query["region"]?
 
     env.response.content_type = "application/json"
 
-    ucid = env.params.url["ucid"]
+    query = Invidious::Search::Query.new(env.params.query, :channel, region)
 
-    query = env.params.query["q"]?
+    # Required because we can't (yet) pass multiple parameter to the
-    query ||= ""
+    # `Search::Query` initializer (in this case, an URL segment)
+    query.channel = env.params.url["ucid"]
 
-    page = env.params.query["page"]?.try &.to_i?
+    begin
-    page ||= 1
+      search_results = query.process
+    rescue ex
+      return error_json(400, ex)
+    end
 
-    search_results = Invidious::Search::Processors.channel(query, page, ucid)
     JSON.build do |json|
       json.array do
         search_results.each do |item|

src/invidious/routes/api/v1/search.cr

@@ -5,34 +5,14 @@ module Invidious::Routes::API::V1::Search
 
     env.response.content_type = "application/json"
 
-    query = env.params.query["q"]?
+    query = Invidious::Search::Query.new(env.params.query, :regular, region)
-    query ||= ""
-
-    page = env.params.query["page"]?.try &.to_i?
-    page ||= 1
-
-    sort_by = env.params.query["sort_by"]?.try &.downcase
-    sort_by ||= "relevance"
-
-    date = env.params.query["date"]?.try &.downcase
-    date ||= ""
-
-    duration = env.params.query["duration"]?.try &.downcase
-    duration ||= ""
-
-    features = env.params.query["features"]?.try &.split(",").map(&.downcase)
-    features ||= [] of String
-
-    content_type = env.params.query["type"]?.try &.downcase
-    content_type ||= "video"
 
     begin
-      search_params = produce_search_params(page, sort_by, date, content_type, duration, features)
+      search_results = query.process
     rescue ex
       return error_json(400, ex)
     end
 
-    search_results = search(query, search_params, region)
     JSON.build do |json|
       json.array do
         search_results.each do |item|

src/invidious/routes/playlists.cr

@@ -212,7 +212,10 @@ module Invidious::Routes::Playlists
   end
 
   def self.add_playlist_items_page(env)
-    locale = env.get("preferences").as(Preferences).locale
+    prefs = env.get("preferences").as(Preferences)
+    locale = prefs.locale
+
+    region = env.params.query["region"]? || prefs.region
 
     user = env.get? "user"
     sid = env.get? "sid"
@@ -236,15 +239,10 @@ module Invidious::Routes::Playlists
       return env.redirect referer
     end
 
-    query = env.params.query["q"]?
+    begin
-    if query
+      query = Invidious::Search::Query.new(env.params.query, :playlist, region)
-      begin
+      videos = query.process.select(SearchVideo).map(&.as(SearchVideo))
-        search_query, items, operators = process_search_query(query, page, user, region: nil)
+    rescue ex
-        videos = items.select(SearchVideo).map(&.as(SearchVideo))
-      rescue ex
-        videos = [] of SearchVideo
-      end
-    else
       videos = [] of SearchVideo
     end
 

src/invidious/routes/search.cr

@@ -37,37 +37,29 @@ module Invidious::Routes::Search
   end
 
   def self.search(env)
-    locale = env.get("preferences").as(Preferences).locale
+    prefs = env.get("preferences").as(Preferences)
-    region = env.params.query["region"]?
+    locale = prefs.locale
 
-    query = env.params.query["search_query"]?
+    region = env.params.query["region"]? || prefs.region
-    query ||= env.params.query["q"]?
 
-    if !query || query.empty?
+    query = Invidious::Search::Query.new(env.params.query, :regular, region)
+
+    if query.empty?
       # Display the full page search box implemented in #1977
       env.set "search", ""
       templated "search_homepage", navbar_search: false
     else
-      page = env.params.query["page"]?.try &.to_i?
-      page ||= 1
-
       user = env.get? "user"
 
       begin
-        search_query, videos, operators = process_search_query(query, page, user, region: region)
+        videos = query.process
       rescue ex : ChannelSearchException
         return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.")
       rescue ex
         return error_template(500, ex)
       end
 
-      operator_hash = {} of String => String
+      env.set "search", query.text
-      operators.each do |operator|
-        key, value = operator.downcase.split(":")
-        operator_hash[key] = value
-      end
-
-      env.set "search", query
       templated "search"
     end
   end

src/invidious/search.cr

@@ -5,113 +5,6 @@ class ChannelSearchException < InfoException
   end
 end
 
-def search(query, search_params = produce_search_params(content_type: "all"), region = nil) : Array(SearchItem)
-  return [] of SearchItem if query.empty?
-
-  client_config = YoutubeAPI::ClientConfig.new(region: region)
-  initial_data = YoutubeAPI.search(query, search_params, client_config: client_config)
-
-  return extract_items(initial_data)
-end
-
-def produce_search_params(page = 1, sort : String = "relevance", date : String = "", content_type : String = "",
-                          duration : String = "", features : Array(String) = [] of String)
-  object = {
-    "1:varint"   => 0_i64,
-    "2:embedded" => {} of String => Int64,
-    "9:varint"   => ((page - 1) * 20).to_i64,
-  }
-
-  case sort
-  when "relevance"
-    object["1:varint"] = 0_i64
-  when "rating"
-    object["1:varint"] = 1_i64
-  when "upload_date", "date"
-    object["1:varint"] = 2_i64
-  when "view_count", "views"
-    object["1:varint"] = 3_i64
-  else
-    raise "No sort #{sort}"
-  end
-
-  case date
-  when "hour"
-    object["2:embedded"].as(Hash)["1:varint"] = 1_i64
-  when "today"
-    object["2:embedded"].as(Hash)["1:varint"] = 2_i64
-  when "week"
-    object["2:embedded"].as(Hash)["1:varint"] = 3_i64
-  when "month"
-    object["2:embedded"].as(Hash)["1:varint"] = 4_i64
-  when "year"
-    object["2:embedded"].as(Hash)["1:varint"] = 5_i64
-  else nil # Ignore
-  end
-
-  case content_type
-  when "video"
-    object["2:embedded"].as(Hash)["2:varint"] = 1_i64
-  when "channel"
-    object["2:embedded"].as(Hash)["2:varint"] = 2_i64
-  when "playlist"
-    object["2:embedded"].as(Hash)["2:varint"] = 3_i64
-  when "movie"
-    object["2:embedded"].as(Hash)["2:varint"] = 4_i64
-  when "show"
-    object["2:embedded"].as(Hash)["2:varint"] = 5_i64
-  when "all"
-    #
-  else
-    object["2:embedded"].as(Hash)["2:varint"] = 1_i64
-  end
-
-  case duration
-  when "short"
-    object["2:embedded"].as(Hash)["3:varint"] = 1_i64
-  when "long"
-    object["2:embedded"].as(Hash)["3:varint"] = 2_i64
-  else nil # Ignore
-  end
-
-  features.each do |feature|
-    case feature
-    when "hd"
-      object["2:embedded"].as(Hash)["4:varint"] = 1_i64
-    when "subtitles"
-      object["2:embedded"].as(Hash)["5:varint"] = 1_i64
-    when "creative_commons", "cc"
-      object["2:embedded"].as(Hash)["6:varint"] = 1_i64
-    when "3d"
-      object["2:embedded"].as(Hash)["7:varint"] = 1_i64
-    when "live", "livestream"
-      object["2:embedded"].as(Hash)["8:varint"] = 1_i64
-    when "purchased"
-      object["2:embedded"].as(Hash)["9:varint"] = 1_i64
-    when "4k"
-      object["2:embedded"].as(Hash)["14:varint"] = 1_i64
-    when "360"
-      object["2:embedded"].as(Hash)["15:varint"] = 1_i64
-    when "location"
-      object["2:embedded"].as(Hash)["23:varint"] = 1_i64
-    when "hdr"
-      object["2:embedded"].as(Hash)["25:varint"] = 1_i64
-    else nil # Ignore
-    end
-  end
-
-  if object["2:embedded"].as(Hash).empty?
-    object.delete("2:embedded")
-  end
-
-  params = object.try { |i| Protodec::Any.cast_json(i) }
-    .try { |i| Protodec::Any.from_json(i) }
-    .try { |i| Base64.urlsafe_encode(i) }
-    .try { |i| URI.encode_www_form(i) }
-
-  return params
-end
-
 def produce_channel_search_continuation(ucid, query, page)
   if page <= 1
     idx = 0_i64
@@ -146,41 +39,10 @@ def produce_channel_search_continuation(ucid, query, page)
 end
 
 def process_search_query(query, page, user, region)
-  channel = nil
+  # Parse legacy query
-  content_type = "all"
+  filters, channel, search_query, subscriptions = Invidious::Search::Filters.from_legacy_filters(query)
-  date = ""
-  duration = ""
-  features = [] of String
-  sort = "relevance"
-  subscriptions = nil
 
-  operators = query.split(" ").select(&.match(/\w+:[\w,]+/))
+  if !channel.nil? && !channel.empty?
-  operators.each do |operator|
-    key, value = operator.downcase.split(":")
-
-    case key
-    when "channel", "user"
-      channel = operator.split(":")[-1]
-    when "content_type", "type"
-      content_type = value
-    when "date"
-      date = value
-    when "duration"
-      duration = value
-    when "feature", "features"
-      features = value.split(",")
-    when "sort"
-      sort = value
-    when "subscriptions"
-      subscriptions = value == "true"
-    else
-      operators.delete(operator)
-    end
-  end
-
-  search_query = (query.split(" ") - operators).join(" ")
-
-  if channel
     items = Invidious::Search::Processors.channel(search_query, page, channel)
   elsif subscriptions
     if user
@@ -190,9 +52,7 @@ def process_search_query(query, page, user, region)
       items = [] of ChannelVideo
     end
   else
-    search_params = produce_search_params(page: page, sort: sort, date: date, content_type: content_type,
+    search_params = filters.to_yt_params(page: page)
-      duration: duration, features: features)
-
     items = search(search_query, search_params, region)
   end
 
@@ -211,5 +71,5 @@ def process_search_query(query, page, user, region)
     end
   end
 
-  {search_query, items_without_category, operators}
+  {search_query, items_without_category, filters}
 end

src/invidious/search/filters.cr

@@ -79,7 +79,7 @@ module Invidious::Search
     )
     end
 
-    def is_default? : Bool
+    def default? : Bool
       return @date.none? && @type.all? && @duration.none? && \
          @features.none? && @sort.relevance?
     end

src/invidious/search/processors.cr

@@ -2,22 +2,32 @@ module Invidious::Search
   module Processors
     extend self
 
+    # Regular search (`/search` endpoint)
+    def regular(query : Query) : Array(SearchItem)
+      search_params = query.filters.to_yt_params(page: query.page)
+
+      client_config = YoutubeAPI::ClientConfig.new(region: query.region)
+      initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config)
+
+      return extract_items(initial_data)
+    end
+
     # Search a youtube channel
     # TODO: clean code, and rely more on YoutubeAPI
-    def channel(query, page, channel) : Array(SearchItem)
+    def channel(query : Query) : Array(SearchItem)
-      response = YT_POOL.client &.get("/channel/#{channel}")
+      response = YT_POOL.client &.get("/channel/#{query.channel}")
 
       if response.status_code == 404
-        response = YT_POOL.client &.get("/user/#{channel}")
+        response = YT_POOL.client &.get("/user/#{query.channel}")
-        response = YT_POOL.client &.get("/c/#{channel}") if response.status_code == 404
+        response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404
         initial_data = extract_initial_data(response.body)
         ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?)
-        raise ChannelSearchException.new(channel) if !ucid
+        raise ChannelSearchException.new(query.channel) if !ucid
       else
-        ucid = channel
+        ucid = query.channel
       end
 
-      continuation = produce_channel_search_continuation(ucid, query, page)
+      continuation = produce_channel_search_continuation(ucid, query.text, query.page)
       response_json = YoutubeAPI.browse(continuation)
 
       continuation_items = response_json["onResponseReceivedActions"]?
@@ -34,7 +44,7 @@ module Invidious::Search
     end
 
     # Search inside of user subscriptions
-    def subscriptions(query, page, user : Invidious::User) : Array(ChannelVideo)
+    def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo)
       view_name = "subscriptions_#{sha256(user.email)}"
 
       return PG_DB.query_all("
@@ -46,7 +56,7 @@ module Invidious::Search
           as document
           FROM #{view_name}
         ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;",
-        query, (page - 1) * 20,
+        query.text, (query.page - 1) * 20,
         as: ChannelVideo
       )
     end

src/invidious/search/query.cr

@@ -110,11 +110,10 @@ module Invidious::Search
 
       case @type
       when .regular?, .playlist?
-        all_items = search(@query, @filters, @page, @region)
+        items = unnest_items(Processors.regular(self))
-        items = unnest_items(all_items)
         #
       when .channel?
-        items = Processors.channel(@query, @page, @channel)
+        items = Processors.channel(self)
         #
       when .subscriptions?
         if user

src/invidious/views/add_playlist_items.ecr

@@ -11,7 +11,9 @@
                 <legend><a href="/playlist?list=<%= playlist.id %>"><%= translate(locale, "Editing playlist `x`", %|"#{HTML.escape(playlist.title)}"|) %></a></legend>
 
                 <fieldset>
-                    <input class="pure-input-1" type="search" name="q" <% if query %>value="<%= HTML.escape(query) %>"<% else %>placeholder="<%= translate(locale, "Search for videos") %>"<% end %>>
+                    <input class="pure-input-1" type="search" name="q"
+                      <% if query %>value="<%= HTML.escape(query.text) %>"<% end %>
+                      placeholder="<%= translate(locale, "Search for videos") %>">
                     <input type="hidden" name="list" value="<%= plid %>">
                 </fieldset>
             </form>
@@ -38,10 +40,11 @@
 </div>
 
 <% if query %>
+    <%- query_encoded = URI.encode_www_form(query.text, space_to_plus: true) -%>
     <div class="pure-g h-box">
         <div class="pure-u-1 pure-u-lg-1-5">
-            <% if page > 1 %>
+            <% if query.page > 1 %>
-                <a href="/add_playlist_items?list=<%= plid %>&q=<%= URI.encode_www_form(query.not_nil!) %>&page=<%= page - 1 %>">
+                <a href="/add_playlist_items?list=<%= plid %>&q=<%= query_encoded %>&page=<%= page - 1 %>">
                     <%= translate(locale, "Previous page") %>
                 </a>
             <% end %>
@@ -49,7 +52,7 @@
         <div class="pure-u-1 pure-u-lg-3-5"></div>
         <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
             <% if videos.size >= 20 %>
-                <a href="/add_playlist_items?list=<%= plid %>&q=<%= URI.encode_www_form(query.not_nil!) %>&page=<%= page + 1 %>">
+                <a href="/add_playlist_items?list=<%= plid %>&q=<%= query_encoded %>&page=<%= page + 1 %>">
                     <%= translate(locale, "Next page") %>
                 </a>
             <% end %>

src/invidious/views/search.ecr

@@ -1,124 +1,38 @@
 <% content_for "header" do %>
-<title><%= search_query.not_nil!.size > 30 ? HTML.escape(query.not_nil![0,30].rstrip(".") + "...") : HTML.escape(query.not_nil!) %> - Invidious</title>
+<title><%= query.text.size > 30 ? HTML.escape(query.text[0,30].rstrip(".")) + "&hellip;" : HTML.escape(query.text) %> - Invidious</title>
+<link rel="stylesheet" href="/css/search.css?v=<%= ASSET_COMMIT %>">
 <% end %>
 
-<% search_query_encoded = env.get?("search").try { |x| URI.encode_www_form(x.as(String), space_to_plus: true) } %>
+<%-
+  search_query_encoded = URI.encode_www_form(query.text, space_to_plus: true)
+  filter_params = query.filters.to_iv_params
+
+  url_prev_page = "/search?q=#{search_query_encoded}&#{filter_params}&page=#{query.page - 1}"
+  url_next_page = "/search?q=#{search_query_encoded}&#{filter_params}&page=#{query.page + 1}"
+-%>
 
 <!-- Search redirection and filtering UI -->
 <% if videos.size == 0 %>
     <h3 style="text-align: center">
         <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Broken? Try another Invidious Instance!") %></a>
     </h3>
-<% else %>
+<%- else -%>
-    <details id="filters">
+    <%= Invidious::Frontend::SearchFilters.generate(query.filters, query.text, query.page, locale) %>
-        <summary>
+<%- end -%>
-            <h3 style="display:inline"> <%= translate(locale, "filter") %> </h3>
-        </summary>
-        <div id="filters" class="pure-g h-box">
-            <div class="pure-u-1-3 pure-u-md-1-5">
-                <b><%= translate(locale, "date") %></b>
-                <hr/>
-                <% ["hour", "today", "week", "month", "year"].each do |date| %>
-                    <div class="pure-u-1 pure-md-1-5">
-                        <% if operator_hash.fetch("date", "all") == date %>
-                            <b><%= translate(locale, date) %></b>
-                        <% else %>
-                            <a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?date:[a-z]+/, "") + " date:" + date) %>&page=<%= page %>">
-                                <%= translate(locale, date) %>
-                            </a>
-                        <% end %>
-                    </div>
-                <% end %>
-            </div>
-            <div class="pure-u-1-3 pure-u-md-1-5">
-                <b><%= translate(locale, "content_type") %></b>
-                <hr/>
-                <% ["video", "channel", "playlist", "movie", "show"].each do |content_type| %>
-                    <div class="pure-u-1 pure-md-1-5">
-                        <% if operator_hash.fetch("content_type", "all") == content_type %>
-                            <b><%= translate(locale, content_type) %></b>
-                        <% else %>
-                            <a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?content_type:[a-z]+/, "") + " content_type:" + content_type) %>&page=<%= page %>">
-                                <%= translate(locale, content_type) %>
-                            </a>
-                        <% end %>
-                    </div>
-                <% end %>
-            </div>
-            <div class="pure-u-1-3 pure-u-md-1-5">
-                <b><%= translate(locale, "duration") %></b>
-                <hr/>
-                <% ["short", "long"].each do |duration| %>
-                    <div class="pure-u-1 pure-md-1-5">
-                        <% if operator_hash.fetch("duration", "all") == duration %>
-                            <b><%= translate(locale, duration) %></b>
-                        <% else %>
-                            <a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?duration:[a-z]+/, "") + " duration:" + duration) %>&page=<%= page %>">
-                                <%= translate(locale, duration) %>
-                            </a>
-                        <% end %>
-                    </div>
-                <% end %>
-            </div>
-            <div class="pure-u-1-3 pure-u-md-1-5">
-                <b><%= translate(locale, "features") %></b>
-                <hr/>
-                <% ["hd", "subtitles", "creative_commons", "3d", "live", "purchased", "4k", "360", "location", "hdr"].each do |feature| %>
-                    <div class="pure-u-1 pure-md-1-5">
-                        <% if operator_hash.fetch("features", "all").includes?(feature) %>
-                            <b><%= translate(locale, feature) %></b>
-                        <% elsif operator_hash.has_key?("features") %>
-                            <a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/features:/, "features:" + feature + ",")) %>&page=<%= page %>">
-                                <%= translate(locale, feature) %>
-                            </a>
-                        <% else %>
-                            <a href="/search?q=<%= URI.encode_www_form(query.not_nil! + " features:" + feature) %>&page=<%= page %>">
-                                <%= translate(locale, feature) %>
-                            </a>
-                        <% end %>
-                    </div>
-                <% end %>
-            </div>
-            <div class="pure-u-1-3 pure-u-md-1-5">
-                <b><%= translate(locale, "sort") %></b>
-                <hr/>
-                <% ["relevance", "rating", "date", "views"].each do |sort| %>
-                    <div class="pure-u-1 pure-md-1-5">
-                        <% if operator_hash.fetch("sort", "relevance") == sort %>
-                            <b><%= translate(locale, sort) %></b>
-                        <% else %>
-                            <a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?sort:[a-z]+/, "") + " sort:" + sort) %>&page=<%= page %>">
-                                <%= translate(locale, sort) %>
-                            </a>
-                        <% end %>
-                    </div>
-                <% end %>
-            </div>
-        </div>
-    </details>
-<% end %>
 
-<% if videos.size == 0 %>
+<% if videos.size == 0 %><hr style="margin: 0;"/><% else %><hr/><% end %>
-    <hr style="margin: 0;"/>
-<% else %>
-    <hr/>
-<% end %>
 
 <div class="pure-g h-box v-box">
     <div class="pure-u-1 pure-u-lg-1-5">
-        <% if page > 1 %>
+        <%- if query.page > 1 -%>
-            <a href="/search?q=<%= search_query_encoded %>&page=<%= page - 1 %>">
+            <a href="<%= url_prev_page %>"><%= translate(locale, "Previous page") %></a>
-                <%= translate(locale, "Previous page") %>
+        <%- end -%>
-            </a>
-        <% end %>
     </div>
     <div class="pure-u-1 pure-u-lg-3-5"></div>
     <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
-        <% if videos.size >= 20 %>
+        <%- if videos.size >= 20 -%>
-            <a href="/search?q=<%= search_query_encoded %>&page=<%= page + 1 %>">
+            <a href="<%= url_next_page %>"><%= translate(locale, "Next page") %></a>
-                <%= translate(locale, "Next page") %>
+        <%- end -%>
-            </a>
-        <% end %>
     </div>
 </div>
 
@@ -130,18 +44,14 @@
 
 <div class="pure-g h-box">
     <div class="pure-u-1 pure-u-lg-1-5">
-        <% if page > 1 %>
+        <%- if query.page > 1 -%>
-            <a href="/search?q=<%= search_query_encoded %>&page=<%= page - 1 %>">
+            <a href="<%= url_prev_page %>"><%= translate(locale, "Previous page") %></a>
-                <%= translate(locale, "Previous page") %>
+        <%- end -%>
-            </a>
-        <% end %>
     </div>
     <div class="pure-u-1 pure-u-lg-3-5"></div>
     <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
-        <% if videos.size >= 20 %>
+        <%- if videos.size >= 20 -%>
-            <a href="/search?q=<%= search_query_encoded %>&page=<%= page + 1 %>">
+            <a href="<%= url_next_page %>"><%= translate(locale, "Next page") %></a>
-                <%= translate(locale, "Next page") %>
+        <%- end -%>
-            </a>
-        <% end %>
     </div>
 </div>