Merge branch 'master' into master

2024-08-15 00:43:26 +00:00 · 2019-01-25 12:48:10 +02:00 · 2019-01-25 12:48:10 +02:00 · 209d7117fb
commit 209d7117fb
parent 3751d11a0b 1af86f6afb
7 changed files with 109 additions and 88 deletions
--- a/README.md
+++ b/README.md
@ -32,6 +32,8 @@ Onion links:
 - kgg2m7yk5aybusll.onion
 - axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid.onion

+[Alternative Invidious instances](https://github.com/omarroth/invidious/wiki/Invidious-Instances)
+
 ## Installation

 ### Docker:
@ -57,71 +59,94 @@ $ docker volume rm invidious_postgresdata
 $ docker-compose build
 ```

-### Arch Linux:
+### Linux:
+
+#### Install dependencies

 ```bash
-# Install dependencies
-$ sudo pacman -S shards crystal imagemagick librsvg
+# Arch Linux
+$ sudo pacman -S shards crystal imagemagick librsvg postgresql

-# Setup PostgresSQL
-$ sudo systemctl enable postgresql
-$ sudo systemctl start postgresql
-$ sudo -i -u postgres
-$ createuser -s YOUR_USER_NAME
-$ createdb      YOUR_USER_NAME
-$ exit
-
-# Setup Invidious
-$ git clone https://github.com/omarroth/invidious
-$ cd invidious
-$ ./setup.sh
-$ shards
-$ crystal build src/invidious.cr --release
-```
-
-### On Ubuntu:
-
-```bash
-# Install dependencies
+# Ubuntu or Debian
 $ curl -sSL https://dist.crystal-lang.org/apt/setup.sh | sudo bash
 $ sudo apt update
 $ sudo apt install crystal libssl-dev libxml2-dev libyaml-dev libgmp-dev libreadline-dev librsvg2-dev postgresql imagemagick libsqlite3-dev
+```

-# Setup PostgreSQL
+#### Add invidious user and clone repository
+
+```bash
+$ useradd -m invidious
+$ sudo -i -u invidious
+$ git clone https://github.com/omarroth/invidious
+$ exit
+```
+
+#### Setup PostgresSQL
+
+```bash
 $ sudo systemctl enable postgresql
 $ sudo systemctl start postgresql
 $ sudo -i -u postgres
-$ createuser -s YOUR_USER_NAME_HERE
-$ createdb      YOUR_USER_NAME_HERE
+$ psql -c "CREATE USER kemal WITH PASSWORD 'kemal';"
+$ createdb -O kemal invidious
+$ psql invidious < /home/invidious/invidious/config/sql/channels.sql
+$ psql invidious < /home/invidious/invidious/config/sql/videos.sql
+$ psql invidious < /home/invidious/invidious/config/sql/channel_videos.sql
+$ psql invidious < /home/invidious/invidious/config/sql/users.sql
+$ psql invidious < /home/invidious/invidious/config/sql/nonces.sql
 $ exit
-
-# Setup Invidious
-$ git clone https://github.com/omarroth/invidious
-$ cd invidious
-$ ./setup.sh
-$ shards
-$ crystal build src/invidious.cr --release
 ```

-### On OSX:
+#### Setup Invidious
+
+```bash
+$ sudo -i -u invidious
+$ cd invidious
+$ shards
+$ crystal build src/invidious.cr --release
+# test compiled binary
+$ ./invidious # stop with ctrl c
+$ exit
+```
+
+#### systemd service
+```bash
+$ sudo cp invidious.service /etc/systemd/system/invidious.service
+$ sudo systemctl enable invidious.service
+$ sudo systemctl start invidious.service
+```
+
+### OSX:

 ```bash
 # Install dependencies
 $ brew update
 $ brew install shards crystal-lang postgres imagemagick librsvg

-# Setup Invidious
+# Clone repository and setup postgres database
 $ git clone https://github.com/omarroth/invidious
 $ cd invidious
-$ ./setup.sh
+$ brew services start postgresql
+$ psql -c "CREATE ROLE kemal WITH LOGIN PASSWORD 'kemal';"
+$ createdb invidious -U kemal
+$ psql invidious < config/sql/channels.sql
+$ psql invidious < config/sql/videos.sql
+$ psql invidious < config/sql/channel_videos.sql
+$ psql invidious < config/sql/users.sql
+$ psql invidious < config/sql/nonces.sql
+
+# Setup Invidious
 $ shards
 $ crystal build src/invidious.cr --release
 ```

+## Update Invidious
+You can find information about how to update in the wiki: [Update Invidious](https://github.com/omarroth/invidious/wiki/Update-Invidious)
+
 ## Usage:

 ```bash
-$ crystal build src/invidious.cr --release
 $ ./invidious -h
 Usage: invidious [arguments]
    -b HOST, --bind HOST             Host to bind (defaults to 0.0.0.0)
@ -131,13 +156,14 @@ Usage: invidious [arguments]
    --ssl-cert-file FILE             SSL certificate file
    -h, --help                       Shows this help
    -t THREADS, --crawl-threads=THREADS
-                                     Number of threads for crawling (default: 1)
+                                     Number of threads for crawling YouTube (default: 0)
    -c THREADS, --channel-threads=THREADS
                                     Number of threads for refreshing channels (default: 1)
    -f THREADS, --feed-threads=THREADS
                                     Number of threads for refreshing feeds (default: 1)
    -v THREADS, --video-threads=THREADS
-                                     Number of threads for refreshing videos (default: 1)
+                                     Number of threads for refreshing videos (default: 0)
+    -o OUTPUT, --output=OUTPUT       Redirect output (default: STDOUT)
 ```

 Or for development:
@ -147,15 +173,8 @@ $ curl -fsSLo- https://raw.githubusercontent.com/samueleaton/sentry/master/insta
 $ ./sentry
 ```

-## Optional
-
-Create a systemd service to run Invidious in background. Edit `invidious.service` to change your installation path and log location. Than copy and enable the systemd service.
-
-```
-$ sudo cp invidious.service /etc/systemd/system/invidious.service
-$ sudo systemctl enable invidious.service
-$ sudo systemctl start invidious.service
-```
+## Documentation
+[Documentation](https://github.com/omarroth/invidious/wiki) can be found in the wiki.

 ## Extensions

--- a/docker/Dockerfile.postgres
+++ b/docker/Dockerfile.postgres
@ -2,7 +2,6 @@ FROM postgres:10

 ENV POSTGRES_USER postgres

-ADD ./setup.sh /setup.sh
 ADD ./config/sql /config/sql
 ADD ./docker/entrypoint.postgres.sh /entrypoint.sh

--- a/docker/entrypoint.postgres.sh
+++ b/docker/entrypoint.postgres.sh
@ -10,7 +10,14 @@ if [ ! -f /var/lib/postgresql/data/setupFinished ]; then
        sleep 5
    done
    >&2 echo "### importing table schemas"
-    su postgres -c "/setup.sh" && touch /var/lib/postgresql/data/setupFinished
+    su postgres -c 'createdb invidious'
+    su postgres -c 'psql -c "CREATE USER kemal WITH PASSWORD '"'kemal'"'"'
+    su postgres -c 'psql invidious < config/sql/channels.sql'
+    su postgres -c 'psql invidious < config/sql/videos.sql'
+    su postgres -c 'psql invidious < config/sql/channel_videos.sql'
+    su postgres -c 'psql invidious < config/sql/users.sql'
+    su postgres -c 'psql invidious < config/sql/nonces.sql'
+    touch /var/lib/postgresql/data/setupFinished
    echo "### invidious database setup finished"
    exit
 fi
--- a/invidious.service
+++ b/invidious.service
@ -7,19 +7,13 @@ After=network.target
 RestartSec=2s
 Type=simple

-# set user and group
 User=invidious
 Group=invidious

-# configure location
-WorkingDirectory=/home/invidious
-ExecStart=/home/invidious/invidious/invidious
+WorkingDirectory=/home/invidious/invidious
+ExecStart=/home/invidious/invidious/invidious -o invidious.log

 Restart=always

-# default log output is syslog, to disable log enable both
-#StandardOutput=null
-#StandardError=null
-
 [Install]
 WantedBy=multi-user.target
--- a/setup.sh
+++ b/setup.sh
@ -1,10 +0,0 @@
-#!/bin/bash
-
-createdb invidious
-#createuser kemal
-psql -c "CREATE USER kemal WITH PASSWORD 'kemal';"
-psql invidious < config/sql/channels.sql
-psql invidious < config/sql/videos.sql
-psql invidious < config/sql/channel_videos.sql
-psql invidious < config/sql/users.sql
-psql invidious < config/sql/nonces.sql
--- a/src/invidious.cr
+++ b/src/invidious.cr
@ -108,17 +108,17 @@ LOCALES = {

 crawl_threads.times do
  spawn do
-    crawl_videos(PG_DB)
+    crawl_videos(PG_DB, logger)
  end
 end

-refresh_channels(PG_DB, channel_threads, CONFIG.full_refresh)
+refresh_channels(PG_DB, logger, channel_threads, CONFIG.full_refresh)

-refresh_feeds(PG_DB, feed_threads)
+refresh_feeds(PG_DB, logger, feed_threads)

 video_threads.times do |i|
  spawn do
-    refresh_videos(PG_DB)
+    refresh_videos(PG_DB, logger)
  end
 end

@ -126,6 +126,8 @@ top_videos = [] of Video
 spawn do
  pull_top_videos(CONFIG, PG_DB) do |videos|
    top_videos = videos
+    sleep 1.minutes
+    Fiber.yield
  end
 end

@ -133,6 +135,8 @@ popular_videos = [] of ChannelVideo
 spawn do
  pull_popular_videos(PG_DB) do |videos|
    popular_videos = videos
+    sleep 1.minutes
+    Fiber.yield
  end
 end

@ -3644,9 +3648,23 @@ get "/videoplayback" do |env|
  host = "https://r#{fvip}---#{mn}.googlevideo.com"
  url = "/videoplayback?#{query_params.to_s}"

+  headers = env.request.headers
+  headers.delete("Host")
+  headers.delete("Cookie")
+  headers.delete("User-Agent")
+  headers.delete("Referer")
+
  region = query_params["region"]?
-  client = make_client(URI.parse(host), proxies, region)
-  response = client.head(url)
+
+  response = HTTP::Client::Response.new(403)
+  loop do
+    begin
+      client = make_client(URI.parse(host), proxies, region)
+      response = client.head(url, headers)
+      break
+    rescue ex
+    end
+  end

  if response.headers["Location"]?
    url = URI.parse(response.headers["Location"])
@ -3664,12 +3682,6 @@ get "/videoplayback" do |env|
    halt env, status_code: 403
  end

-  headers = env.request.headers
-  headers.delete("Host")
-  headers.delete("Cookie")
-  headers.delete("User-Agent")
-  headers.delete("Referer")
-
  client = make_client(URI.parse(host), proxies, region)
  client.get(url, headers) do |response|
    env.response.status_code = response.status_code
--- a/src/invidious/jobs.cr
+++ b/src/invidious/jobs.cr
@ -1,4 +1,4 @@
-def crawl_videos(db)
+def crawl_videos(db, logger)
  ids = Deque(String).new
  random = Random.new

@ -21,7 +21,7 @@ def crawl_videos(db)
      id = ids[0]
      video = get_video(id, db)
    rescue ex
-      STDOUT << id << " : " << ex.message << "\n"
+      logger.write("#{id} : #{ex.message}\n")
      next
    ensure
      ids.delete(id)
@ -46,7 +46,7 @@ def crawl_videos(db)
  end
 end

-def refresh_channels(db, max_threads = 1, full_refresh = false)
+def refresh_channels(db, logger, max_threads = 1, full_refresh = false)
  max_channel = Channel(Int32).new

  spawn do
@ -73,7 +73,7 @@ def refresh_channels(db, max_threads = 1, full_refresh = false)

              db.exec("UPDATE channels SET updated = $1, author = $2 WHERE id = $3", Time.now, channel.author, id)
            rescue ex
-              STDOUT << id << " : " << ex.message << "\n"
+              logger.write("#{id} : #{ex.message}\n")
            end

            active_channel.send(true)
@ -86,7 +86,7 @@ def refresh_channels(db, max_threads = 1, full_refresh = false)
  max_channel.send(max_threads)
 end

-def refresh_videos(db)
+def refresh_videos(db, logger)
  loop do
    db.query("SELECT id FROM videos ORDER BY updated") do |rs|
      rs.each do
@ -94,7 +94,7 @@ def refresh_videos(db)
          id = rs.read(String)
          video = get_video(id, db)
        rescue ex
-          STDOUT << id << " : " << ex.message << "\n"
+          logger.write("#{id} : #{ex.message}\n")
          next
        end
      end
@ -104,7 +104,7 @@ def refresh_videos(db)
  end
 end

-def refresh_feeds(db, max_threads = 1)
+def refresh_feeds(db, logger, max_threads = 1)
  max_channel = Channel(Int32).new

  spawn do
@ -129,7 +129,7 @@ def refresh_feeds(db, max_threads = 1)
            begin
              db.exec("REFRESH MATERIALIZED VIEW #{view_name}")
            rescue ex
-              STDOUT << "REFRESH " << email << " : " << ex.message << "\n"
+              logger.write("REFRESH #{email} : #{ex.message}\n")
            end

            active_channel.send(true)