Merge branch 'master' into master

This commit is contained in:
Esmail EL BoB 2019-01-25 12:48:10 +02:00 committed by GitHub
commit 209d7117fb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 109 additions and 88 deletions

115
README.md
View file

@ -32,6 +32,8 @@ Onion links:
- kgg2m7yk5aybusll.onion
- axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid.onion
[Alternative Invidious instances](https://github.com/omarroth/invidious/wiki/Invidious-Instances)
## Installation
### Docker:
@ -57,71 +59,94 @@ $ docker volume rm invidious_postgresdata
$ docker-compose build
```
### Arch Linux:
### Linux:
#### Install dependencies
```bash
# Install dependencies
$ sudo pacman -S shards crystal imagemagick librsvg
# Arch Linux
$ sudo pacman -S shards crystal imagemagick librsvg postgresql
# Setup PostgresSQL
$ sudo systemctl enable postgresql
$ sudo systemctl start postgresql
$ sudo -i -u postgres
$ createuser -s YOUR_USER_NAME
$ createdb YOUR_USER_NAME
$ exit
# Setup Invidious
$ git clone https://github.com/omarroth/invidious
$ cd invidious
$ ./setup.sh
$ shards
$ crystal build src/invidious.cr --release
```
### On Ubuntu:
```bash
# Install dependencies
# Ubuntu or Debian
$ curl -sSL https://dist.crystal-lang.org/apt/setup.sh | sudo bash
$ sudo apt update
$ sudo apt install crystal libssl-dev libxml2-dev libyaml-dev libgmp-dev libreadline-dev librsvg2-dev postgresql imagemagick libsqlite3-dev
```
# Setup PostgreSQL
#### Add invidious user and clone repository
```bash
$ useradd -m invidious
$ sudo -i -u invidious
$ git clone https://github.com/omarroth/invidious
$ exit
```
#### Setup PostgresSQL
```bash
$ sudo systemctl enable postgresql
$ sudo systemctl start postgresql
$ sudo -i -u postgres
$ createuser -s YOUR_USER_NAME_HERE
$ createdb YOUR_USER_NAME_HERE
$ psql -c "CREATE USER kemal WITH PASSWORD 'kemal';"
$ createdb -O kemal invidious
$ psql invidious < /home/invidious/invidious/config/sql/channels.sql
$ psql invidious < /home/invidious/invidious/config/sql/videos.sql
$ psql invidious < /home/invidious/invidious/config/sql/channel_videos.sql
$ psql invidious < /home/invidious/invidious/config/sql/users.sql
$ psql invidious < /home/invidious/invidious/config/sql/nonces.sql
$ exit
# Setup Invidious
$ git clone https://github.com/omarroth/invidious
$ cd invidious
$ ./setup.sh
$ shards
$ crystal build src/invidious.cr --release
```
### On OSX:
#### Setup Invidious
```bash
$ sudo -i -u invidious
$ cd invidious
$ shards
$ crystal build src/invidious.cr --release
# test compiled binary
$ ./invidious # stop with ctrl c
$ exit
```
#### systemd service
```bash
$ sudo cp invidious.service /etc/systemd/system/invidious.service
$ sudo systemctl enable invidious.service
$ sudo systemctl start invidious.service
```
### OSX:
```bash
# Install dependencies
$ brew update
$ brew install shards crystal-lang postgres imagemagick librsvg
# Setup Invidious
# Clone repository and setup postgres database
$ git clone https://github.com/omarroth/invidious
$ cd invidious
$ ./setup.sh
$ brew services start postgresql
$ psql -c "CREATE ROLE kemal WITH LOGIN PASSWORD 'kemal';"
$ createdb invidious -U kemal
$ psql invidious < config/sql/channels.sql
$ psql invidious < config/sql/videos.sql
$ psql invidious < config/sql/channel_videos.sql
$ psql invidious < config/sql/users.sql
$ psql invidious < config/sql/nonces.sql
# Setup Invidious
$ shards
$ crystal build src/invidious.cr --release
```
## Update Invidious
You can find information about how to update in the wiki: [Update Invidious](https://github.com/omarroth/invidious/wiki/Update-Invidious)
## Usage:
```bash
$ crystal build src/invidious.cr --release
$ ./invidious -h
Usage: invidious [arguments]
-b HOST, --bind HOST Host to bind (defaults to 0.0.0.0)
@ -131,13 +156,14 @@ Usage: invidious [arguments]
--ssl-cert-file FILE SSL certificate file
-h, --help Shows this help
-t THREADS, --crawl-threads=THREADS
Number of threads for crawling (default: 1)
Number of threads for crawling YouTube (default: 0)
-c THREADS, --channel-threads=THREADS
Number of threads for refreshing channels (default: 1)
-f THREADS, --feed-threads=THREADS
Number of threads for refreshing feeds (default: 1)
-v THREADS, --video-threads=THREADS
Number of threads for refreshing videos (default: 1)
Number of threads for refreshing videos (default: 0)
-o OUTPUT, --output=OUTPUT Redirect output (default: STDOUT)
```
Or for development:
@ -147,15 +173,8 @@ $ curl -fsSLo- https://raw.githubusercontent.com/samueleaton/sentry/master/insta
$ ./sentry
```
## Optional
Create a systemd service to run Invidious in background. Edit `invidious.service` to change your installation path and log location. Than copy and enable the systemd service.
```
$ sudo cp invidious.service /etc/systemd/system/invidious.service
$ sudo systemctl enable invidious.service
$ sudo systemctl start invidious.service
```
## Documentation
[Documentation](https://github.com/omarroth/invidious/wiki) can be found in the wiki.
## Extensions

View file

@ -2,7 +2,6 @@ FROM postgres:10
ENV POSTGRES_USER postgres
ADD ./setup.sh /setup.sh
ADD ./config/sql /config/sql
ADD ./docker/entrypoint.postgres.sh /entrypoint.sh

View file

@ -10,7 +10,14 @@ if [ ! -f /var/lib/postgresql/data/setupFinished ]; then
sleep 5
done
>&2 echo "### importing table schemas"
su postgres -c "/setup.sh" && touch /var/lib/postgresql/data/setupFinished
su postgres -c 'createdb invidious'
su postgres -c 'psql -c "CREATE USER kemal WITH PASSWORD '"'kemal'"'"'
su postgres -c 'psql invidious < config/sql/channels.sql'
su postgres -c 'psql invidious < config/sql/videos.sql'
su postgres -c 'psql invidious < config/sql/channel_videos.sql'
su postgres -c 'psql invidious < config/sql/users.sql'
su postgres -c 'psql invidious < config/sql/nonces.sql'
touch /var/lib/postgresql/data/setupFinished
echo "### invidious database setup finished"
exit
fi

View file

@ -7,19 +7,13 @@ After=network.target
RestartSec=2s
Type=simple
# set user and group
User=invidious
Group=invidious
# configure location
WorkingDirectory=/home/invidious
ExecStart=/home/invidious/invidious/invidious
WorkingDirectory=/home/invidious/invidious
ExecStart=/home/invidious/invidious/invidious -o invidious.log
Restart=always
# default log output is syslog, to disable log enable both
#StandardOutput=null
#StandardError=null
[Install]
WantedBy=multi-user.target

View file

@ -1,10 +0,0 @@
#!/bin/bash
createdb invidious
#createuser kemal
psql -c "CREATE USER kemal WITH PASSWORD 'kemal';"
psql invidious < config/sql/channels.sql
psql invidious < config/sql/videos.sql
psql invidious < config/sql/channel_videos.sql
psql invidious < config/sql/users.sql
psql invidious < config/sql/nonces.sql

View file

@ -108,17 +108,17 @@ LOCALES = {
crawl_threads.times do
spawn do
crawl_videos(PG_DB)
crawl_videos(PG_DB, logger)
end
end
refresh_channels(PG_DB, channel_threads, CONFIG.full_refresh)
refresh_channels(PG_DB, logger, channel_threads, CONFIG.full_refresh)
refresh_feeds(PG_DB, feed_threads)
refresh_feeds(PG_DB, logger, feed_threads)
video_threads.times do |i|
spawn do
refresh_videos(PG_DB)
refresh_videos(PG_DB, logger)
end
end
@ -126,6 +126,8 @@ top_videos = [] of Video
spawn do
pull_top_videos(CONFIG, PG_DB) do |videos|
top_videos = videos
sleep 1.minutes
Fiber.yield
end
end
@ -133,6 +135,8 @@ popular_videos = [] of ChannelVideo
spawn do
pull_popular_videos(PG_DB) do |videos|
popular_videos = videos
sleep 1.minutes
Fiber.yield
end
end
@ -3644,9 +3648,23 @@ get "/videoplayback" do |env|
host = "https://r#{fvip}---#{mn}.googlevideo.com"
url = "/videoplayback?#{query_params.to_s}"
headers = env.request.headers
headers.delete("Host")
headers.delete("Cookie")
headers.delete("User-Agent")
headers.delete("Referer")
region = query_params["region"]?
client = make_client(URI.parse(host), proxies, region)
response = client.head(url)
response = HTTP::Client::Response.new(403)
loop do
begin
client = make_client(URI.parse(host), proxies, region)
response = client.head(url, headers)
break
rescue ex
end
end
if response.headers["Location"]?
url = URI.parse(response.headers["Location"])
@ -3664,12 +3682,6 @@ get "/videoplayback" do |env|
halt env, status_code: 403
end
headers = env.request.headers
headers.delete("Host")
headers.delete("Cookie")
headers.delete("User-Agent")
headers.delete("Referer")
client = make_client(URI.parse(host), proxies, region)
client.get(url, headers) do |response|
env.response.status_code = response.status_code

View file

@ -1,4 +1,4 @@
def crawl_videos(db)
def crawl_videos(db, logger)
ids = Deque(String).new
random = Random.new
@ -21,7 +21,7 @@ def crawl_videos(db)
id = ids[0]
video = get_video(id, db)
rescue ex
STDOUT << id << " : " << ex.message << "\n"
logger.write("#{id} : #{ex.message}\n")
next
ensure
ids.delete(id)
@ -46,7 +46,7 @@ def crawl_videos(db)
end
end
def refresh_channels(db, max_threads = 1, full_refresh = false)
def refresh_channels(db, logger, max_threads = 1, full_refresh = false)
max_channel = Channel(Int32).new
spawn do
@ -73,7 +73,7 @@ def refresh_channels(db, max_threads = 1, full_refresh = false)
db.exec("UPDATE channels SET updated = $1, author = $2 WHERE id = $3", Time.now, channel.author, id)
rescue ex
STDOUT << id << " : " << ex.message << "\n"
logger.write("#{id} : #{ex.message}\n")
end
active_channel.send(true)
@ -86,7 +86,7 @@ def refresh_channels(db, max_threads = 1, full_refresh = false)
max_channel.send(max_threads)
end
def refresh_videos(db)
def refresh_videos(db, logger)
loop do
db.query("SELECT id FROM videos ORDER BY updated") do |rs|
rs.each do
@ -94,7 +94,7 @@ def refresh_videos(db)
id = rs.read(String)
video = get_video(id, db)
rescue ex
STDOUT << id << " : " << ex.message << "\n"
logger.write("#{id} : #{ex.message}\n")
next
end
end
@ -104,7 +104,7 @@ def refresh_videos(db)
end
end
def refresh_feeds(db, max_threads = 1)
def refresh_feeds(db, logger, max_threads = 1)
max_channel = Channel(Int32).new
spawn do
@ -129,7 +129,7 @@ def refresh_feeds(db, max_threads = 1)
begin
db.exec("REFRESH MATERIALIZED VIEW #{view_name}")
rescue ex
STDOUT << "REFRESH " << email << " : " << ex.message << "\n"
logger.write("REFRESH #{email} : #{ex.message}\n")
end
active_channel.send(true)