mirror of
https://gitea.invidious.io/iv-org/instances-api.git
synced 2024-08-15 00:53:15 +00:00
Merge ca8a99c9d1 into e46bf0d710
This commit is contained in:
syeopite
GitHub
commit
37f58fa6bc
6 changed files with 164 additions and 40 deletions
|
|
@ -6,11 +6,12 @@ COPY ./src/ ./src/
|
|||
RUN crystal build ./src/instances.cr --release
|
||||
|
||||
FROM alpine:latest
|
||||
RUN apk add --no-cache gc pcre libgcc
|
||||
RUN apk add --no-cache gc pcre libgcc yaml
|
||||
WORKDIR /app
|
||||
RUN addgroup -g 1000 -S invidious && \
|
||||
adduser -u 1000 -S invidious -G invidious
|
||||
COPY ./assets/ ./assets/
|
||||
COPY ./config.yml ./config.yml
|
||||
COPY --from=builder /app/instances .
|
||||
|
||||
EXPOSE 3000
|
||||
|
|
|
|||
9
config.yml
Normal file
9
config.yml
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
# Uses the system's CURL binary to do so.
|
||||
fetch_onion_instance_stats: true
|
||||
|
||||
# TOR's Sock proxy address that CURL uses to connect to hidden services
|
||||
tor_sock_proxy_address: "127.0.0.1"
|
||||
tor_sock_proxy_port: 9050
|
||||
|
||||
# Minutes before refreshing the instance stats
|
||||
minutes_between_refresh: 30
|
||||
|
|
@ -1,5 +1,13 @@
|
|||
version: '3'
|
||||
services:
|
||||
tor-socks-proxy:
|
||||
container_name: tor-socks-proxy
|
||||
image: peterdavehello/tor-socks-proxy:latest
|
||||
ports:
|
||||
- "127.0.0.1:8853:53/udp"
|
||||
- "127.0.0.1:9150:9150/tcp"
|
||||
restart: unless-stopped
|
||||
|
||||
instances:
|
||||
build: .
|
||||
restart: unless-stopped
|
||||
|
|
|
|||
117
src/fetch.cr
Normal file
117
src/fetch.cr
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
def fetch_country(md)
|
||||
region = md["flag"]?.try { |region| region.codepoints.map { |codepoint| (codepoint - 0x1f1a5).chr }.join("") }
|
||||
flag = md["flag"]?
|
||||
country_name = md["country_name"]?
|
||||
|
||||
return flag, region, country_name
|
||||
end
|
||||
|
||||
def fetch_notes(md)
|
||||
notes = md["notes"].strip("|")
|
||||
if notes.empty?
|
||||
notes = nil
|
||||
end
|
||||
|
||||
return notes
|
||||
end
|
||||
|
||||
def prepare_http_instance(md, instances, monitors)
|
||||
uri = URI.parse(md["uri"])
|
||||
host = md["host"]
|
||||
|
||||
flag, region, country_name = fetch_country(md)
|
||||
|
||||
status_url = md["status_url"]?
|
||||
|
||||
privacy_policy = md["privacy_policy"]?
|
||||
|
||||
ddos_protection = md["ddos_protection"].strip
|
||||
if ddos_protection == "None"
|
||||
ddos_protection = nil
|
||||
end
|
||||
|
||||
owner = {name: md["owner"].strip("@"), url: md["owner_url"]}
|
||||
notes = fetch_notes(md)
|
||||
|
||||
client = HTTP::Client.new(uri)
|
||||
client.connect_timeout = 5.seconds
|
||||
client.read_timeout = 5.seconds
|
||||
|
||||
begin
|
||||
stats = JSON.parse(client.get("/api/v1/stats").body)
|
||||
rescue ex
|
||||
stats = nil
|
||||
end
|
||||
|
||||
monitor = monitors.try &.select { |monitor| monitor["name"].try &.as_s == host }[0]?
|
||||
return {flag: flag, region: region, country_name: country_name, stats: stats, type: "https", uri: uri.to_s, status_url: status_url,
|
||||
privacy_policy: privacy_policy, ddos_protection: ddos_protection,
|
||||
owner: owner, notes: notes, monitor: monitor || instances[host]?.try &.[:monitor]?}
|
||||
end
|
||||
|
||||
def prepare_onion_instance(md, instances)
|
||||
uri = URI.parse(md["uri"])
|
||||
host = md["host"]
|
||||
|
||||
clearnet_url = md["clearnet_url"]
|
||||
flag, region, country_name = fetch_country(md)
|
||||
privacy_policy = md["privacy_policy"]?
|
||||
owner = {name: md["owner"].strip("@"), url: md["owner_url"]}
|
||||
notes = fetch_notes(md)
|
||||
|
||||
if CONFIG["fetch_onion_instance_stats"]?
|
||||
begin
|
||||
args = Process.parse_arguments("--socks5-hostname '#{CONFIG["tor_sock_proxy_address"]}:#{CONFIG["tor_sock_proxy_port"]}' 'http://#{uri.host}/api/v1/stats'")
|
||||
response = nil
|
||||
Process.run("curl", args: args) do |result|
|
||||
data = result.output.read_line
|
||||
response = JSON.parse(data)
|
||||
end
|
||||
|
||||
stats = response
|
||||
rescue ex
|
||||
stats = nil
|
||||
end
|
||||
else
|
||||
stats = nil
|
||||
end
|
||||
|
||||
return {flag: flag, region: region, country_name: country_name, stats: stats, type: "onion", uri: uri.to_s, clearnet_url: clearnet_url,
|
||||
privacy_policy: privacy_policy, owner: owner, notes: notes,
|
||||
monitor: nil}
|
||||
end
|
||||
|
||||
def get_clearnet_instances(body, instances, monitors)
|
||||
# Crystal currently lacks a markdown parser that supports tables. So...
|
||||
clear_net_regexes = [
|
||||
/\[(?<host>[^ \]]+)\]\((?<uri>[^\)]+)\)/, # Address column
|
||||
/(?<flag>[\x{1f100}-\x{1f1ff}]{2}) (?<country_name>[^ ]+)/, # Country column
|
||||
/((\[[^\]]+\]\(.*\){1}\])\((?<status_url>.*)\)|(None))/, # Status column
|
||||
/((\[[^ \]]+\]\((?<privacy_policy>[^\)]+)\))|(None))/, # privacy policy column
|
||||
/(?<ddos_protection>.*)/, # DDOS protection column
|
||||
/\[(?<owner>[^ \]]+)\]\((?<owner_url>[^\)]+)\)/, # Owner column
|
||||
/(?<notes>.*)/, # Note column
|
||||
]
|
||||
|
||||
body.scan(/#{clear_net_regexes.join(/ +\| +/)}/mx).each do |md|
|
||||
host = md["host"]
|
||||
instances[host] = prepare_http_instance(md, instances, monitors)
|
||||
end
|
||||
end
|
||||
|
||||
def get_onion_instances(body, instances)
|
||||
# Crystal currently lacks a markdown parser that supports tables. So...
|
||||
clear_net_regexes = [
|
||||
/\[(?<host>[^ \]]+)\]\((?<uri>[^\)]+)\)/, # Address column
|
||||
/(?<flag>[\x{1f100}-\x{1f1ff}]{2}) (?<country_name>[^ ]+)/, # Country column
|
||||
/\[(?<clearnet_host>[^ \]]+)\]\((?<clearnet_url>[^\)]+)\)/, # Clearnet instance column
|
||||
/((\[[^ \]]+\]\((?<privacy_policy>[^\)]+)\))|(None))/, # privacy policy column
|
||||
/\[(?<owner>[^ \]]+)\]\((?<owner_url>[^\)]+)\)/, # Owner column
|
||||
/(?<notes>.*)/, # Notes column
|
||||
]
|
||||
|
||||
body.scan(/#{clear_net_regexes.join(/ +\| +/)}/mx).each do |md|
|
||||
host = md["host"]
|
||||
instances[host] = prepare_onion_instance(md, instances)
|
||||
end
|
||||
end
|
||||
6
src/helpers/helpers.cr
Normal file
6
src/helpers/helpers.cr
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
require "yaml"
|
||||
|
||||
def load_config
|
||||
config = YAML.parse(File.read("config.yml"))
|
||||
return config
|
||||
end
|
||||
|
|
@ -18,15 +18,22 @@ require "http/client"
|
|||
require "kemal"
|
||||
require "uri"
|
||||
|
||||
require "./fetch.cr"
|
||||
require "./helpers/helpers.cr"
|
||||
|
||||
CONFIG = load_config()
|
||||
|
||||
Kemal::CLI.new ARGV
|
||||
|
||||
macro rendered(filename)
|
||||
render "src/instances/views/#{{{filename}}}.ecr"
|
||||
end
|
||||
|
||||
alias Instance = NamedTuple(flag: String?, region: String?, stats: JSON::Any?, type: String, uri: String, monitor: JSON::Any?)
|
||||
alias Owner = NamedTuple(name: String, url: String)
|
||||
alias ClearNetInstance = NamedTuple(flag: String?, region: String?, country_name: String?, stats: JSON::Any?, type: String, uri: String, status_url: String?, privacy_policy: String?, ddos_protection: String?, owner: Owner, notes: String?, monitor: JSON::Any?)
|
||||
alias OnionInstance = NamedTuple(flag: String?, region: String?, country_name: String?, stats: JSON::Any?, type: String, uri: String, clearnet_url: String?, privacy_policy: String?, owner: Owner, notes: String?, monitor: JSON::Any?)
|
||||
|
||||
INSTANCES = {} of String => Instance
|
||||
INSTANCES = {} of String => ClearNetInstance | OnionInstance
|
||||
|
||||
spawn do
|
||||
loop do
|
||||
|
|
@ -51,44 +58,20 @@ spawn do
|
|||
end
|
||||
end
|
||||
begin
|
||||
body = HTTP::Client.get(URI.parse("https://raw.githubusercontent.com/iv-org/documentation/master/Invidious-Instances.md")).body
|
||||
# Needs to be replaced once merged!
|
||||
body = HTTP::Client.get(URI.parse("https://raw.githubusercontent.com/TheFrenchGhosty/documentation/instances-list-rewrite/Public-Instances.md")).body
|
||||
rescue ex
|
||||
body = ""
|
||||
end
|
||||
|
||||
instances = {} of String => Instance
|
||||
|
||||
body = body.split("### Blocked:")[0]
|
||||
body.scan(/\[(?<host>[^ \]]+)\]\((?<uri>[^\)]+)\)( .(?<region>[\x{1f100}-\x{1f1ff}]{2}))?/mx).each do |md|
|
||||
region = md["region"]?.try { |region| region.codepoints.map { |codepoint| (codepoint - 0x1f1a5).chr }.join("") }
|
||||
flag = md["region"]?
|
||||
|
||||
uri = URI.parse(md["uri"])
|
||||
host = md["host"]
|
||||
|
||||
case type = host.split(".")[-1]
|
||||
when "onion"
|
||||
when "i2p"
|
||||
else
|
||||
type = uri.scheme.not_nil!
|
||||
client = HTTP::Client.new(uri)
|
||||
client.connect_timeout = 5.seconds
|
||||
client.read_timeout = 5.seconds
|
||||
begin
|
||||
stats = JSON.parse(client.get("/api/v1/stats").body)
|
||||
rescue ex
|
||||
stats = nil
|
||||
end
|
||||
end
|
||||
|
||||
monitor = monitors.try &.select { |monitor| monitor["name"].try &.as_s == host }[0]?
|
||||
instances[host] = {flag: flag, region: region, stats: stats, type: type, uri: uri.to_s, monitor: monitor || instances[host]?.try &.[:monitor]?}
|
||||
end
|
||||
instances = {} of String => ClearNetInstance | OnionInstance
|
||||
get_clearnet_instances(body, instances, monitors)
|
||||
get_onion_instances(body, instances)
|
||||
|
||||
INSTANCES.clear
|
||||
INSTANCES.merge! instances
|
||||
|
||||
sleep 5.minutes
|
||||
sleep CONFIG["minutes_between_refresh"].as_i.minutes
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -134,13 +117,13 @@ static_headers do |response, filepath, filestat|
|
|||
end
|
||||
|
||||
SORT_PROCS = {
|
||||
"health" => ->(name : String, instance : Instance) { -(instance[:monitor]?.try &.["30dRatio"]["ratio"].as_s.to_f || 0.0) },
|
||||
"location" => ->(name : String, instance : Instance) { instance[:region]? || "ZZ" },
|
||||
"name" => ->(name : String, instance : Instance) { name },
|
||||
"signup" => ->(name : String, instance : Instance) { instance[:stats]?.try &.["openRegistrations"]?.try { |bool| bool.as_bool ? 0 : 1 } || 2 },
|
||||
"type" => ->(name : String, instance : Instance) { instance[:type] },
|
||||
"users" => ->(name : String, instance : Instance) { -(instance[:stats]?.try &.["usage"]?.try &.["users"]["total"].as_i || 0) },
|
||||
"version" => ->(name : String, instance : Instance) { instance[:stats]?.try &.["software"]?.try &.["version"].as_s.try &.split("-", 2)[0].split(".").map { |a| -a.to_i } || [0, 0, 0] },
|
||||
"health" => ->(name : String, instance : ClearNetInstance | OnionInstance) { -(instance[:monitor]?.try &.["30dRatio"]["ratio"].as_s.to_f || 0.0) },
|
||||
"location" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:region]? || "ZZ" },
|
||||
"name" => ->(name : String, instance : ClearNetInstance | OnionInstance) { name },
|
||||
"signup" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:stats]?.try &.["openRegistrations"]?.try { |bool| bool.as_bool ? 0 : 1 } || 2 },
|
||||
"type" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:type] },
|
||||
"users" => ->(name : String, instance : ClearNetInstance | OnionInstance) { -(instance[:stats]?.try &.["usage"]?.try &.["users"]["total"].as_i || 0) },
|
||||
"version" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:stats]?.try &.["software"]?.try &.["version"].as_s.try &.split("-", 2)[0].split(".").map { |a| -a.to_i } || [0, 0, 0] },
|
||||
}
|
||||
|
||||
def sort_instances(instances, sort_by)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue