Add logic to automatically generate instance list

.github/workflows/ci.yml

@@ -0,0 +1,38 @@
+name: Create markdown list
+
+on: 
+  pull_request:
+    branches: "*"
+    paths:
+     - 'instances.yaml'
+
+jobs:
+  build:
+    
+    runs-on: ubuntu-latest
+    
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.head_ref }}
+      
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9.5
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r src/requirements.txt
+      
+      - name: Create markdown instance list
+        run: |
+          python src/create-instance-md.py
+    
+      # Auto commit resulting md file
+      - name: "Auto commiting resulting markdown instance list"
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: Update Invidious-Instances.md          
+          file_pattern: Invidious-Instances.md

instances.yaml

@@ -0,0 +1,286 @@
+adding_instance:
+  prerequisites:
+    - Instances must have been updated in the last month. An instance that hasn't been updated in the last month is considered unmaintained and is removed from the list.
+    - Instances must have statistics (/api/v1/stats) enabled (statistics_enabled:true in the configuration file)
+    - Instances must be served via domain name.
+    - Instances must be served via domain name.
+    - Instances using any DDoS Protection / MITM are marked as such.
+    - Instances using any type of anti-bot protection are marked as such.
+    - "Instances using any type of analytics are marked as such, must be GDPR compliant (if it's usable in the EU), must be CCPA compliant (if it's usable in California), and must respect the AGPL by explaining their changes and by publishing their source code. In short: instances shouldn't run analytics, because it's not worth it."
+    - Instances running a modified source code must respect the AGPL by publishing their source code and stating their changes before they are be added to the list, and must publish any later modification in a timely manner.
+  directions:
+    - Fork the documentation repo.
+    - Open `instances.yaml` for editing."
+    - Append your instance to the bottom of the HTTPS (or onion) list. See the examples in the yaml file for more info.
+    - Make a pull request.
+
+instances:
+  https:
+    # Template instance
+    # - url: example.com
+    #   country:
+    #     flag: 🇫🇮
+    #     name: Finland
+    #   status_url: status.example.com
+    #   privacy_policy: privacy.example.com
+    #   ddos_mitm_protection: Cloudflare
+    #   owner: https://github.com/example
+    #   modified:
+    #     is_modified: False
+    #     source_url: null
+
+
+    - url: https://invidious.snopyta.org
+      country:
+        flag: 🇫🇮
+        name: Finland
+      status: 
+        url: https://status.unixfox.eu/783898765
+        display_content: https://img.shields.io/uptimerobot/status/m783898765-2a4efa67aa8d1c7be6b1dd9d
+        display_content_is_image: True
+        display_content_image_fallback: Uptime Robot status
+      privacy_policy: https://snopyta.org/privacy_policy
+      ddos_mitm_protection: null
+      owner: https://github.com/Perflyst
+      modified:
+        is_modified: False
+        source_url: null
+
+    - url: https://yewtu.be
+      country:
+        flag: 🇫🇷
+        name: France
+      status: null
+      privacy_policy: null
+      ddos_mitm_protection: null
+      owner: https://github.com/unixfox
+      modified:
+        is_modified: False
+        source_url: null
+
+    - url: https://invidious.kavin.rocks
+      country:
+        flag: 🇮🇳
+        name: India
+      status:
+        url: https://status.kavin.rocks/786132664
+        display_content: https://img.shields.io/uptimerobot/status/m786132664-f9fa738fba1c4dc2f7364f71
+        display_content_is_image: True
+        display_content_image_fallback: Uptime Robot status
+      privacy_policy:
+      ddos_mitm_protection: Cloudflare
+      owner: https://github.com/FireMasterK
+      modified:
+        is_modified: False
+        source_url: null
+
+    - url: https://invidious-us.kavin.rocks
+      country:
+        flag: 🇺🇸
+        name: USA
+      status: 
+        url: https://status.kavin.rocks/788216947
+        display_content: https://img.shields.io/uptimerobot/status/m788216947-f3f63d30899a10dbe9a0338a
+        display_content_is_image: True
+        display_content_image_fallback: Uptime Robot status
+      privacy_policy:
+      ddos_mitm_protection: Cloudflare
+      owner: https://github.com/FireMasterK
+      modified:
+        is_modified: False
+        source_url: null
+
+    - url: https://invidious.048596.xyz
+      country:
+        flag: 🇨🇦
+        name: Canada
+      status: 
+        url: https://status.048596.xyz/786792286
+        display_content: https://img.shields.io/uptimerobot/status/m786792286-b5894e4e11c42b8332375076
+        display_content_is_image: True
+        display_content_image_fallback: Uptime Robot status
+      privacy_policy:
+      ddos_mitm_protection:
+      owner: https://github.com/tenpura-shrimp
+      modified:
+        is_modified: False
+        source_url: null
+
+    - url: https://vid.puffyan.us
+      country:
+        flag: 🇺🇸
+        name: USA
+      status: 
+        url: https://stats.uptimerobot.com/n7A08HGVl6/786947233
+        display_content: https://img.shields.io/uptimerobot/status/m786947233-1131c3f67b9a20621b1926d3
+        display_content_is_image: True
+        display_content_image_fallback: Uptime Robot status
+      privacy_policy:
+      ddos_mitm_protection:
+      owner: https://github.com/ItsSt0ne
+      modified:
+        is_modified: False
+        source_url: null
+
+    - url: https://ytprivate.com
+      country:
+        flag: 🇺🇸
+        name: USA
+      status: 
+        url: https://status.ytprivate.com/786947505
+        display_content:  https://img.shields.io/uptimerobot/status/m786947505-2a50cf3262906bb28c6cf8fc
+        display_content_is_image: True
+        display_content_image_fallback: Uptime Robot status
+      privacy_policy:
+      ddos_mitm_protection: DDoS-Guard
+      owner: https://github.com/ytprivatecom
+      modified:
+        is_modified: True
+        source_url: https://github.com/ytprivatecom/invidious
+
+    - url: https://invidious.namazso.eu
+      country:
+        flag: 🇩🇪
+        name: Germany
+      status: 
+      privacy_policy: https://namazso.eu/privacy.html
+      ddos_mitm_protection:
+      owner: https://github.com/namazso
+      modified:
+        is_modified: False
+        source_url: null
+
+    - url: https://invidious.silkky.cloud
+      country:
+        flag: 🇫🇮
+        name: Finland
+      status: 
+        url: https://status.silkky.cloud/787784614
+        display_content:  https://img.shields.io/uptimerobot/status/m787784614-79d1acc4b425d1ed813fc793
+        display_content_is_image: True
+        display_content_image_fallback: Uptime Robot status
+      privacy_policy:
+      ddos_mitm_protection:
+      owner: https://github.com/TheSilkky
+      modified:
+        is_modified: False
+        source_url: null
+
+    - url: https://invidious.exonip.de
+      country:
+        flag: 🇩🇪
+        name: Germany
+      status: 
+        url: https://status.exonip.de/
+        display_content:  Uptime Robot status
+        display_content_is_image: False
+      privacy_policy:
+      ddos_mitm_protection:
+      owner: https://github.com/Exonip
+      modified:
+        is_modified: False
+        source_url: null
+
+    - url: https://notyoutube.org
+      country:
+        flag: 🇫🇮
+        name: Finland
+      status:
+      privacy_policy:
+      ddos_mitm_protection:
+      owner: https://github.com/Eggo-Plant
+      modified:
+        is_modified: False
+        source_url: null
+
+    - url: https://inv.riverside.rocks
+      country:
+        flag: 🇺🇸
+        name: USA
+      status:
+      privacy_policy:
+      ddos_mitm_protection:
+      owner: https://github.com/RiversideRocks
+      modified:
+        is_modified: False
+        source_url: null
+
+    - url: https://invidious.blamefran.net
+      country:
+        flag: 🇺🇸
+        name: USA
+      status:
+      privacy_policy:
+      ddos_mitm_protection:
+      owner: https://github.com/Aidan16
+      modified:
+        is_modified: False
+        source_url: null
+
+  onion:
+    # Template onion instance
+    # - url: http://example.onion
+    #   country:
+    #     flag: 🇺🇸
+    #     name: USA
+    #   associated_clearnet_instance: example.com
+    #   privacy_policy: privacy.example.com
+    #   owner: https://github.com/example
+    #   modified:
+    #     is_modified: False
+    #     source_url: null
+
+    - url: http://c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid.onion
+      country:
+        flag: 🇫🇮
+        name: Finland
+      invidious.snopyta.org: invidious.snopyta.org
+      privacy_policy:
+      owner: https://github.com/Perflyst
+      modified:
+        is_modified: False
+        source_url: null
+
+    - url: http://w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd.onion/
+      country:
+        flag: 🇮🇳
+        name: India
+      associated_clearnet_instance: https://invidious.kavin.rocks
+      privacy_policy:
+      owner: https://github.com/FireMasterK
+      modified:
+        is_modified: False
+        source_url: null
+
+    - url: http://kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad.onion/
+      country:
+        flag: 🇳🇱
+        name: Netherlands
+      associated_clearnet_instance:
+      privacy_policy:
+      owner: https://github.com/tirz
+      modified:
+        is_modified: False
+        source_url: null
+
+    - url: http://grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad.onion/
+      country:
+        flag: 🇺🇸
+        name: USA
+      associated_clearnet_instance: https://vid.puffyan.us
+      privacy_policy:
+      owner: https://github.com/ItsSt0ne
+      modified:
+        is_modified: False
+        source_url: null
+
+    - url: http://hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid.onion/
+      country:
+        flag: 🇺🇸
+        name: USA
+      associated_clearnet_instance: https://invidious-us.kavin.rocks/
+      privacy_policy:
+      owner: https://github.com/FireMasterK
+      modified:
+        is_modified: False
+        source_url: null

src/create-instance-md.py

@@ -0,0 +1,92 @@
+"""Extremely quick and dirty module for creating a markdown file from the instances.yaml file"""
+from urllib.parse import urlparse
+
+import yaml
+from mdutils.mdutils import MdUtils
+
+
+def create_table(table_data, instance_data):
+    rows = []
+    for field, value in instance_data.items():
+        if value is None:
+            rows.append("")
+        
+        # Use markdown links for Addresses
+        elif field in ["url", "associated_clearnet_instance"]:
+            url = urlparse(value).hostname
+            rows.append(f"[{url}]({value})")
+
+        elif field == "status" and value is not None:
+            if value.get("display_content_is_image"):
+                rows.append(f"[![{value['display_content_image_fallback']}]({value['display_content']})]({value['url']})")
+            else:
+                rows.append(f"[{value['display_content']}]({value['url']})")
+        
+        elif field == "country" and value:
+            rows.append(f"{value['flag']} {value['name']}")
+
+        elif field == "modified":
+            if value.get("is_modified"):
+                rows.append(f"[Yes]({value['source_url']})")
+            else:
+                rows.append("No")
+
+        # We're going to use a markdown link here
+        elif field == "privacy_policy":
+            rows.append(f"[Here]({value})")
+
+        # Handle author name
+        elif field == "owner":
+            # Assuming github url
+            author_name = value.split("/")
+            rows.append(f"[@{author_name[-1]}]({value})")
+        else:
+            rows.append(value)
+
+    table_data.extend(rows)
+
+
+with open("instances.yaml") as configuration_yaml:
+    data = yaml.safe_load(configuration_yaml)
+
+instance_list = data["instances"]
+
+# Initial information
+md_instance_list = MdUtils(file_name='Invidious-Instances.md')
+md_instance_list.new_header(level=1, title='Public Instances')
+md_instance_list.new_paragraph("Uptime History: [uptime.invidious.io](https://uptime.invidious.io)")
+md_instance_list.new_paragraph("Instances API: [api.invidious.io](api.invidious.io)")
+
+
+# Clearnet instances
+md_instance_list.new_header(level=1, title='Instances list')
+table_data = ["Address", "Country", "Status", "Privacy policy", "DDos Protection / MITM", "Owner", "Modified"]
+for instance_data in instance_list["https"]:
+    create_table(table_data, instance_data)
+
+md_instance_list.new_line()
+md_instance_list.new_table(columns=7, rows=len(instance_list["https"]) + 1, text=table_data, text_align='center')
+
+
+# Onion instances
+md_instance_list.new_header(level=1, title='Tor onion instances list')
+table_data = ["Address", "Country", "Associated clearnet instance", "Privacy policy", "Owner", "Modified"]
+for instance_data in instance_list["onion"]:
+    create_table(table_data, instance_data) 
+
+md_instance_list.new_line()
+md_instance_list.new_table(columns=6, rows=len(instance_list["onion"]) + 1, text=table_data, text_align='center')
+
+
+# Instance adding directions and prerequisites
+md_instance_list.new_header(level=1, title='Adding your instance')
+
+# Prerequisites
+md_instance_list.new_header(level=2, title='Prerequisites')
+md_instance_list.new_list(data["adding_instance"]["prerequisites"])
+md_instance_list.new_line()
+
+# Directions
+md_instance_list.new_header(level=2, title='Directions')
+md_instance_list.new_list(data["adding_instance"]["directions"], marked_with="1")
+md_instance_list.create_md_file()

src/requirements.txt

@@ -0,0 +1,2 @@
+mdutils==1.3.0
+PyYAML==5.4.1