diff --git a/files/compose.service.j2 b/files/compose.service.j2 index 3f5bf07..e2811e8 100644 --- a/files/compose.service.j2 +++ b/files/compose.service.j2 @@ -10,10 +10,8 @@ After=network-online.target Type=oneshot RemainAfterExit=true WorkingDirectory={{ env_dict.working_directory }} -ExecStart=/usr/bin/docker-compose up -d --remove-orphans -ExecStop=/usr/bin/docker-compose down - -{{ systemd.service_security() }} +ExecStart={{ env_dict.compose_command }} up -d --remove-orphans +ExecStop={{ env_dict.compose_command }} down [Install] WantedBy=multi-user.target diff --git a/files/piped/config.properties b/files/piped/config.properties index 37aadb6..7e494c1 100644 --- a/files/piped/config.properties +++ b/files/piped/config.properties @@ -57,6 +57,8 @@ SUBSCRIPTIONS_EXPIRY:30 # Use Sentry to log errors and trace performance #SENTRY_DSN:INSERT_HERE +CONSENT_COOKIE:true + # Matrix Client Server URL MATRIX_SERVER:https://matrix-client.matrix.org # Matrix Access Token diff --git a/tasks/docker.py b/tasks/docker.py index 2203318..f884427 100644 --- a/tasks/docker.py +++ b/tasks/docker.py @@ -1,6 +1,7 @@ import logging import subprocess import json +import enum import tempfile from pathlib import Path from typing import Optional @@ -14,6 +15,7 @@ from .install_consul_server import template_and_install_systemd DEFAULTS = { "docker_registry_image": "registry:2.8.1", + "docker_runtime": "docker", } @@ -21,10 +23,20 @@ DEFAULTS = { def install_docker(): linux_name = host.get_fact(LinuxName) if linux_name == "Fedora": - dnf.packages(["docker", "docker-compose"]) - systemd.service("docker", enabled=True, running=True) - else: - apt.packages(["docker.io", "docker-compose"]) + + runtime = host.data.get("docker_runtime", "docker") + if runtime == "docker": + dnf.packages(["docker", "docker-compose"]) + systemd.service("docker", enabled=True, running=True) + elif runtime == "podman": + dnf.packages(["podman", "podman-compose"]) + + elif linux_name in ("Ubuntu", "Debian"): + runtime = host.data.get("docker_runtime", "docker") + if runtime == "docker": + apt.packages(["docker.io", "docker-compose"]) + elif runtime == "podman": + raise Exception("not supported yet") class TailscaleIPs(FactBase): @@ -39,30 +51,41 @@ class TailscaleIPs(FactBase): return [line] +class Runtime(enum.Enum): + DOCKER = "docker" + PODMAN = "podman" + + @property + def cmd(self): + if self == Runtime.DOCKER: + return "docker" + elif self == Runtime.PODMAN: + return "podman" + + +def get_host_docker_runtime(): + runtime_from_data = host.data.get("docker_runtime", "docker") + if runtime_from_data == "docker": + return Runtime.DOCKER + elif runtime_from_data == "podman": + return Runtime.PODMAN + else: + raise Exception(f"unknown docker_runtime {runtime_from_data}") + + class DockerImage(FactBase): - requires_command = "docker" - def command(self, object_id): - return f"docker image inspect {object_id} || true" + runtime = get_host_docker_runtime() + if runtime == Runtime.DOCKER: + return f"docker image inspect {object_id} || true" + elif runtime == Runtime.PODMAN: + return f"podman image inspect {object_id} || true" def process(self, output): joined_out = "".join(output) return json.loads(joined_out) -class DockerManifestInspect(FactBase): - requires_command = "docker" - - def command(self, object_id): - return f"docker image inspect {object_id} || true" - - def process(self, output): - if "no such manifest" in output: - return None - joined_out = "".join(output) - return json.loads(joined_out) - - log = logging.getLogger(__name__) @@ -76,8 +99,10 @@ def docker_image_from_host_to_target(image_reference: str): "hello, sending image %r to host %s@%s", image_reference, username, hostname ) + runtime = get_host_docker_runtime() + with tempfile.NamedTemporaryFile() as f: - cmdline = f"docker save {image_reference} | gzip | pv > {f.name}" + cmdline = f"{runtime.cmd} save {image_reference} | gzip | pv > {f.name}" log.warning("exec %r", cmdline) subprocess.check_output(cmdline, shell=True) @@ -102,14 +127,17 @@ def docker_image_from_host_to_target(image_reference: str): send_cmdline = f"croc --yes {transfer_code}" server.shell(send_cmdline, _chdir="/tmp") server.shell( - f"cat {target_path} | docker load", _chdir="/tmp", name="load image file" + f"cat {target_path} | {runtime.cmd} load", + _chdir="/tmp", + name="load image file", ) server.shell(f"rm /tmp/{target_path}", name="remove image file after importing") -@operation +@operation() def docker_image(image_reference: str): images = host.get_fact(DockerImage, image_reference) + runtime = get_host_docker_runtime() if not images: name, *_version = image_reference.split(":") if name in host.data.manual_docker_images: @@ -123,7 +151,12 @@ def docker_image(image_reference: str): ) else: # take it from given image ref - yield f"docker pull {image_reference}" + if runtime == Runtime.PODMAN: + if not image_reference.startswith("docker.io"): + log.warning( + "image does not contain reference to docker.io, podman will fail" + ) + yield f"{runtime.cmd} pull {image_reference}" def template_and_install_compose( @@ -149,11 +182,18 @@ def template_and_install_compose( name=f"sending compose file {systemd_service_name}", ) + runtime = get_host_docker_runtime() + if runtime == Runtime.DOCKER: + compose_command = "/usr/bin/docker-compose" + elif runtime == Runtime.PODMAN: + compose_command = "/usr/bin/podman-compose" + template_and_install_systemd( "files/compose.service.j2", env_dict={ "service_name": systemd_service_name, "working_directory": working_directory, + "compose_command": compose_command, }, service_name=systemd_service, ) diff --git a/tasks/operations/git.py b/tasks/operations/git.py index c90e835..8c0f691 100644 --- a/tasks/operations/git.py +++ b/tasks/operations/git.py @@ -2,24 +2,21 @@ from pyinfra.operations import apk, files, server, git, systemd, python from pyinfra import host from pyinfra.facts.files import Directory -from pyinfra.facts.git import GitBranch +from pyinfra.facts.git import GitBranch, GitFactBase from pyinfra.api import deploy, operation, FactBase from pyinfra.operations.util.files import chown, unix_path_join -class CoolerGitBranch(FactBase): - requires_command = "git" - - @staticmethod - def command(repo): +class CoolerGitBranch(GitFactBase): + def command(self, repo) -> str: # TODO should inject _sudo / _sudo_user if user is provided in repo() return "! test -d {0} || (cd {0} && git rev-parse --abbrev-ref HEAD)".format( repo ) -class GitFetch(FactBase): +class GitFetch(GitFactBase): def command(self, repo: str): return f"git -C {repo} fetch" @@ -27,9 +24,10 @@ class GitFetch(FactBase): return output -class GitRevListComparison(FactBase): +class GitRevListComparison(GitFactBase): def command(self, repo: str, branch: str): - assert branch + if not branch: + raise AssertionError(f"branch must be provided. its now {branch!r}") return f"git -C {repo} rev-list HEAD..origin/{branch} | wc -l" def process(self, output): @@ -48,11 +46,7 @@ class RawCommandOutput(FactBase): return "\n".join(output) # re-join and return the output lines -@operation( - pipeline_facts={ - "git_branch": "target", - } -) +@operation() def repo( src, dest, @@ -91,7 +85,7 @@ def repo( """ # Ensure our target directory exists - yield from files.directory(dest) + yield from files.directory._inner(dest) if ssh_keyscan: raise NotImplementedError("TODO copypaste ssh_keyscan code") @@ -108,13 +102,14 @@ def repo( else: git_commands.append("clone {0} .".format(src)) - host.create_fact(GitBranch, kwargs={"repo": dest}, data=branch) - host.create_fact(CoolerGitBranch, kwargs={"repo": dest}, data=branch) - host.create_fact( - Directory, - kwargs={"path": git_dir}, - data={"user": user, "group": group}, - ) + # pyinfra-v3: not needed anymore + # host.create_fact(GitBranch, kwargs={"repo": dest}, data=branch) + # host.create_fact(CoolerGitBranch, kwargs={"repo": dest}, data=branch) + # host.create_fact( + # Directory, + # kwargs={"path": git_dir}, + # data={"user": user, "group": group}, + # ) # Ensuring existing repo else: diff --git a/tasks/piped.py b/tasks/piped.py index 8925aa6..143a2e4 100644 --- a/tasks/piped.py +++ b/tasks/piped.py @@ -1,6 +1,6 @@ from pyinfra import host from pyinfra.api import deploy -from pyinfra.operations import files, server, dnf, postgresql, lxd +from pyinfra.operations import files, server, dnf, postgres, lxd from pyinfra.facts.server import Which from pyinfra.facts.lxd import LxdContainers @@ -13,7 +13,10 @@ from .yts import lxc_shell @deploy( "create lxc container that'll run piped", - data_defaults={"piped_container_name": "piped"}, + data_defaults={ + "piped_container_name": "piped", + "piped_container_image": "images:fedora/38", + }, ) def install_lxc_container(): containers = host.get_fact(LxdContainers) @@ -28,7 +31,7 @@ def install_lxc_container(): lxd.container( name="create piped container", id=ct_name, - image="images:fedora/38", + image=host.data.piped_container_image, ) # validate the ct is good @@ -40,7 +43,7 @@ def install(): install_postgresql() dnf.packages( [ - "java-17-openjdk-headless", + "java-21-openjdk-headless", ] ) @@ -52,14 +55,14 @@ def install(): if has_postgres: postgres_kwargs = {"_sudo": True, "_sudo_user": "postgres"} - postgresql.role( + postgres.role( role=host.data.piped_db_user, password=with_secrets.piped_db_password, login=True, **postgres_kwargs, ) - postgresql.database( + postgres.database( database=host.data.piped_db_name, owner=host.data.piped_db_user, encoding="UTF8", diff --git a/tasks/pleroma.py b/tasks/pleroma.py index 9499d3d..6f4c4e9 100644 --- a/tasks/pleroma.py +++ b/tasks/pleroma.py @@ -1,5 +1,5 @@ from pyinfra import host -from pyinfra.operations import dnf, server, files, systemd, postgresql +from pyinfra.operations import dnf, server, files, systemd, postgres from pyinfra.api import deploy from pyinfra.facts.server import Which @@ -24,6 +24,17 @@ class WithSecrets: return getattr(host.data, field) +class FallbackForHost: + def __init__(self, data): + self.data = data + + def __getattr__(self, field): + print(field, self.data) + if field in self.data: + return self.data[field] + return getattr(host.data, field) + + @deploy("install pleroma") def install(): install_elixir() @@ -79,16 +90,36 @@ def install(): ) remote_config_path = f"{remote_main_pleroma_path}/config/prod.secret.exs" - with_secrets = WithSecrets( + with_host_secrets = WithSecrets( ( - "pleroma_secret_key_base", - "pleroma_db_password", - "pleroma_webpush_public_key", - "pleroma_webpush_private_key", + host.data.pleroma_secret_fields["pleroma_secret_key_base"], + host.data.pleroma_secret_fields["pleroma_db_password"], + host.data.pleroma_secret_fields["pleroma_webpush_public_key"], + host.data.pleroma_secret_fields["pleroma_webpush_private_key"], ) ) + with_secrets = FallbackForHost( + { + "pleroma_secret_key_base": getattr( + with_host_secrets, + host.data.pleroma_secret_fields["pleroma_secret_key_base"], + ), + "pleroma_db_password": getattr( + with_host_secrets, + host.data.pleroma_secret_fields["pleroma_db_password"], + ), + "pleroma_webpush_public_key": getattr( + with_host_secrets, + host.data.pleroma_secret_fields["pleroma_webpush_public_key"], + ), + "pleroma_webpush_private_key": getattr( + with_host_secrets, + host.data.pleroma_secret_fields["pleroma_webpush_private_key"], + ), + } + ) config_output = files.template( - "./files/pleroma/prod.secret.exs", + host.data.pleroma_local_config_path, dest=remote_config_path, user=runner_user, group=runner_user, @@ -137,14 +168,14 @@ def install(): if has_postgres: postgres_kwargs = {"_sudo": True, "_sudo_user": "postgres"} - postgresql.role( + postgres.role( role=host.data.pleroma_db_user, password=with_secrets.pleroma_db_password, login=True, **postgres_kwargs, ) - db_result = postgresql.database( + db_result = postgres.database( database=host.data.pleroma_db_name, owner=host.data.pleroma_db_user, encoding="UTF8", @@ -154,7 +185,7 @@ def install(): # is it possible to configure pg_hba.conf to add md5 auth to local v4/v6 if db_result.changed: - postgresql.sql( + postgres.sql( """ CREATE EXTENSION IF NOT EXISTS citext; CREATE EXTENSION IF NOT EXISTS pg_trgm; diff --git a/tasks/postgresql.py b/tasks/postgresql.py index 45b053c..af84752 100644 --- a/tasks/postgresql.py +++ b/tasks/postgresql.py @@ -40,3 +40,5 @@ def install(): enabled=True, daemon_reload=True, ) + else: + raise AssertionError(f"Unsupported linux_name{linux_name}")