sync 2024-10-09, pyinfra v3

files/compose.service.j2

@@ -10,10 +10,8 @@ After=network-online.target
 Type=oneshot
 RemainAfterExit=true
 WorkingDirectory={{ env_dict.working_directory }}
-ExecStart=/usr/bin/docker-compose up -d --remove-orphans
+ExecStart={{ env_dict.compose_command }} up -d --remove-orphans
-ExecStop=/usr/bin/docker-compose down
+ExecStop={{ env_dict.compose_command }} down
-
-{{ systemd.service_security() }}
 
 [Install]
 WantedBy=multi-user.target

files/piped/config.properties

@@ -57,6 +57,8 @@ SUBSCRIPTIONS_EXPIRY:30
 # Use Sentry to log errors and trace performance
 #SENTRY_DSN:INSERT_HERE
 
+CONSENT_COOKIE:true
+
 # Matrix Client Server URL
 MATRIX_SERVER:https://matrix-client.matrix.org
 # Matrix Access Token

tasks/docker.py

@@ -1,6 +1,7 @@
 import logging
 import subprocess
 import json
+import enum
 import tempfile
 from pathlib import Path
 from typing import Optional
@@ -14,6 +15,7 @@ from .install_consul_server import template_and_install_systemd
 
 DEFAULTS = {
     "docker_registry_image": "registry:2.8.1",
+    "docker_runtime": "docker",
 }
 
 
@@ -21,10 +23,20 @@ DEFAULTS = {
 def install_docker():
     linux_name = host.get_fact(LinuxName)
     if linux_name == "Fedora":
+
+        runtime = host.data.get("docker_runtime", "docker")
+        if runtime == "docker":
             dnf.packages(["docker", "docker-compose"])
             systemd.service("docker", enabled=True, running=True)
-    else:
+        elif runtime == "podman":
+            dnf.packages(["podman", "podman-compose"])
+
+    elif linux_name in ("Ubuntu", "Debian"):
+        runtime = host.data.get("docker_runtime", "docker")
+        if runtime == "docker":
             apt.packages(["docker.io", "docker-compose"])
+        elif runtime == "podman":
+            raise Exception("not supported yet")
 
 
 class TailscaleIPs(FactBase):
@@ -39,30 +51,41 @@ class TailscaleIPs(FactBase):
             return [line]
 
 
+class Runtime(enum.Enum):
+    DOCKER = "docker"
+    PODMAN = "podman"
+
+    @property
+    def cmd(self):
+        if self == Runtime.DOCKER:
+            return "docker"
+        elif self == Runtime.PODMAN:
+            return "podman"
+
+
+def get_host_docker_runtime():
+    runtime_from_data = host.data.get("docker_runtime", "docker")
+    if runtime_from_data == "docker":
+        return Runtime.DOCKER
+    elif runtime_from_data == "podman":
+        return Runtime.PODMAN
+    else:
+        raise Exception(f"unknown docker_runtime {runtime_from_data}")
+
+
 class DockerImage(FactBase):
-    requires_command = "docker"
-
     def command(self, object_id):
+        runtime = get_host_docker_runtime()
+        if runtime == Runtime.DOCKER:
             return f"docker image inspect {object_id} || true"
+        elif runtime == Runtime.PODMAN:
+            return f"podman image inspect {object_id} || true"
 
     def process(self, output):
         joined_out = "".join(output)
         return json.loads(joined_out)
 
 
-class DockerManifestInspect(FactBase):
-    requires_command = "docker"
-
-    def command(self, object_id):
-        return f"docker image inspect {object_id} || true"
-
-    def process(self, output):
-        if "no such manifest" in output:
-            return None
-        joined_out = "".join(output)
-        return json.loads(joined_out)
-
-
 log = logging.getLogger(__name__)
 
 
@@ -76,8 +99,10 @@ def docker_image_from_host_to_target(image_reference: str):
         "hello, sending image %r to host %s@%s", image_reference, username, hostname
     )
 
+    runtime = get_host_docker_runtime()
+
     with tempfile.NamedTemporaryFile() as f:
-        cmdline = f"docker save {image_reference} | gzip | pv > {f.name}"
+        cmdline = f"{runtime.cmd} save {image_reference} | gzip | pv > {f.name}"
         log.warning("exec %r", cmdline)
         subprocess.check_output(cmdline, shell=True)
 
@@ -102,14 +127,17 @@ def docker_image_from_host_to_target(image_reference: str):
             send_cmdline = f"croc --yes {transfer_code}"
             server.shell(send_cmdline, _chdir="/tmp")
         server.shell(
-            f"cat {target_path} | docker load", _chdir="/tmp", name="load image file"
+            f"cat {target_path} | {runtime.cmd} load",
+            _chdir="/tmp",
+            name="load image file",
         )
         server.shell(f"rm /tmp/{target_path}", name="remove image file after importing")
 
 
-@operation
+@operation()
 def docker_image(image_reference: str):
     images = host.get_fact(DockerImage, image_reference)
+    runtime = get_host_docker_runtime()
     if not images:
         name, *_version = image_reference.split(":")
         if name in host.data.manual_docker_images:
@@ -123,7 +151,12 @@ def docker_image(image_reference: str):
             )
         else:
             # take it from given image ref
-            yield f"docker pull {image_reference}"
+            if runtime == Runtime.PODMAN:
+                if not image_reference.startswith("docker.io"):
+                    log.warning(
+                        "image does not contain reference to docker.io, podman will fail"
+                    )
+            yield f"{runtime.cmd} pull {image_reference}"
 
 
 def template_and_install_compose(
@@ -149,11 +182,18 @@ def template_and_install_compose(
         name=f"sending compose file {systemd_service_name}",
     )
 
+    runtime = get_host_docker_runtime()
+    if runtime == Runtime.DOCKER:
+        compose_command = "/usr/bin/docker-compose"
+    elif runtime == Runtime.PODMAN:
+        compose_command = "/usr/bin/podman-compose"
+
     template_and_install_systemd(
         "files/compose.service.j2",
         env_dict={
             "service_name": systemd_service_name,
             "working_directory": working_directory,
+            "compose_command": compose_command,
         },
         service_name=systemd_service,
     )

tasks/operations/git.py

@@ -2,24 +2,21 @@ from pyinfra.operations import apk, files, server, git, systemd, python
 from pyinfra import host
 
 from pyinfra.facts.files import Directory
-from pyinfra.facts.git import GitBranch
+from pyinfra.facts.git import GitBranch, GitFactBase
 
 from pyinfra.api import deploy, operation, FactBase
 from pyinfra.operations.util.files import chown, unix_path_join
 
 
-class CoolerGitBranch(FactBase):
+class CoolerGitBranch(GitFactBase):
-    requires_command = "git"
+    def command(self, repo) -> str:
-
-    @staticmethod
-    def command(repo):
         # TODO should inject _sudo / _sudo_user if user is provided in repo()
         return "! test -d {0} || (cd {0} && git rev-parse --abbrev-ref HEAD)".format(
             repo
         )
 
 
-class GitFetch(FactBase):
+class GitFetch(GitFactBase):
     def command(self, repo: str):
         return f"git -C {repo} fetch"
 
@@ -27,9 +24,10 @@ class GitFetch(FactBase):
         return output
 
 
-class GitRevListComparison(FactBase):
+class GitRevListComparison(GitFactBase):
     def command(self, repo: str, branch: str):
-        assert branch
+        if not branch:
+            raise AssertionError(f"branch must be provided. its now {branch!r}")
         return f"git -C {repo} rev-list HEAD..origin/{branch} | wc -l"
 
     def process(self, output):
@@ -48,11 +46,7 @@ class RawCommandOutput(FactBase):
         return "\n".join(output)  # re-join and return the output lines
 
 
-@operation(
+@operation()
-    pipeline_facts={
-        "git_branch": "target",
-    }
-)
 def repo(
     src,
     dest,
@@ -91,7 +85,7 @@ def repo(
     """
 
     # Ensure our target directory exists
-    yield from files.directory(dest)
+    yield from files.directory._inner(dest)
 
     if ssh_keyscan:
         raise NotImplementedError("TODO copypaste ssh_keyscan code")
@@ -108,13 +102,14 @@ def repo(
         else:
             git_commands.append("clone {0} .".format(src))
 
-        host.create_fact(GitBranch, kwargs={"repo": dest}, data=branch)
+        # pyinfra-v3: not needed anymore
-        host.create_fact(CoolerGitBranch, kwargs={"repo": dest}, data=branch)
+        # host.create_fact(GitBranch, kwargs={"repo": dest}, data=branch)
-        host.create_fact(
+        # host.create_fact(CoolerGitBranch, kwargs={"repo": dest}, data=branch)
-            Directory,
+        # host.create_fact(
-            kwargs={"path": git_dir},
+        #    Directory,
-            data={"user": user, "group": group},
+        #    kwargs={"path": git_dir},
-        )
+        #    data={"user": user, "group": group},
+        # )
 
     # Ensuring existing repo
     else:

tasks/piped.py

@@ -1,6 +1,6 @@
 from pyinfra import host
 from pyinfra.api import deploy
-from pyinfra.operations import files, server, dnf, postgresql, lxd
+from pyinfra.operations import files, server, dnf, postgres, lxd
 from pyinfra.facts.server import Which
 from pyinfra.facts.lxd import LxdContainers
 
@@ -13,7 +13,10 @@ from .yts import lxc_shell
 
 @deploy(
     "create lxc container that'll run piped",
-    data_defaults={"piped_container_name": "piped"},
+    data_defaults={
+        "piped_container_name": "piped",
+        "piped_container_image": "images:fedora/38",
+    },
 )
 def install_lxc_container():
     containers = host.get_fact(LxdContainers)
@@ -28,7 +31,7 @@ def install_lxc_container():
         lxd.container(
             name="create piped container",
             id=ct_name,
-            image="images:fedora/38",
+            image=host.data.piped_container_image,
         )
 
     # validate the ct is good
@@ -40,7 +43,7 @@ def install():
     install_postgresql()
     dnf.packages(
         [
-            "java-17-openjdk-headless",
+            "java-21-openjdk-headless",
         ]
     )
 
@@ -52,14 +55,14 @@ def install():
     if has_postgres:
         postgres_kwargs = {"_sudo": True, "_sudo_user": "postgres"}
 
-    postgresql.role(
+    postgres.role(
         role=host.data.piped_db_user,
         password=with_secrets.piped_db_password,
         login=True,
         **postgres_kwargs,
     )
 
-    postgresql.database(
+    postgres.database(
         database=host.data.piped_db_name,
         owner=host.data.piped_db_user,
         encoding="UTF8",

tasks/pleroma.py

@@ -1,5 +1,5 @@
 from pyinfra import host
-from pyinfra.operations import dnf, server, files, systemd, postgresql
+from pyinfra.operations import dnf, server, files, systemd, postgres
 from pyinfra.api import deploy
 from pyinfra.facts.server import Which
 
@@ -24,6 +24,17 @@ class WithSecrets:
         return getattr(host.data, field)
 
 
+class FallbackForHost:
+    def __init__(self, data):
+        self.data = data
+
+    def __getattr__(self, field):
+        print(field, self.data)
+        if field in self.data:
+            return self.data[field]
+        return getattr(host.data, field)
+
+
 @deploy("install pleroma")
 def install():
     install_elixir()
@@ -79,16 +90,36 @@ def install():
     )
 
     remote_config_path = f"{remote_main_pleroma_path}/config/prod.secret.exs"
-    with_secrets = WithSecrets(
+    with_host_secrets = WithSecrets(
         (
-            "pleroma_secret_key_base",
+            host.data.pleroma_secret_fields["pleroma_secret_key_base"],
-            "pleroma_db_password",
+            host.data.pleroma_secret_fields["pleroma_db_password"],
-            "pleroma_webpush_public_key",
+            host.data.pleroma_secret_fields["pleroma_webpush_public_key"],
-            "pleroma_webpush_private_key",
+            host.data.pleroma_secret_fields["pleroma_webpush_private_key"],
         )
     )
+    with_secrets = FallbackForHost(
+        {
+            "pleroma_secret_key_base": getattr(
+                with_host_secrets,
+                host.data.pleroma_secret_fields["pleroma_secret_key_base"],
+            ),
+            "pleroma_db_password": getattr(
+                with_host_secrets,
+                host.data.pleroma_secret_fields["pleroma_db_password"],
+            ),
+            "pleroma_webpush_public_key": getattr(
+                with_host_secrets,
+                host.data.pleroma_secret_fields["pleroma_webpush_public_key"],
+            ),
+            "pleroma_webpush_private_key": getattr(
+                with_host_secrets,
+                host.data.pleroma_secret_fields["pleroma_webpush_private_key"],
+            ),
+        }
+    )
     config_output = files.template(
-        "./files/pleroma/prod.secret.exs",
+        host.data.pleroma_local_config_path,
         dest=remote_config_path,
         user=runner_user,
         group=runner_user,
@@ -137,14 +168,14 @@ def install():
     if has_postgres:
         postgres_kwargs = {"_sudo": True, "_sudo_user": "postgres"}
 
-    postgresql.role(
+    postgres.role(
         role=host.data.pleroma_db_user,
         password=with_secrets.pleroma_db_password,
         login=True,
         **postgres_kwargs,
     )
 
-    db_result = postgresql.database(
+    db_result = postgres.database(
         database=host.data.pleroma_db_name,
         owner=host.data.pleroma_db_user,
         encoding="UTF8",
@@ -154,7 +185,7 @@ def install():
     # is it possible to configure pg_hba.conf to add md5 auth to local v4/v6
 
     if db_result.changed:
-        postgresql.sql(
+        postgres.sql(
             """
             CREATE EXTENSION IF NOT EXISTS citext;
             CREATE EXTENSION IF NOT EXISTS pg_trgm;

tasks/postgresql.py

@@ -40,3 +40,5 @@ def install():
             enabled=True,
             daemon_reload=True,
         )
+    else:
+        raise AssertionError(f"Unsupported linux_name{linux_name}")