luna/tower-of-babel-public
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

sync 2024-10-09, pyinfra v3

Browse source
This commit is contained in:
Luna 2024-10-09 14:33:22 -03:00
parent b3b451ce96
commit 924e170299
7 changed files with 137 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
files/compose.service.j2
View file

@ -10,10 +10,8 @@ After=network-online.target
Type=oneshot
RemainAfterExit=true
WorkingDirectory={{ env_dict.working_directory }}
ExecStart=/usr/bin/docker-compose up -d --remove-orphans
ExecStop=/usr/bin/docker-compose down
{{ systemd.service_security() }}
ExecStart={{ env_dict.compose_command }} up -d --remove-orphans
ExecStop={{ env_dict.compose_command }} down
[Install]
WantedBy=multi-user.target

2
files/piped/config.properties
View file

@ -57,6 +57,8 @@ SUBSCRIPTIONS_EXPIRY:30
# Use Sentry to log errors and trace performance
#SENTRY_DSN:INSERT_HERE
CONSENT_COOKIE:true
# Matrix Client Server URL
MATRIX_SERVER:https://matrix-client.matrix.org
# Matrix Access Token

88
tasks/docker.py
View file

@ -1,6 +1,7 @@
import logging
import subprocess
import json
import enum
import tempfile
from pathlib import Path
from typing import Optional
@ -14,6 +15,7 @@ from .install_consul_server import template_and_install_systemd
DEFAULTS = {
"docker_registry_image": "registry:2.8.1",
"docker_runtime": "docker",
}
@ -21,10 +23,20 @@ DEFAULTS = {
def install_docker():
linux_name = host.get_fact(LinuxName)
if linux_name == "Fedora":
dnf.packages(["docker", "docker-compose"])
systemd.service("docker", enabled=True, running=True)
else:
apt.packages(["docker.io", "docker-compose"])
runtime = host.data.get("docker_runtime", "docker")
if runtime == "docker":
dnf.packages(["docker", "docker-compose"])
systemd.service("docker", enabled=True, running=True)
elif runtime == "podman":
dnf.packages(["podman", "podman-compose"])
elif linux_name in ("Ubuntu", "Debian"):
runtime = host.data.get("docker_runtime", "docker")
if runtime == "docker":
apt.packages(["docker.io", "docker-compose"])
elif runtime == "podman":
raise Exception("not supported yet")
class TailscaleIPs(FactBase):
@ -39,30 +51,41 @@ class TailscaleIPs(FactBase):
return [line]
class Runtime(enum.Enum):
DOCKER = "docker"
PODMAN = "podman"
@property
def cmd(self):
if self == Runtime.DOCKER:
return "docker"
elif self == Runtime.PODMAN:
return "podman"
def get_host_docker_runtime():
runtime_from_data = host.data.get("docker_runtime", "docker")
if runtime_from_data == "docker":
return Runtime.DOCKER
elif runtime_from_data == "podman":
return Runtime.PODMAN
else:
raise Exception(f"unknown docker_runtime {runtime_from_data}")
class DockerImage(FactBase):
requires_command = "docker"
def command(self, object_id):
return f"docker image inspect {object_id} || true"
runtime = get_host_docker_runtime()
if runtime == Runtime.DOCKER:
return f"docker image inspect {object_id} || true"
elif runtime == Runtime.PODMAN:
return f"podman image inspect {object_id} || true"
def process(self, output):
joined_out = "".join(output)
return json.loads(joined_out)
class DockerManifestInspect(FactBase):
requires_command = "docker"
def command(self, object_id):
return f"docker image inspect {object_id} || true"
def process(self, output):
if "no such manifest" in output:
return None
joined_out = "".join(output)
return json.loads(joined_out)
log = logging.getLogger(__name__)
@ -76,8 +99,10 @@ def docker_image_from_host_to_target(image_reference: str):
"hello, sending image %r to host %s@%s", image_reference, username, hostname
)
runtime = get_host_docker_runtime()
with tempfile.NamedTemporaryFile() as f:
cmdline = f"docker save {image_reference} | gzip | pv > {f.name}"
cmdline = f"{runtime.cmd} save {image_reference} | gzip | pv > {f.name}"
log.warning("exec %r", cmdline)
subprocess.check_output(cmdline, shell=True)
@ -102,14 +127,17 @@ def docker_image_from_host_to_target(image_reference: str):
send_cmdline = f"croc --yes {transfer_code}"
server.shell(send_cmdline, _chdir="/tmp")
server.shell(
f"cat {target_path} | docker load", _chdir="/tmp", name="load image file"
f"cat {target_path} | {runtime.cmd} load",
_chdir="/tmp",
name="load image file",
)
server.shell(f"rm /tmp/{target_path}", name="remove image file after importing")
@operation
@operation()
def docker_image(image_reference: str):
images = host.get_fact(DockerImage, image_reference)
runtime = get_host_docker_runtime()
if not images:
name, *_version = image_reference.split(":")
if name in host.data.manual_docker_images:
@ -123,7 +151,12 @@ def docker_image(image_reference: str):
)
else:
# take it from given image ref
yield f"docker pull {image_reference}"
if runtime == Runtime.PODMAN:
if not image_reference.startswith("docker.io"):
log.warning(
"image does not contain reference to docker.io, podman will fail"
)
yield f"{runtime.cmd} pull {image_reference}"
def template_and_install_compose(
@ -149,11 +182,18 @@ def template_and_install_compose(
name=f"sending compose file {systemd_service_name}",
)
runtime = get_host_docker_runtime()
if runtime == Runtime.DOCKER:
compose_command = "/usr/bin/docker-compose"
elif runtime == Runtime.PODMAN:
compose_command = "/usr/bin/podman-compose"
template_and_install_systemd(
"files/compose.service.j2",
env_dict={
"service_name": systemd_service_name,
"working_directory": working_directory,
"compose_command": compose_command,
},
service_name=systemd_service,
)

39
tasks/operations/git.py
View file

@ -2,24 +2,21 @@ from pyinfra.operations import apk, files, server, git, systemd, python
from pyinfra import host
from pyinfra.facts.files import Directory
from pyinfra.facts.git import GitBranch
from pyinfra.facts.git import GitBranch, GitFactBase
from pyinfra.api import deploy, operation, FactBase
from pyinfra.operations.util.files import chown, unix_path_join
class CoolerGitBranch(FactBase):
requires_command = "git"
@staticmethod
def command(repo):
class CoolerGitBranch(GitFactBase):
def command(self, repo) -> str:
# TODO should inject _sudo / _sudo_user if user is provided in repo()
return "! test -d {0} || (cd {0} && git rev-parse --abbrev-ref HEAD)".format(
repo
)
class GitFetch(FactBase):
class GitFetch(GitFactBase):
def command(self, repo: str):
return f"git -C {repo} fetch"
@ -27,9 +24,10 @@ class GitFetch(FactBase):
return output
class GitRevListComparison(FactBase):
class GitRevListComparison(GitFactBase):
def command(self, repo: str, branch: str):
assert branch
if not branch:
raise AssertionError(f"branch must be provided. its now {branch!r}")
return f"git -C {repo} rev-list HEAD..origin/{branch} | wc -l"
def process(self, output):
@ -48,11 +46,7 @@ class RawCommandOutput(FactBase):
return "\n".join(output) # re-join and return the output lines
@operation(
pipeline_facts={
"git_branch": "target",
}
)
@operation()
def repo(
src,
dest,
@ -91,7 +85,7 @@ def repo(
"""
# Ensure our target directory exists
yield from files.directory(dest)
yield from files.directory._inner(dest)
if ssh_keyscan:
raise NotImplementedError("TODO copypaste ssh_keyscan code")
@ -108,13 +102,14 @@ def repo(
else:
git_commands.append("clone {0} .".format(src))
host.create_fact(GitBranch, kwargs={"repo": dest}, data=branch)
host.create_fact(CoolerGitBranch, kwargs={"repo": dest}, data=branch)
host.create_fact(
Directory,
kwargs={"path": git_dir},
data={"user": user, "group": group},
)
# pyinfra-v3: not needed anymore
# host.create_fact(GitBranch, kwargs={"repo": dest}, data=branch)
# host.create_fact(CoolerGitBranch, kwargs={"repo": dest}, data=branch)
# host.create_fact(
# Directory,
# kwargs={"path": git_dir},
# data={"user": user, "group": group},
# )
# Ensuring existing repo
else:

15
tasks/piped.py
View file

@ -1,6 +1,6 @@
from pyinfra import host
from pyinfra.api import deploy
from pyinfra.operations import files, server, dnf, postgresql, lxd
from pyinfra.operations import files, server, dnf, postgres, lxd
from pyinfra.facts.server import Which
from pyinfra.facts.lxd import LxdContainers
@ -13,7 +13,10 @@ from .yts import lxc_shell
@deploy(
"create lxc container that'll run piped",
data_defaults={"piped_container_name": "piped"},
data_defaults={
"piped_container_name": "piped",
"piped_container_image": "images:fedora/38",
},
)
def install_lxc_container():
containers = host.get_fact(LxdContainers)
@ -28,7 +31,7 @@ def install_lxc_container():
lxd.container(
name="create piped container",
id=ct_name,
image="images:fedora/38",
image=host.data.piped_container_image,
)
# validate the ct is good
@ -40,7 +43,7 @@ def install():
install_postgresql()
dnf.packages(
[
"java-17-openjdk-headless",
"java-21-openjdk-headless",
]
)
@ -52,14 +55,14 @@ def install():
if has_postgres:
postgres_kwargs = {"_sudo": True, "_sudo_user": "postgres"}
postgresql.role(
postgres.role(
role=host.data.piped_db_user,
password=with_secrets.piped_db_password,
login=True,
**postgres_kwargs,
)
postgresql.database(
postgres.database(
database=host.data.piped_db_name,
owner=host.data.piped_db_user,
encoding="UTF8",

51
tasks/pleroma.py
View file

@ -1,5 +1,5 @@
from pyinfra import host
from pyinfra.operations import dnf, server, files, systemd, postgresql
from pyinfra.operations import dnf, server, files, systemd, postgres
from pyinfra.api import deploy
from pyinfra.facts.server import Which
@ -24,6 +24,17 @@ class WithSecrets:
return getattr(host.data, field)
class FallbackForHost:
def __init__(self, data):
self.data = data
def __getattr__(self, field):
print(field, self.data)
if field in self.data:
return self.data[field]
return getattr(host.data, field)
@deploy("install pleroma")
def install():
install_elixir()
@ -79,16 +90,36 @@ def install():
)
remote_config_path = f"{remote_main_pleroma_path}/config/prod.secret.exs"
with_secrets = WithSecrets(
with_host_secrets = WithSecrets(
(
"pleroma_secret_key_base",
"pleroma_db_password",
"pleroma_webpush_public_key",
"pleroma_webpush_private_key",
host.data.pleroma_secret_fields["pleroma_secret_key_base"],
host.data.pleroma_secret_fields["pleroma_db_password"],
host.data.pleroma_secret_fields["pleroma_webpush_public_key"],
host.data.pleroma_secret_fields["pleroma_webpush_private_key"],
)
)
with_secrets = FallbackForHost(
{
"pleroma_secret_key_base": getattr(
with_host_secrets,
host.data.pleroma_secret_fields["pleroma_secret_key_base"],
),
"pleroma_db_password": getattr(
with_host_secrets,
host.data.pleroma_secret_fields["pleroma_db_password"],
),
"pleroma_webpush_public_key": getattr(
with_host_secrets,
host.data.pleroma_secret_fields["pleroma_webpush_public_key"],
),
"pleroma_webpush_private_key": getattr(
with_host_secrets,
host.data.pleroma_secret_fields["pleroma_webpush_private_key"],
),
}
)
config_output = files.template(
"./files/pleroma/prod.secret.exs",
host.data.pleroma_local_config_path,
dest=remote_config_path,
user=runner_user,
group=runner_user,
@ -137,14 +168,14 @@ def install():
if has_postgres:
postgres_kwargs = {"_sudo": True, "_sudo_user": "postgres"}
postgresql.role(
postgres.role(
role=host.data.pleroma_db_user,
password=with_secrets.pleroma_db_password,
login=True,
**postgres_kwargs,
)
db_result = postgresql.database(
db_result = postgres.database(
database=host.data.pleroma_db_name,
owner=host.data.pleroma_db_user,
encoding="UTF8",
@ -154,7 +185,7 @@ def install():
# is it possible to configure pg_hba.conf to add md5 auth to local v4/v6
if db_result.changed:
postgresql.sql(
postgres.sql(
"""
CREATE EXTENSION IF NOT EXISTS citext;
CREATE EXTENSION IF NOT EXISTS pg_trgm;

2
tasks/postgresql.py
View file

@ -40,3 +40,5 @@ def install():
enabled=True,
daemon_reload=True,
)
else:
raise AssertionError(f"Unsupported linux_name{linux_name}")