sync 2024-10-09, pyinfra v3
This commit is contained in:
parent
b3b451ce96
commit
924e170299
7 changed files with 137 additions and 66 deletions
|
@ -10,10 +10,8 @@ After=network-online.target
|
||||||
Type=oneshot
|
Type=oneshot
|
||||||
RemainAfterExit=true
|
RemainAfterExit=true
|
||||||
WorkingDirectory={{ env_dict.working_directory }}
|
WorkingDirectory={{ env_dict.working_directory }}
|
||||||
ExecStart=/usr/bin/docker-compose up -d --remove-orphans
|
ExecStart={{ env_dict.compose_command }} up -d --remove-orphans
|
||||||
ExecStop=/usr/bin/docker-compose down
|
ExecStop={{ env_dict.compose_command }} down
|
||||||
|
|
||||||
{{ systemd.service_security() }}
|
|
||||||
|
|
||||||
[Install]
|
[Install]
|
||||||
WantedBy=multi-user.target
|
WantedBy=multi-user.target
|
||||||
|
|
|
@ -57,6 +57,8 @@ SUBSCRIPTIONS_EXPIRY:30
|
||||||
# Use Sentry to log errors and trace performance
|
# Use Sentry to log errors and trace performance
|
||||||
#SENTRY_DSN:INSERT_HERE
|
#SENTRY_DSN:INSERT_HERE
|
||||||
|
|
||||||
|
CONSENT_COOKIE:true
|
||||||
|
|
||||||
# Matrix Client Server URL
|
# Matrix Client Server URL
|
||||||
MATRIX_SERVER:https://matrix-client.matrix.org
|
MATRIX_SERVER:https://matrix-client.matrix.org
|
||||||
# Matrix Access Token
|
# Matrix Access Token
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
import logging
|
import logging
|
||||||
import subprocess
|
import subprocess
|
||||||
import json
|
import json
|
||||||
|
import enum
|
||||||
import tempfile
|
import tempfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
@ -14,6 +15,7 @@ from .install_consul_server import template_and_install_systemd
|
||||||
|
|
||||||
DEFAULTS = {
|
DEFAULTS = {
|
||||||
"docker_registry_image": "registry:2.8.1",
|
"docker_registry_image": "registry:2.8.1",
|
||||||
|
"docker_runtime": "docker",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -21,10 +23,20 @@ DEFAULTS = {
|
||||||
def install_docker():
|
def install_docker():
|
||||||
linux_name = host.get_fact(LinuxName)
|
linux_name = host.get_fact(LinuxName)
|
||||||
if linux_name == "Fedora":
|
if linux_name == "Fedora":
|
||||||
dnf.packages(["docker", "docker-compose"])
|
|
||||||
systemd.service("docker", enabled=True, running=True)
|
runtime = host.data.get("docker_runtime", "docker")
|
||||||
else:
|
if runtime == "docker":
|
||||||
apt.packages(["docker.io", "docker-compose"])
|
dnf.packages(["docker", "docker-compose"])
|
||||||
|
systemd.service("docker", enabled=True, running=True)
|
||||||
|
elif runtime == "podman":
|
||||||
|
dnf.packages(["podman", "podman-compose"])
|
||||||
|
|
||||||
|
elif linux_name in ("Ubuntu", "Debian"):
|
||||||
|
runtime = host.data.get("docker_runtime", "docker")
|
||||||
|
if runtime == "docker":
|
||||||
|
apt.packages(["docker.io", "docker-compose"])
|
||||||
|
elif runtime == "podman":
|
||||||
|
raise Exception("not supported yet")
|
||||||
|
|
||||||
|
|
||||||
class TailscaleIPs(FactBase):
|
class TailscaleIPs(FactBase):
|
||||||
|
@ -39,30 +51,41 @@ class TailscaleIPs(FactBase):
|
||||||
return [line]
|
return [line]
|
||||||
|
|
||||||
|
|
||||||
|
class Runtime(enum.Enum):
|
||||||
|
DOCKER = "docker"
|
||||||
|
PODMAN = "podman"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def cmd(self):
|
||||||
|
if self == Runtime.DOCKER:
|
||||||
|
return "docker"
|
||||||
|
elif self == Runtime.PODMAN:
|
||||||
|
return "podman"
|
||||||
|
|
||||||
|
|
||||||
|
def get_host_docker_runtime():
|
||||||
|
runtime_from_data = host.data.get("docker_runtime", "docker")
|
||||||
|
if runtime_from_data == "docker":
|
||||||
|
return Runtime.DOCKER
|
||||||
|
elif runtime_from_data == "podman":
|
||||||
|
return Runtime.PODMAN
|
||||||
|
else:
|
||||||
|
raise Exception(f"unknown docker_runtime {runtime_from_data}")
|
||||||
|
|
||||||
|
|
||||||
class DockerImage(FactBase):
|
class DockerImage(FactBase):
|
||||||
requires_command = "docker"
|
|
||||||
|
|
||||||
def command(self, object_id):
|
def command(self, object_id):
|
||||||
return f"docker image inspect {object_id} || true"
|
runtime = get_host_docker_runtime()
|
||||||
|
if runtime == Runtime.DOCKER:
|
||||||
|
return f"docker image inspect {object_id} || true"
|
||||||
|
elif runtime == Runtime.PODMAN:
|
||||||
|
return f"podman image inspect {object_id} || true"
|
||||||
|
|
||||||
def process(self, output):
|
def process(self, output):
|
||||||
joined_out = "".join(output)
|
joined_out = "".join(output)
|
||||||
return json.loads(joined_out)
|
return json.loads(joined_out)
|
||||||
|
|
||||||
|
|
||||||
class DockerManifestInspect(FactBase):
|
|
||||||
requires_command = "docker"
|
|
||||||
|
|
||||||
def command(self, object_id):
|
|
||||||
return f"docker image inspect {object_id} || true"
|
|
||||||
|
|
||||||
def process(self, output):
|
|
||||||
if "no such manifest" in output:
|
|
||||||
return None
|
|
||||||
joined_out = "".join(output)
|
|
||||||
return json.loads(joined_out)
|
|
||||||
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@ -76,8 +99,10 @@ def docker_image_from_host_to_target(image_reference: str):
|
||||||
"hello, sending image %r to host %s@%s", image_reference, username, hostname
|
"hello, sending image %r to host %s@%s", image_reference, username, hostname
|
||||||
)
|
)
|
||||||
|
|
||||||
|
runtime = get_host_docker_runtime()
|
||||||
|
|
||||||
with tempfile.NamedTemporaryFile() as f:
|
with tempfile.NamedTemporaryFile() as f:
|
||||||
cmdline = f"docker save {image_reference} | gzip | pv > {f.name}"
|
cmdline = f"{runtime.cmd} save {image_reference} | gzip | pv > {f.name}"
|
||||||
log.warning("exec %r", cmdline)
|
log.warning("exec %r", cmdline)
|
||||||
subprocess.check_output(cmdline, shell=True)
|
subprocess.check_output(cmdline, shell=True)
|
||||||
|
|
||||||
|
@ -102,14 +127,17 @@ def docker_image_from_host_to_target(image_reference: str):
|
||||||
send_cmdline = f"croc --yes {transfer_code}"
|
send_cmdline = f"croc --yes {transfer_code}"
|
||||||
server.shell(send_cmdline, _chdir="/tmp")
|
server.shell(send_cmdline, _chdir="/tmp")
|
||||||
server.shell(
|
server.shell(
|
||||||
f"cat {target_path} | docker load", _chdir="/tmp", name="load image file"
|
f"cat {target_path} | {runtime.cmd} load",
|
||||||
|
_chdir="/tmp",
|
||||||
|
name="load image file",
|
||||||
)
|
)
|
||||||
server.shell(f"rm /tmp/{target_path}", name="remove image file after importing")
|
server.shell(f"rm /tmp/{target_path}", name="remove image file after importing")
|
||||||
|
|
||||||
|
|
||||||
@operation
|
@operation()
|
||||||
def docker_image(image_reference: str):
|
def docker_image(image_reference: str):
|
||||||
images = host.get_fact(DockerImage, image_reference)
|
images = host.get_fact(DockerImage, image_reference)
|
||||||
|
runtime = get_host_docker_runtime()
|
||||||
if not images:
|
if not images:
|
||||||
name, *_version = image_reference.split(":")
|
name, *_version = image_reference.split(":")
|
||||||
if name in host.data.manual_docker_images:
|
if name in host.data.manual_docker_images:
|
||||||
|
@ -123,7 +151,12 @@ def docker_image(image_reference: str):
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
# take it from given image ref
|
# take it from given image ref
|
||||||
yield f"docker pull {image_reference}"
|
if runtime == Runtime.PODMAN:
|
||||||
|
if not image_reference.startswith("docker.io"):
|
||||||
|
log.warning(
|
||||||
|
"image does not contain reference to docker.io, podman will fail"
|
||||||
|
)
|
||||||
|
yield f"{runtime.cmd} pull {image_reference}"
|
||||||
|
|
||||||
|
|
||||||
def template_and_install_compose(
|
def template_and_install_compose(
|
||||||
|
@ -149,11 +182,18 @@ def template_and_install_compose(
|
||||||
name=f"sending compose file {systemd_service_name}",
|
name=f"sending compose file {systemd_service_name}",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
runtime = get_host_docker_runtime()
|
||||||
|
if runtime == Runtime.DOCKER:
|
||||||
|
compose_command = "/usr/bin/docker-compose"
|
||||||
|
elif runtime == Runtime.PODMAN:
|
||||||
|
compose_command = "/usr/bin/podman-compose"
|
||||||
|
|
||||||
template_and_install_systemd(
|
template_and_install_systemd(
|
||||||
"files/compose.service.j2",
|
"files/compose.service.j2",
|
||||||
env_dict={
|
env_dict={
|
||||||
"service_name": systemd_service_name,
|
"service_name": systemd_service_name,
|
||||||
"working_directory": working_directory,
|
"working_directory": working_directory,
|
||||||
|
"compose_command": compose_command,
|
||||||
},
|
},
|
||||||
service_name=systemd_service,
|
service_name=systemd_service,
|
||||||
)
|
)
|
||||||
|
|
|
@ -2,24 +2,21 @@ from pyinfra.operations import apk, files, server, git, systemd, python
|
||||||
from pyinfra import host
|
from pyinfra import host
|
||||||
|
|
||||||
from pyinfra.facts.files import Directory
|
from pyinfra.facts.files import Directory
|
||||||
from pyinfra.facts.git import GitBranch
|
from pyinfra.facts.git import GitBranch, GitFactBase
|
||||||
|
|
||||||
from pyinfra.api import deploy, operation, FactBase
|
from pyinfra.api import deploy, operation, FactBase
|
||||||
from pyinfra.operations.util.files import chown, unix_path_join
|
from pyinfra.operations.util.files import chown, unix_path_join
|
||||||
|
|
||||||
|
|
||||||
class CoolerGitBranch(FactBase):
|
class CoolerGitBranch(GitFactBase):
|
||||||
requires_command = "git"
|
def command(self, repo) -> str:
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def command(repo):
|
|
||||||
# TODO should inject _sudo / _sudo_user if user is provided in repo()
|
# TODO should inject _sudo / _sudo_user if user is provided in repo()
|
||||||
return "! test -d {0} || (cd {0} && git rev-parse --abbrev-ref HEAD)".format(
|
return "! test -d {0} || (cd {0} && git rev-parse --abbrev-ref HEAD)".format(
|
||||||
repo
|
repo
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class GitFetch(FactBase):
|
class GitFetch(GitFactBase):
|
||||||
def command(self, repo: str):
|
def command(self, repo: str):
|
||||||
return f"git -C {repo} fetch"
|
return f"git -C {repo} fetch"
|
||||||
|
|
||||||
|
@ -27,9 +24,10 @@ class GitFetch(FactBase):
|
||||||
return output
|
return output
|
||||||
|
|
||||||
|
|
||||||
class GitRevListComparison(FactBase):
|
class GitRevListComparison(GitFactBase):
|
||||||
def command(self, repo: str, branch: str):
|
def command(self, repo: str, branch: str):
|
||||||
assert branch
|
if not branch:
|
||||||
|
raise AssertionError(f"branch must be provided. its now {branch!r}")
|
||||||
return f"git -C {repo} rev-list HEAD..origin/{branch} | wc -l"
|
return f"git -C {repo} rev-list HEAD..origin/{branch} | wc -l"
|
||||||
|
|
||||||
def process(self, output):
|
def process(self, output):
|
||||||
|
@ -48,11 +46,7 @@ class RawCommandOutput(FactBase):
|
||||||
return "\n".join(output) # re-join and return the output lines
|
return "\n".join(output) # re-join and return the output lines
|
||||||
|
|
||||||
|
|
||||||
@operation(
|
@operation()
|
||||||
pipeline_facts={
|
|
||||||
"git_branch": "target",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
def repo(
|
def repo(
|
||||||
src,
|
src,
|
||||||
dest,
|
dest,
|
||||||
|
@ -91,7 +85,7 @@ def repo(
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# Ensure our target directory exists
|
# Ensure our target directory exists
|
||||||
yield from files.directory(dest)
|
yield from files.directory._inner(dest)
|
||||||
|
|
||||||
if ssh_keyscan:
|
if ssh_keyscan:
|
||||||
raise NotImplementedError("TODO copypaste ssh_keyscan code")
|
raise NotImplementedError("TODO copypaste ssh_keyscan code")
|
||||||
|
@ -108,13 +102,14 @@ def repo(
|
||||||
else:
|
else:
|
||||||
git_commands.append("clone {0} .".format(src))
|
git_commands.append("clone {0} .".format(src))
|
||||||
|
|
||||||
host.create_fact(GitBranch, kwargs={"repo": dest}, data=branch)
|
# pyinfra-v3: not needed anymore
|
||||||
host.create_fact(CoolerGitBranch, kwargs={"repo": dest}, data=branch)
|
# host.create_fact(GitBranch, kwargs={"repo": dest}, data=branch)
|
||||||
host.create_fact(
|
# host.create_fact(CoolerGitBranch, kwargs={"repo": dest}, data=branch)
|
||||||
Directory,
|
# host.create_fact(
|
||||||
kwargs={"path": git_dir},
|
# Directory,
|
||||||
data={"user": user, "group": group},
|
# kwargs={"path": git_dir},
|
||||||
)
|
# data={"user": user, "group": group},
|
||||||
|
# )
|
||||||
|
|
||||||
# Ensuring existing repo
|
# Ensuring existing repo
|
||||||
else:
|
else:
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
from pyinfra import host
|
from pyinfra import host
|
||||||
from pyinfra.api import deploy
|
from pyinfra.api import deploy
|
||||||
from pyinfra.operations import files, server, dnf, postgresql, lxd
|
from pyinfra.operations import files, server, dnf, postgres, lxd
|
||||||
from pyinfra.facts.server import Which
|
from pyinfra.facts.server import Which
|
||||||
from pyinfra.facts.lxd import LxdContainers
|
from pyinfra.facts.lxd import LxdContainers
|
||||||
|
|
||||||
|
@ -13,7 +13,10 @@ from .yts import lxc_shell
|
||||||
|
|
||||||
@deploy(
|
@deploy(
|
||||||
"create lxc container that'll run piped",
|
"create lxc container that'll run piped",
|
||||||
data_defaults={"piped_container_name": "piped"},
|
data_defaults={
|
||||||
|
"piped_container_name": "piped",
|
||||||
|
"piped_container_image": "images:fedora/38",
|
||||||
|
},
|
||||||
)
|
)
|
||||||
def install_lxc_container():
|
def install_lxc_container():
|
||||||
containers = host.get_fact(LxdContainers)
|
containers = host.get_fact(LxdContainers)
|
||||||
|
@ -28,7 +31,7 @@ def install_lxc_container():
|
||||||
lxd.container(
|
lxd.container(
|
||||||
name="create piped container",
|
name="create piped container",
|
||||||
id=ct_name,
|
id=ct_name,
|
||||||
image="images:fedora/38",
|
image=host.data.piped_container_image,
|
||||||
)
|
)
|
||||||
|
|
||||||
# validate the ct is good
|
# validate the ct is good
|
||||||
|
@ -40,7 +43,7 @@ def install():
|
||||||
install_postgresql()
|
install_postgresql()
|
||||||
dnf.packages(
|
dnf.packages(
|
||||||
[
|
[
|
||||||
"java-17-openjdk-headless",
|
"java-21-openjdk-headless",
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -52,14 +55,14 @@ def install():
|
||||||
if has_postgres:
|
if has_postgres:
|
||||||
postgres_kwargs = {"_sudo": True, "_sudo_user": "postgres"}
|
postgres_kwargs = {"_sudo": True, "_sudo_user": "postgres"}
|
||||||
|
|
||||||
postgresql.role(
|
postgres.role(
|
||||||
role=host.data.piped_db_user,
|
role=host.data.piped_db_user,
|
||||||
password=with_secrets.piped_db_password,
|
password=with_secrets.piped_db_password,
|
||||||
login=True,
|
login=True,
|
||||||
**postgres_kwargs,
|
**postgres_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
postgresql.database(
|
postgres.database(
|
||||||
database=host.data.piped_db_name,
|
database=host.data.piped_db_name,
|
||||||
owner=host.data.piped_db_user,
|
owner=host.data.piped_db_user,
|
||||||
encoding="UTF8",
|
encoding="UTF8",
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
from pyinfra import host
|
from pyinfra import host
|
||||||
from pyinfra.operations import dnf, server, files, systemd, postgresql
|
from pyinfra.operations import dnf, server, files, systemd, postgres
|
||||||
from pyinfra.api import deploy
|
from pyinfra.api import deploy
|
||||||
from pyinfra.facts.server import Which
|
from pyinfra.facts.server import Which
|
||||||
|
|
||||||
|
@ -24,6 +24,17 @@ class WithSecrets:
|
||||||
return getattr(host.data, field)
|
return getattr(host.data, field)
|
||||||
|
|
||||||
|
|
||||||
|
class FallbackForHost:
|
||||||
|
def __init__(self, data):
|
||||||
|
self.data = data
|
||||||
|
|
||||||
|
def __getattr__(self, field):
|
||||||
|
print(field, self.data)
|
||||||
|
if field in self.data:
|
||||||
|
return self.data[field]
|
||||||
|
return getattr(host.data, field)
|
||||||
|
|
||||||
|
|
||||||
@deploy("install pleroma")
|
@deploy("install pleroma")
|
||||||
def install():
|
def install():
|
||||||
install_elixir()
|
install_elixir()
|
||||||
|
@ -79,16 +90,36 @@ def install():
|
||||||
)
|
)
|
||||||
|
|
||||||
remote_config_path = f"{remote_main_pleroma_path}/config/prod.secret.exs"
|
remote_config_path = f"{remote_main_pleroma_path}/config/prod.secret.exs"
|
||||||
with_secrets = WithSecrets(
|
with_host_secrets = WithSecrets(
|
||||||
(
|
(
|
||||||
"pleroma_secret_key_base",
|
host.data.pleroma_secret_fields["pleroma_secret_key_base"],
|
||||||
"pleroma_db_password",
|
host.data.pleroma_secret_fields["pleroma_db_password"],
|
||||||
"pleroma_webpush_public_key",
|
host.data.pleroma_secret_fields["pleroma_webpush_public_key"],
|
||||||
"pleroma_webpush_private_key",
|
host.data.pleroma_secret_fields["pleroma_webpush_private_key"],
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
with_secrets = FallbackForHost(
|
||||||
|
{
|
||||||
|
"pleroma_secret_key_base": getattr(
|
||||||
|
with_host_secrets,
|
||||||
|
host.data.pleroma_secret_fields["pleroma_secret_key_base"],
|
||||||
|
),
|
||||||
|
"pleroma_db_password": getattr(
|
||||||
|
with_host_secrets,
|
||||||
|
host.data.pleroma_secret_fields["pleroma_db_password"],
|
||||||
|
),
|
||||||
|
"pleroma_webpush_public_key": getattr(
|
||||||
|
with_host_secrets,
|
||||||
|
host.data.pleroma_secret_fields["pleroma_webpush_public_key"],
|
||||||
|
),
|
||||||
|
"pleroma_webpush_private_key": getattr(
|
||||||
|
with_host_secrets,
|
||||||
|
host.data.pleroma_secret_fields["pleroma_webpush_private_key"],
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
config_output = files.template(
|
config_output = files.template(
|
||||||
"./files/pleroma/prod.secret.exs",
|
host.data.pleroma_local_config_path,
|
||||||
dest=remote_config_path,
|
dest=remote_config_path,
|
||||||
user=runner_user,
|
user=runner_user,
|
||||||
group=runner_user,
|
group=runner_user,
|
||||||
|
@ -137,14 +168,14 @@ def install():
|
||||||
if has_postgres:
|
if has_postgres:
|
||||||
postgres_kwargs = {"_sudo": True, "_sudo_user": "postgres"}
|
postgres_kwargs = {"_sudo": True, "_sudo_user": "postgres"}
|
||||||
|
|
||||||
postgresql.role(
|
postgres.role(
|
||||||
role=host.data.pleroma_db_user,
|
role=host.data.pleroma_db_user,
|
||||||
password=with_secrets.pleroma_db_password,
|
password=with_secrets.pleroma_db_password,
|
||||||
login=True,
|
login=True,
|
||||||
**postgres_kwargs,
|
**postgres_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
db_result = postgresql.database(
|
db_result = postgres.database(
|
||||||
database=host.data.pleroma_db_name,
|
database=host.data.pleroma_db_name,
|
||||||
owner=host.data.pleroma_db_user,
|
owner=host.data.pleroma_db_user,
|
||||||
encoding="UTF8",
|
encoding="UTF8",
|
||||||
|
@ -154,7 +185,7 @@ def install():
|
||||||
# is it possible to configure pg_hba.conf to add md5 auth to local v4/v6
|
# is it possible to configure pg_hba.conf to add md5 auth to local v4/v6
|
||||||
|
|
||||||
if db_result.changed:
|
if db_result.changed:
|
||||||
postgresql.sql(
|
postgres.sql(
|
||||||
"""
|
"""
|
||||||
CREATE EXTENSION IF NOT EXISTS citext;
|
CREATE EXTENSION IF NOT EXISTS citext;
|
||||||
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
||||||
|
|
|
@ -40,3 +40,5 @@ def install():
|
||||||
enabled=True,
|
enabled=True,
|
||||||
daemon_reload=True,
|
daemon_reload=True,
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
|
raise AssertionError(f"Unsupported linux_name{linux_name}")
|
||||||
|
|
Loading…
Reference in a new issue