#!/usr/bin/env python3 """Assemble a persistent Arch USB on macOS using Python 3.14 and Apple tools. Example: python3 tools/arch_usb/build.py --device diskN --clone arch@10.0.0.165 --writer-app Repeat without contacting the clone using --cached-clone. Cached OS/driver/CEF assets are reused; no software is downloaded or installed on either host. Use --assets and --output to override the checkout-local cache/output paths. Select --device with `diskutil list external physical`. An attached USB is required for identity and size. After reconnect or reboot, rebuild the prepared plan to avoid stale disk-number reuse. Inputs are supplied by the caller; this command does not download anything. The assets directory supplies arch-base.raw, resolved-packages.json, and seed/ containing mirror.tar.*, realtek-firmware.tar, fanatec.tar.gz and cargo-cache.tar. Use --base-image for a different pristine raw Arch cloud image. All executable build/setup/writer logic lives beside this script, independently of the cache. The source snapshot is refreshed from this checkout unless --cached-source is set. Use --include-checkout apps/sandbox to package an optional local Sandbox clone; the default snapshot does not require a network or private clone. SSH is enabled for arch using a memorable password generated on first boot and shown on the local console. No personal SSH key is copied into the image. Use sudo makepad-ssh show/disable/enable/reset-password on the booted USB. Outputs: clean arch-boot.raw, CIDATA ISO, backup GPT, a pinned write-plan.json, and a validation report. Device identity is read-only metadata; this command does not write the USB. The optional native writer checks the selected physical USB and all hashes, replaces the image, verifies every written byte, and ejects after macOS authentication. """ import argparse import binascii import hashlib import json import os from pathlib import Path import plistlib import shutil import struct import subprocess import sys import tarfile import time import uuid from btrfs_read import Btrfs from efi_boot import Fat, prepare from iso9660 import validate_seed from usb_device import snapshot_device from clone_assets import refresh_packages, refresh_source, verify_source HERE = Path(__file__).resolve().parent REPO = HERE.parents[1] BLOCK = 512 WAIT_UNITS = ('systemd-networkd-wait-online.service', 'systemd-time-wait-sync.service', 'pacman-init.service') def digest(path): with path.open('rb') as stream: return hashlib.file_digest(stream, 'sha256').hexdigest() def copy(source, target): target.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(source, target) def resolve_include_checkout(value): path = Path(value) if not path.is_absolute(): path = REPO / path path = path.resolve() if not path.is_dir() or not path.is_relative_to(REPO) or path == REPO: raise ValueError(f'include-checkout must be a directory strictly inside the repository: {value}') try: toplevel = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'], cwd=path, text=True).strip() except subprocess.CalledProcessError as exc: raise ValueError(f'include-checkout must be a nested git checkout: {value}') from exc if Path(toplevel).resolve() != path: raise ValueError(f'include-checkout must be a nested git checkout: {value}') return path def package_source(seed, assets, include_checkouts=()): names = subprocess.check_output(['git', 'ls-files', '-z', '--cached', '--others', '--exclude-standard'], cwd=REPO).split(b'\0') if (REPO/'Cargo.lock').is_file(): names.append(b'Cargo.lock') included = [] for value in include_checkouts: checkout = resolve_include_checkout(value) relative_checkout = checkout.relative_to(REPO) listed = subprocess.check_output(['git', '-C', str(checkout), 'ls-files', '--cached', '--others', '--exclude-standard', '-z']).split(b'\0') if (checkout/'Cargo.lock').is_file(): listed.append(b'Cargo.lock') prefixed = [] for raw in listed: if not raw: continue inner = Path(os.fsdecode(raw)) if inner.is_absolute() or '..' in inner.parts: raise ValueError(f'path traversal in include-checkout {relative_checkout.as_posix()}: {inner}') prefixed.append(os.fsencode((relative_checkout / inner).as_posix())) if not prefixed: raise ValueError(f'include-checkout has no files: {relative_checkout.as_posix()}') names.extend(prefixed) head = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=checkout, text=True).strip() included.append((relative_checkout.as_posix(), head)) count = 0 with tarfile.open(seed/'makepad-source.tgz', 'w:gz', compresslevel=3) as archive: for raw in sorted(set(names)): if not raw: continue relative = Path(os.fsdecode(raw)) if any(part in ('local', '.claude', '.grok', '.git', '__pycache__') or part.startswith('target') for part in relative.parts): continue if relative.name.startswith('.env') or relative.suffix in ('.log', '.pem', '.key', '.pyc'): continue path = REPO/relative if path.is_relative_to(seed.parent) or path.is_relative_to(assets): continue if not path.is_file() and not path.is_symlink(): continue if path.is_symlink(): try: path.resolve().relative_to(REPO) except ValueError: continue archive.add(path, arcname=str(relative), recursive=False) count += 1 head = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=REPO, text=True).strip() revision = [f'Base HEAD: {head}\n'] for relative_checkout, checkout_head in included: revision.append(f'Included checkout: {relative_checkout}\n') revision.append(f'Included checkout HEAD: {checkout_head}\n') revision.append(f'Working-tree snapshot: {time.strftime("%Y-%m-%dT%H:%M:%S%z")}\n') revision.append(f'Files: {count}\n') (seed/'source-revision.txt').write_text(''.join(revision)) print(f'Packaged current Makepad source: {count} files.', flush=True) def prepare_seed(args, output): seed = output/'seed' if seed.exists(): shutil.rmtree(seed) seed.mkdir() original = args.assets/'seed' metadata = json.loads((args.assets/'resolved-packages.json').read_text()) packages = (HERE/'packages.txt').read_text().splitlines() assert set(packages) <= {item['NAME'][0] for item in metadata}, 'Requested package absent from the supplied cache' assert 'pacman' in packages and not {'cloud-init'} & set(packages) for path in sorted(original.glob('mirror.tar.*')): copy(path, seed/('mirror-'+path.suffix[1:]+'.tar')) assert list(seed.glob('mirror-*.tar')), 'Offline package mirror is missing' for name in ('realtek-firmware.tar', 'fanatec.tar.gz', 'cargo-cache.tar'): copy(original/name, seed/('fanatec.tgz' if name == 'fanatec.tar.gz' else name)) # Verify the source cache's original payload hashes before reusing it. cached_hashes = dict(line.split(' ', 1)[::-1] for line in (original/'SHA256SUMS').read_text().splitlines()) for path in seed.iterdir(): original_name = ('mirror.tar.'+path.stem.removeprefix('mirror-')) if path.name.startswith('mirror-') else ('fanatec.tar.gz' if path.name == 'fanatec.tgz' else path.name) assert digest(path) == cached_hashes[original_name], f'Cached payload changed: {path.name}' clone = None if args.clone or args.cached_clone: clone, manifest = verify_source(args.assets) for name in ('makepad-source.tgz', 'cargo-cache.tar', 'source-revision.txt'): copy(clone/name, seed/name) for name in ('wm', 'makepad-ai-hub'): copy(clone/'binaries'/name, seed/name) (seed/'clone.json').write_text(json.dumps(manifest, indent=2)+'\n') elif args.cached_source: if args.include_checkout: raise ValueError('--cached-source is incompatible with --include-checkout') for name, target in (('makepad-source.tar.gz', 'makepad-source.tgz'), ('SOURCE-REVISION.txt', 'source-revision.txt')): copy(original/name, seed/target) else: package_source(seed, args.assets, args.include_checkout) cef_archive = args.cef_archive or args.assets/'cef-linux.tar.bz2' assert cef_archive.is_file(), f'Cached Linux CEF archive missing: {cef_archive}' # Keep the untouched distribution, including resources, helper libraries # and headers. A build never needs to fetch CEF from the internet. with tarfile.open(cef_archive, 'r:bz2') as archive: members = archive.getnames() roots = {Path(name).parts[0] for name in members if Path(name).parts} assert len(roots) == 1, 'Expected one CEF distribution directory' cef_root = next(iter(roots)) assert not cef_root.startswith('.') and cef_root.endswith('_linux64') assert all(not Path(name).is_absolute() and '..' not in Path(name).parts for name in members) assert cef_root+'/Release/libcef.so' in members assert cef_root+'/include/cef_version.h' in members copy(cef_archive, seed/'cef-linux.bz2') (seed/'cef-directory.txt').write_text(cef_root+'\n') for name in ('firstboot.sh', 'ssh.sh', 'provision.sh', 'mount-win.sh', 'status.sh', 'wm-session.sh', 'aihub-session.sh', 'gbelt-bind.sh'): copy(HERE/name, seed/name) subprocess.run(['/bin/bash', '-n', str(seed/name)], check=True) for name in ('makepad-wm.service', 'wm.env', 'makepad-aihub.service', 'aihub.env', '70-makepad-game-hardware.rules'): copy(HERE/name, seed/name) if clone: # Preserve the known-working service setup. SSH and firstboot logic # always come from this repository's current builder. for path in (clone/'config').iterdir(): copy(path, seed/path.name) if path.suffix == '.sh': subprocess.run(['/bin/bash', '-n', str(seed/path.name)], check=True) assert cef_root == manifest['cef_directory'], 'CEF archive differs from the installed clone' (seed/'requested-packages.txt').write_text('\n'.join(packages)+'\n') # Cloned USB/internal roots must identify their own adapter to DHCP. (seed/'wired.network').write_text('[Match]\nName=eth* en*\n\n[Network]\nDHCP=ipv4\nIPv6AcceptRA=yes\n\n[DHCPv4]\nClientIdentifier=mac\n\n[Link]\nRequiredForOnline=no\n') (seed/'wifi.network').write_text('[Match]\nName=wl*\n\n[Network]\nDHCP=ipv4\nIPv6AcceptRA=yes\n\n[DHCPv4]\nClientIdentifier=mac\nRouteMetric=2048\n\n[IPv6AcceptRA]\nRouteMetric=2048\n\n[Link]\nRequiredForOnline=no\n') (seed/'no-bluetooth.conf').write_text(''.join(f'blacklist {name}\ninstall {name} /bin/false\n' for name in ('bluetooth', 'btusb'))) (seed/'iwd.conf').write_text('[General]\nEnableNetworkConfiguration=false\n') (seed/'fanatec-access.rules').write_text('SUBSYSTEM=="usb", ATTR{idVendor}=="0eb7", GROUP="games", MODE="0660"\nSUBSYSTEM=="hidraw", ATTRS{idVendor}=="0eb7", GROUP="games", MODE="0660", TAG+="uaccess"\n') (seed/'makepad-provision.service').write_text('''[Unit] Description=Install the cached Makepad development stack After=sshd.service network.target Wants=sshd.service ConditionPathExists=!/var/lib/makepad-provision/complete [Service] Type=exec ExecStart=/usr/local/sbin/makepad-provision StandardOutput=journal StandardError=journal [Install] WantedBy=multi-user.target ''') (seed/'makepad-mount-win.service').write_text('''[Unit] Description=Mount the Windows NTFS volume read-only at /mnt/win After=local-fs.target [Service] Type=oneshot ExecStart=/usr/local/sbin/makepad-mount-win RemainAfterExit=yes [Install] WantedBy=multi-user.target ''') # Execute the setup directly from durable read-only media. Avoid generated # write_files/runcmd files and their once-per-instance state on the root. command = '''set -eu mkdir -p /run/makepad-seed if ! mountpoint -q /run/makepad-seed; then device=/dev/disk/by-label/CIDATA test -e "$device" || device=/dev/disk/by-label/cidata mount -o ro "$device" /run/makepad-seed fi exec /usr/bin/bash /run/makepad-seed/firstboot.sh''' config = {'cloud_init_modules': ['bootcmd'], 'cloud_config_modules': [], 'cloud_final_modules': [], 'growpart': {'mode': 'off'}, 'resize_rootfs': False, 'users': [], 'bootcmd': [['/usr/bin/bash', '-c', command]]} (seed/'user-data').write_text('#cloud-config\n'+json.dumps(config, indent=2)+'\n') (seed/'meta-data').write_text(json.dumps({'instance-id': args.build_id, 'local-hostname': 'makepad-arch'})+'\n') # The installed renderer uses the interface's name, not nested match.name. (seed/'network-config').write_text(json.dumps({'version': 2, 'renderer': 'networkd', 'ethernets': {'eth0': {'dhcp4': True, 'dhcp6': False, 'optional': True}}}, indent=2)+'\n') small = [p for p in seed.iterdir() if p.name.endswith(('.sh', '.conf', '.network', '.rules', '.service')) or p.name == 'realtek-firmware.tar'] (seed/'config.sha256').write_text(''.join(f'{digest(p)} {p.name}\n' for p in sorted(small))) (seed/'sha256sums').write_text(''.join(f'{digest(p)} {p.name}\n' for p in sorted(seed.iterdir()))) iso = output/'makepad-seed.iso' subprocess.run(['hdiutil', 'makehybrid', '-o', str(iso), str(seed), '-iso', '-joliet', '-iso-volume-name', 'CIDATA', '-joliet-volume-name', 'CIDATA', '-ov'], check=True) expected = {p.name: digest(p) for p in seed.iterdir()} validate_seed(iso, expected) print('Plain ISO9660/Linux and Joliet filenames and contents verified.', flush=True) # Also read back through macOS' filesystem driver. attached = plistlib.loads(subprocess.check_output(['hdiutil', 'attach', '-readonly', '-nobrowse', '-plist', str(iso)])) mounted = next(entity for entity in attached['system-entities'] if 'mount-point' in entity) try: root = Path(mounted['mount-point']) for line in (seed/'sha256sums').read_text().splitlines(): expected, name = line.split(' ', 1) assert digest(root/name) == expected, f'ISO readback mismatch: {name}' finally: subprocess.run(['hdiutil', 'detach', mounted['dev-entry']], check=True) print('ISO payload readback verified.', flush=True) return iso def assemble(args, output, iso): source = args.base_image with source.open('rb') as stream: mbr = bytearray(stream.read(512)); header = bytearray(stream.read(512)) assert header[:8] == b'EFI PART', 'Expected a pristine raw GPT Arch cloud image' size, crc = struct.unpack_from(' identifier "{identifier}"', str(app)], check=True) subprocess.run(['codesign', '--verify', '--strict', str(app)], check=True) subprocess.run([str(executable), '--check-configuration'], check=True) def main(): if sys.flags.optimize: raise RuntimeError('Python optimization is not allowed; safety assertions must remain enabled') parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('--assets', type=Path, default=REPO/'local/agent_state/arch-usb', help='Offline OS/package assets cache') parser.add_argument('--output', type=Path, default=REPO/'local/arch-usb-image', help='Prepared image and writer output') parser.add_argument('--base-image', type=Path) parser.add_argument('--disk-bytes', type=int, help='Optional size assertion; must match the attached USB') parser.add_argument('--device', required=True, help='Whole disk from diskutil list external physical (diskN)') parser.add_argument('--build-id', default='makepad-clean-'+time.strftime('%Y%m%d-%H%M%S')) parser.add_argument('--cached-source', action='store_true') clone = parser.add_mutually_exclusive_group() clone.add_argument('--clone', metavar='USER@HOST', help='Refresh packages and working WM/AI Hub/source from an installed clone over SSH') clone.add_argument('--cached-clone', action='store_true', help='Reuse the last verified clone snapshot without SSH') parser.add_argument('--clone-root', default='/home/arch/makepad', help='Source directory on the installed clone') parser.add_argument('--include-checkout', action='append', default=[], type=Path, metavar='PATH', help='Optional nested git checkout to package (repeatable; e.g. apps/sandbox)') parser.add_argument('--cef-archive', type=Path, help='Cached Linux x86_64 CEF tar.bz2 (default: ASSETS/cef-linux.tar.bz2)') parser.add_argument('--writer-app', type=Path, nargs='?', const=Path.home()/'Applications/Makepad USB Writer.app') args = parser.parse_args() if args.cached_source and args.include_checkout: parser.error('--cached-source is incompatible with --include-checkout') if (args.clone or args.cached_clone) and (args.cached_source or args.include_checkout): parser.error('Clone snapshots cannot be combined with --cached-source or --include-checkout') args.assets = args.assets.expanduser().resolve(); output = args.output.expanduser().resolve() assert output != args.assets and args.assets not in output.parents and output not in args.assets.parents, 'Keep output separate from input assets' assert output != REPO and output not in REPO.parents, 'Keep output separate from the repository root' assert output != HERE and HERE not in output.parents and output not in HERE.parents, 'Keep output separate from the builder sources' args.base_image = (args.base_image or args.assets/'arch-base.raw').expanduser().resolve() if args.cef_archive is None and not (args.assets/'cef-linux.tar.bz2').exists(): candidates = sorted((REPO/'local/cef-prebuilt').glob('cef_binary_*_linux64.tar.bz2')) if len(candidates) == 1: args.cef_archive = candidates[0] args.cef_archive = (args.cef_archive or args.assets/'cef-linux.tar.bz2').expanduser().resolve() for source in (args.base_image, args.cef_archive): assert source != output and output not in source.parents, 'Keep source inputs outside the output directory' args.device_info = snapshot_device(args.device) args.device = args.device_info['device'] if args.disk_bytes is not None: assert args.disk_bytes == args.device_info['size'], f'--disk-bytes {args.disk_bytes} does not match attached device size {args.device_info["size"]}' args.disk_bytes = args.device_info['size'] print(f'Selected {args.device}: {args.device_info["registry_name"]}, {args.disk_bytes} bytes', flush=True) for path in (args.base_image, args.cef_archive): if not path.is_file(): raise ValueError(f'Required cached OS/CEF input is missing: {path}') if args.clone: refresh_packages(args.assets, args.clone, (HERE/'packages.txt').read_text().splitlines()) refresh_source(args.assets, args.clone, args.clone_root) output.mkdir(parents=True, exist_ok=True) assert args.disk_bytes % BLOCK == 0 iso = prepare_seed(args, output) plan = assemble(args, output, iso) copy(HERE/'write_usb.py', output/'write_usb.py') copy(HERE/'usb_device.py', output/'usb_device.py') if args.writer_app: args.writer_app = args.writer_app.expanduser().resolve() writer_app(args, output) report = {'build_id': args.build_id, 'assembled_at': time.strftime('%Y-%m-%dT%H:%M:%S%z'), 'base_sha256': digest(args.base_image), 'write_plan_sha256': digest(output/'write-plan.json'), 'checks': ['cached payload SHA256', 'shell syntax', 'plain ISO9660/Linux and Joliet exact filenames and content hashes', 'full ISO readback hashes', 'pacman/SSH/sudo binaries in base', 'FAT loader/configuration readback', 'stock root and initramfs byte comparison', 'no access repair overlay'], 'hardware_boot': 'pending', 'bytes_to_write': sum(c['length'] for c in plan['components'])} (output/'build-complete.json').write_text(json.dumps(report, indent=2)+'\n') print(f'Clean image assembled and checked: {output}', flush=True) print('Hardware boot and SSH verification remain pending. No physical disk was written.', flush=True) if __name__ == '__main__': main()