#!/usr/bin/env bash # LLVM source-coverage for the nigig-email domain, with an enforced floor. # # The email domain lives in nigig-core alongside a great deal of unrelated # code, and nigig-core depends on Makepad (email_worker posts widget actions), # so unlike the PDF engine it cannot be copied out into a standalone pure # workspace. This runs the domain tests in place, instruments the build, and # then reports coverage over ONLY the seven email source files -- Makepad's # generated code and the rest of nigig-core are excluded from the count. # # Why a floor and not a report: a number that is only printed drifts down. # The whole point of Phase A-E was turning "the SMTP password serialises" and # "the bulk tab cannot bulk-send" from findings into tests; a floor keeps # that from silently reverting. Same reasoning as tools/test-pdf-coverage.sh. # # Usage: # ./tools/test-email-coverage.sh # EMAIL_COVERAGE_REPORT_ONLY=1 ./tools/test-email-coverage.sh # # Runs entirely inside one temporary directory (its own rustup, cargo home # and target dir), removed on exit. It does not touch ~/.cargo, the workspace # target directory or any source file. set -Eeuo pipefail IFS=$'\n\t' ROOT="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" TOOLCHAIN="${RUST_TOOLCHAIN:-1.97.1}" # The whole-domain floor, set a little under today's measurement (~90.7%) # so ordinary refactoring does not trip it while a real loss of coverage # does. The remaining uncovered lines are the actual network I/O (SMTP # socket, reqwest/fetch, the feature-gated IMAP socket) and platform file # I/O, which the plan records as "not host-verified" rather than testable. TOTAL_FLOOR="${EMAIL_COVERAGE_TOTAL_FLOOR:-88}" # Per-file floors for the files that have actually harboured the bugs the # remediation fixed. A single whole-domain number hides exactly the failure # this is meant to catch: a regression in the proxy parser or the recipient # splitter moves the total by less than a point. Each entry is "path:floor". # # email_worker's floor is lower because its uncovered lines are the actual # SMTP I/O and the wasm `#[cfg]` blocks -- genuinely untestable without a # live server or a browser (plan "What I have not verified"). The same # reasoning puts mail_proxy's floor a little under its measured value: its # remaining gaps are the reqwest/fetch transports, not the parsing logic. PER_FILE_FLOORS="${EMAIL_COVERAGE_PER_FILE_FLOORS:-\ nigig-core/src/mail_proxy.rs:85 nigig-core/src/mail_backend.rs:92 nigig-core/src/email_worker.rs:55 nigig-core/src/email_send.rs:95 nigig-core/src/email_store.rs:95 nigig-core/src/email_account.rs:95 nigig-core/src/secret.rs:95 nigig-core/src/email_pacing.rs:90 nigig-core/src/email_bulk.rs:85 nigig-core/src/credential_store.rs:90 nigig-core/src/email_cache.rs:80 nigig-core/src/email_session.rs:90 nigig-core/src/imap_client.rs:85 nigig-core/src/email_receipts.rs:90 nigig-core/src/finance_report.rs:80}" WORK="$(mktemp -d "${TMPDIR:-/tmp}/email-coverage.XXXXXXXX")" cleanup() { local status=$? if [ "${KEEP_TEST_ENV:-0}" = "1" ]; then printf 'kept coverage environment at %s\n' "$WORK" >&2 else chmod -R u+w "$WORK" 2>/dev/null || true rm -rf "$WORK" fi exit "$status" } trap cleanup EXIT HUP INT TERM export RUSTUP_HOME="$WORK/rustup" export CARGO_HOME="$WORK/cargo" export CARGO_TARGET_DIR="$WORK/target" export PATH="$CARGO_HOME/bin:$PATH" export LLVM_PROFILE_FILE="$WORK/profiles/%p-%m.profraw" # `-C codegen-units=1` is deliberately NOT set: the email domain is compiled # alongside Makepad (naga, etc.) and a single codegen unit on those crates # spikes memory hard enough to get OOM-killed on small runners. Line # coverage is unaffected; only edge/region precision would be. export RUSTFLAGS="-C instrument-coverage -C opt-level=0" # nigig-core depends on the Makepad fork (a large git repo). libgit2 can fail # to fetch it into a fresh CARGO_HOME with "unrecoverable internal error: # 'writer.open == 0'"; using the system git CLI for fetches avoids that. export CARGO_NET_GIT_FETCH_WITH_CLI=true mkdir -p "$WORK/profiles" printf 'installing Rust %s with llvm-tools\n' "$TOOLCHAIN" curl --fail --silent --show-error --location https://sh.rustup.rs \ -o "$WORK/rustup-init" chmod 700 "$WORK/rustup-init" "$WORK/rustup-init" -y --profile minimal --default-toolchain "$TOOLCHAIN" \ --component llvm-tools-preview --no-modify-path >/dev/null printf 'running the email domain tests under instrumentation\n' cd "$ROOT" cargo test --locked -p nigig-core --lib -- \ email_ secret:: mail_backend:: mail_proxy:: imap_client:: credential_store:: finance_report:: \ >"$WORK/test.log" 2>&1 || { cat "$WORK/test.log"; exit 1; } grep -E 'test result' "$WORK/test.log" | tail -5 HOST="$(rustc -vV | sed -n 's/^host: //p')" LLVM_BIN="$RUSTUP_HOME/toolchains/$TOOLCHAIN-$HOST/lib/rustlib/$HOST/bin" "$LLVM_BIN/llvm-profdata" merge -sparse "$WORK/profiles"/*.profraw \ -o "$WORK/coverage.profdata" # The lib test binary is the one that ran the email domain tests. nigig-core # is a single crate, so unlike the three-crate PDF engine a single binary is # complete, not an under-report. BIN="$(find "$CARGO_TARGET_DIR/debug/deps" -maxdepth 1 -type f -executable \ -name 'nigig_core-*' | head -1)" if [ -z "$BIN" ]; then echo 'no nigig-core test binary was produced' >&2 exit 1 fi # Only the email files count. Makepad's generated code, other nigig-core # modules and the registry are all excluded. IGNORE='(/cargo/registry|/cargo/git|/rustc/)' EMAIL_FILES=( "$ROOT/crates/nigig-core/src/email_account.rs" "$ROOT/crates/nigig-core/src/email_send.rs" "$ROOT/crates/nigig-core/src/email_store.rs" "$ROOT/crates/nigig-core/src/email_worker.rs" "$ROOT/crates/nigig-core/src/mail_backend.rs" "$ROOT/crates/nigig-core/src/mail_proxy.rs" "$ROOT/crates/nigig-core/src/secret.rs" "$ROOT/crates/nigig-core/src/email_pacing.rs" "$ROOT/crates/nigig-core/src/email_bulk.rs" "$ROOT/crates/nigig-core/src/credential_store.rs" "$ROOT/crates/nigig-core/src/email_cache.rs" "$ROOT/crates/nigig-core/src/email_session.rs" "$ROOT/crates/nigig-core/src/imap_client.rs" "$ROOT/crates/nigig-core/src/email_receipts.rs" "$ROOT/crates/nigig-core/src/finance_report.rs" ) printf '\n=== per-file coverage ===\n' "$LLVM_BIN/llvm-cov" report "$BIN" \ -instr-profile="$WORK/coverage.profdata" \ -ignore-filename-regex="$IGNORE" \ "${EMAIL_FILES[@]}" | tee "$WORK/report.txt" # Machine-readable totals, filtered to the email files. The export covers # every file in the binary, so the floor is computed over exactly the files # this script reported on, matched by path suffix. "$LLVM_BIN/llvm-cov" export "$BIN" \ -instr-profile="$WORK/coverage.profdata" \ -ignore-filename-regex="$IGNORE" > "$WORK/coverage.json" if [ "${EMAIL_COVERAGE_REPORT_ONLY:-0}" = "1" ]; then echo 'report-only mode: the floor was not enforced' exit 0 fi python3 - "$WORK/coverage.json" "$TOTAL_FLOOR" "$PER_FILE_FLOORS" <<'PY' import json, sys path, total_floor, per_file = sys.argv[1], float(sys.argv[2]), sys.argv[3] with open(path) as fh: data = json.load(fh) keep = ("email_account.rs", "email_send.rs", "email_store.rs", "email_worker.rs", "mail_backend.rs", "mail_proxy.rs", "secret.rs", "email_pacing.rs", "email_bulk.rs", "credential_store.rs", "email_cache.rs", "email_session.rs", "imap_client.rs", "email_receipts.rs", "finance_report.rs") files = [f for f in data["data"][0]["files"] if f["filename"].endswith(keep)] total_lines = sum(f["summary"]["lines"]["count"] for f in files) covered = sum(f["summary"]["lines"]["covered"] for f in files) total = 100.0 * covered / total_lines if total_lines else 0.0 print(f"\ntotal line coverage over the email domain: {total:.2f}% " f"(floor {total_floor:.2f}%)") failures = [] if total < total_floor: failures.append(f" total {total:.2f}% is below the floor of " f"{total_floor:.2f}%") measured = {} for f in files: measured[f["filename"]] = f["summary"]["lines"]["percent"] for line in per_file.split(): if not line.strip(): continue name, _, floor = line.rpartition(":") floor = float(floor) matches = [v for k, v in measured.items() if k.endswith(name)] if not matches: failures.append( f" {name} has a floor but was not measured - was it renamed or " "deleted? A floor on a file that no longer exists silently " "protects nothing.") continue got = matches[0] if got < floor: failures.append(f" {name} {got:.2f}% is below its floor of " f"{floor:.2f}%") if failures: print("coverage floors not met:", file=sys.stderr) print("\n".join(failures), file=sys.stderr) print( "\nEither the change removed tested code, or it added untested code.\n" "Lowering a floor is a reviewable edit to tools/test-email-coverage.sh,\n" "not something to do quietly.", file=sys.stderr, ) sys.exit(1) print(f"all coverage floors met (total {total:.2f}%)") PY