#!/usr/bin/env bash # Temporary LLVM source-coverage run for the nigig-build CAD engine. # # WHAT THIS COVERS # ---------------- # The CAD module is 26 files. Fourteen of them are the *engine*: pure # geometry, scene graph, undo/redo, exporters and file I/O whose only # Makepad imports are the math types (DVec2/Vec3f/Vec4f/Mat4f), the CSG # library and the two log macros. Those fourteen are copied here into a # host-only crate that carries the SAME module path as the real crate # (`nigig_build::construction_frame::pages::workspace::cad::*`), so the # sources compile byte-for-byte with no edits, no GUI, no windowing # system and no platform startup. # # The other twelve (mod.rs, viewport*.rs, workspace*.rs, script_bindings, # cad_editor_sheet, code_editor, tools, profile_benchmarks) are widget # code: they need `live_design!`, `Cx`, `Widget` and a real event loop. # They are gated by the `full-crate-check` job in # .forgejo/workflows/nigig-build.yml, not by this script, and this script # does not pretend to measure them. # # WHAT IT EXCLUDES FROM THE REPORT (step 4 of the coverage plan) # - the Makepad checkout -- generated/vendored upstream code # - the cargo registry and git dirs -- third-party code # - the rustc sysroot -- std # - harness/src/lib.rs, shim/, picker/ -- the platform-startup stand-ins # this script writes itself; they are scaffolding, not CAD code, and # counting them would flatter (or deflate) the number for no reason. # # USAGE # ./tools/test-cad-coverage.sh # run, report, clean up # KEEP_COVERAGE=1 ./tools/test-cad-coverage.sh # keep env + uncovered lines # # By default everything -- toolchain, cargo home, target dir, profraw # data, the fetched Makepad tree and the report -- lives under a single # mktemp directory that a shell trap removes on success, failure, # interrupt or termination. Nothing is written into the repository or # $HOME. That is the mode CI runs, and it is the only mode whose result # is reproducible from nothing. # # Three opt-in knobs make the measure-edit-measure loop bearable # locally, where a cold run spends ~90s installing a toolchain and ~60s # compiling printpdf before it measures anything: # # CAD_COV_MAKEPAD=/path/to/makepad reuse a Makepad checkout # CAD_COV_TOOLCHAIN_HOME=/path reuse RUSTUP_HOME + CARGO_HOME # (expects $path/rustup and # $path/cargo; installs into # them once if absent) # CAD_COV_TARGET_DIR=/path reuse the build cache # # A reused directory is NOT deleted by the trap -- it lives outside # $WORK by definition, and silently deleting a path the caller named # would be a surprise. Set none of them and the run is hermetic. # # # first run seeds the cache, later runs take seconds # export CAD_COV_TOOLCHAIN_HOME=~/.cache/cad-cov CAD_COV_TARGET_DIR=~/.cache/cad-cov/target # # BENCHMARK MODE # # CAD_BENCH=1 ./tools/test-cad-coverage.sh # # Same harness, one extra file (profile_benchmarks.rs), built --release # with no instrumentation, running the `#[ignore]`-d benchmarks. This is # the host-only equivalent of # # cargo test -p nigig-build cad::profile_benchmarks -- --nocapture --test-threads=1 # # which REVIEWS/PAY_CAD_IMPLEMENTATION_STATUS.md listed as blocked on "no # Cargo toolchain available". Coverage instrumentation is deliberately # off here: -C instrument-coverage -C opt-level=0 makes every number # meaningless. Compare the output against BENCH_BASELINE.md. set -Eeuo pipefail IFS=$'\n\t' ROOT="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" WORK="$(mktemp -d "${TMPDIR:-/tmp}/cad-coverage.XXXXXXXX")" KEEP_COVERAGE="${KEEP_COVERAGE:-0}" # Floors, set a couple of points under today's measurement so ordinary # refactoring does not trip them while a real loss of coverage does. # A single total hides the failure this is meant to catch: losing every # test in persistence.rs moves the total by under two points. # # The low floors are the honest ones, not the aspirational ones: # arch_pdf -- a large emitter whose remaining gaps are # byte-layout paths reached only by a real # PDF consumer. # exporters -- the ExportTarget::Prompt arm needs a # windowing system. TOTAL_FLOOR="${CAD_COVERAGE_TOTAL_FLOOR:-96}" PER_FILE_FLOORS="${CAD_COVERAGE_PER_FILE_FLOORS:-\ math.rs:97 section_shape.rs:99 construction_geometry.rs:95 constants.rs:95 cad_scene.rs:96 commands.rs:94 scene_holder.rs:99 arch_stl.rs:98 arch_svg.rs:97 arch_gltf.rs:97 arch_pdf.rs:86 exporters.rs:88 persistence.rs:90 send_sync_audit.rs:99 tools.rs:95 nav_pad.rs:99 render_budget.rs:99 cull.rs:99 batching.rs:99}" TOOLCHAIN="$(sed -n 's/^channel = "\(.*\)"/\1/p' "$ROOT/rust-toolchain.toml")" HOST_TRIPLE="${CAD_COV_HOST:-x86_64-unknown-linux-gnu}" cleanup() { local status=$? if [[ "$KEEP_COVERAGE" == "1" ]]; then echo "coverage environment retained: $WORK" >&2 else rm -rf "$WORK" echo "cleaned isolated coverage environment" >&2 fi exit "$status" } trap cleanup EXIT HUP INT TERM CAD="$ROOT/crates/apps/nigig-build/src/construction_frame/pages/workspace/cad" MANIFEST="$ROOT/crates/apps/nigig-build/Cargo.toml" # The engine files, in dependency order for a human reader. Adding a new # pure module to the CAD directory means adding it here too, otherwise it # is silently unmeasured -- so the script checks for that at the end. ENGINE_FILES=( math.rs section_shape.rs construction_geometry.rs constants.rs cad_scene.rs commands.rs scene_holder.rs arch_stl.rs arch_svg.rs arch_gltf.rs arch_pdf.rs exporters.rs persistence.rs send_sync_audit.rs tools.rs nav_pad.rs render_budget.rs cull.rs batching.rs ) # profile_benchmarks.rs is engine code too -- its only Makepad imports # are the math types -- but it is measured separately: the benchmarks are # `#[ignore]`-d, so counting their lines would report the file as mostly # uncovered and say nothing useful. BENCH_MODE="${CAD_BENCH:-0}" if [[ "$BENCH_MODE" == "1" ]]; then ENGINE_FILES+=(profile_benchmarks.rs) fi # --------------------------------------------------------------------------- # 1. Isolated toolchain with the coverage instrumentation components # --------------------------------------------------------------------------- TOOLCHAIN_HOME="${CAD_COV_TOOLCHAIN_HOME:-$WORK}" export RUSTUP_HOME="$TOOLCHAIN_HOME/rustup" export CARGO_HOME="$TOOLCHAIN_HOME/cargo" export CARGO_TARGET_DIR="${CAD_COV_TARGET_DIR:-$WORK/target}" export PATH="$CARGO_HOME/bin:$PATH" export LLVM_PROFILE_FILE="$WORK/profiles/%p-%m.profraw" if [[ "$BENCH_MODE" == "1" ]]; then # No instrumentation and no -O0: a benchmark built that way measures # the instrumentation. export RUSTFLAGS="" else export RUSTFLAGS="-C instrument-coverage -C codegen-units=1 -C opt-level=0" fi mkdir -p "$WORK/profiles" if [[ -x "$CARGO_HOME/bin/cargo" ]]; then echo "reusing toolchain: $CARGO_HOME" >&2 else curl --fail --location https://sh.rustup.rs -o "$WORK/rustup-init" chmod 700 "$WORK/rustup-init" "$WORK/rustup-init" -y --profile minimal --default-toolchain "$TOOLCHAIN" \ --component llvm-tools-preview --no-modify-path fi # --------------------------------------------------------------------------- # 2. Makepad sources for makepad-math + makepad-csg, at the pinned rev # --------------------------------------------------------------------------- # Only these two libraries are needed. Both are pure Rust with no system # dependencies -- that is the whole reason the engine can be measured # without a desktop stack. if [[ -n "${CAD_COV_MAKEPAD:-}" ]]; then MAKEPAD="$CAD_COV_MAKEPAD" echo "reusing Makepad checkout: $MAKEPAD" >&2 else MAKEPAD_REV="$(sed -n 's/.*makepad-widgets.*rev = "\([0-9a-f]\{40\}\)".*/\1/p' \ "$MANIFEST" | head -1)" [[ -n "$MAKEPAD_REV" ]] || { echo "cannot read the pinned makepad rev"; exit 1; } MAKEPAD="$WORK/makepad" git init --quiet "$MAKEPAD" git -C "$MAKEPAD" remote add origin https://gitdab.com/andodeki/makepad # Sparse + blobless + depth 1: three directories instead of the whole # fork. 29 MB and two seconds, against 319 MB for a plain shallow # checkout of a repository that is mostly shaders, fonts and demos. # The closure of path dependencies, not just the two crates named # above: math -> micro_serde -> live_id, each with a proc-macro # sibling. A missing one fails at manifest-read time, not at compile # time, so it is worth listing them explicitly. git -C "$MAKEPAD" sparse-checkout set --cone \ libs/math libs/csg libs/micro_serde libs/live_id libs/micro_proc_macro git -C "$MAKEPAD" fetch --quiet --depth 1 --filter=blob:none origin "$MAKEPAD_REV" git -C "$MAKEPAD" checkout --quiet FETCH_HEAD fi # The full path-dependency closure of makepad-math + makepad-csg, # resolved from the manifests rather than guessed: # math -> micro_serde -> {derive, live_id -> id_macros}, and # micro_serde's derive -> micro_proc_macro; csg -> six csg_* siblings. for manifest in libs/math libs/csg/csg libs/csg/csg_math libs/csg/csg_mesh \ libs/csg/csg_boolean libs/csg/csg_exact libs/csg/csg_primitives \ libs/csg/csg_sdf libs/micro_serde libs/micro_serde/derive \ libs/micro_proc_macro libs/live_id libs/live_id/id_macros; do test -f "$MAKEPAD/$manifest/Cargo.toml" \ || { echo "missing $manifest in the Makepad checkout"; exit 1; } done # --------------------------------------------------------------------------- # 3. Assemble the host-only harness # --------------------------------------------------------------------------- HARNESS="$WORK/harness" mkdir -p "$HARNESS/src/construction_frame/pages/workspace/cad" \ "$HARNESS/tests" "$WORK/shim/src" "$WORK/picker/src" cat > "$WORK/shim/Cargo.toml" < "$WORK/shim/src/lib.rs" <<'EOF' //! Host-only stand-in for `makepad_widgets`. It re-exports exactly what //! the CAD engine files take from Makepad -- the math types, the CSG //! library and the two log macros -- and nothing from the GUI or //! platform layer, which is what keeps this build headless. pub use makepad_csg; pub use makepad_math; pub use makepad_math::*; #[macro_export] macro_rules! log { ($($arg:tt)*) => {{ let _ = format_args!($($arg)*); }}; } #[macro_export] macro_rules! error { ($($arg:tt)*) => {{ let _ = format_args!($($arg)*); }}; } EOF cat > "$WORK/picker/Cargo.toml" <<'EOF' [package] name = "robius-file-picker" version = "0.1.0" edition = "2021" [lib] name = "robius_file_picker" path = "src/lib.rs" EOF cat > "$WORK/picker/src/lib.rs" <<'EOF' //! Headless stand-in for `robius-file-picker`. A save dialog cannot be //! raised without a windowing system, so `save_data` reports the //! unsupported condition rather than pretending a file was written. //! Nothing in the suite asserts against this behaviour: the //! `ExportTarget::Prompt` arm of `exporters::deliver_bytes` is dialog //! bound and is reported as uncovered, which is the honest result. use std::io; use std::path::PathBuf; #[derive(Default)] pub struct FileDialog { title: String, file_name: String, } impl FileDialog { pub fn new() -> Self { Self::default() } pub fn set_title(mut self, title: impl Into) -> Self { self.title = title.into(); self } pub fn set_file_name(mut self, name: impl Into) -> Self { self.file_name = name.into(); self } pub fn save_data(self, _bytes: Vec, _cb: F) -> Result<(), io::Error> where F: FnOnce(Result, io::Error>) + Send + 'static, { let _ = (&self.title, &self.file_name); Err(io::Error::new( io::ErrorKind::Unsupported, "no windowing system in the coverage harness", )) } } EOF cat > "$HARNESS/Cargo.toml" < "$HARNESS/src/lib.rs" <<'EOF' //! Coverage harness root. Scaffolding only -- excluded from the report. pub use makepad_csg; /// Stand-in for `nigig_core::dir`, which pulls in the platform layer. pub mod dir { use std::path::PathBuf; pub fn app_data_dir() -> PathBuf { std::env::var_os("NIGIG_CAD_COV_DATA_DIR") .map(PathBuf::from) .unwrap_or_else(std::env::temp_dir) } } pub mod construction_frame { pub mod pages { pub mod workspace { pub mod cad; } } } EOF DEST="$HARNESS/src/construction_frame/pages/workspace/cad" for f in "${ENGINE_FILES[@]}"; do cp "$CAD/$f" "$DEST/$f" done cp "$ROOT/crates/apps/nigig-build/tests/cad_integration.rs" "$HARNESS/tests/" { echo "//! Engine-only view of the CAD module. Generated by" echo "//! tools/test-cad-coverage.sh -- do not edit." for f in "${ENGINE_FILES[@]}"; do echo "pub mod ${f%.rs};" done # The real mod.rs re-exports the scene types so `use super::*` in a # child module reaches them. tools.rs relies on that for PartKind. echo "pub(crate) use cad_scene::*;" echo "pub(crate) use math::*;" # ...and the Makepad math types the real mod.rs pulls in, so a child # module's `use super::*` reaches DVec2 the way it does in production. echo "pub(crate) use makepad_widgets::{DVec2, Mat4f, Vec3f, Vec4f};" } > "$DEST/mod.rs" # The real mod.rs is widget-bound and cannot compile host-only, but it # also declares a handful of PLAIN types -- ordinary structs and enums # with ordinary derives -- that the engine files and the integration # suite use. Those are EXTRACTED from the real file at run time rather # than copied here, so the harness cannot drift away from the crate: if a # declaration changes shape, this run compiles the new shape or fails. # # tools.rs is the reason the list is longer than two. It is 253 lines of # pure tool-state logic (tool cycling, work-plane axis mapping, inclined # UCS maths) whose own header comment claims its types must live in # mod.rs because "they use #[derive] macros that need the script_mod! # context". They do not: every one of them derives Clone/Copy/Debug/ # PartialEq and nothing else, and none is inside the script_mod! block. # The real obstacle to moving them out is narrower -- DrawingState, # SnapSettings and InclinedPlane have private fields that mod.rs and # viewport.rs read directly, so a move needs those widened first. That # is a refactor for someone with a compiler for the widget layer; # mirroring the declarations here measures the logic today without # touching production code at all. python3 - "$CAD/mod.rs" "$DEST/mod.rs" <<'PY' import re, sys src, dest = sys.argv[1], sys.argv[2] text = open(src, encoding="utf-8").read() # (name, kind) pairs. Order matters only for readability. WANTED = [ ("ViewMode", "enum"), ("SelectionMode", "enum"), ("CadTool", "enum"), ("WorkPlane", "enum"), ("AxisLock", "enum"), ("RefPlane", "struct"), ("InclinedPlane", "struct"), ("DrawingState", "struct"), ("SnapSettings", "struct"), ] out = ["\n// Extracted verbatim from the crate's mod.rs at run time. Everything", "// here is a plain data declaration; the rest of mod.rs is widget code.", "// Visibility is widened to pub(crate) so the mirrored module tree can", "// see them from the same relative position the real one does.\n"] for name, kind in WANTED: m = re.search( r"(#\[derive\([^)]*\)\]\n)?(?:pub(?:\(crate\))? )?%s %s \{.*?\n\}" % (kind, name), text, re.S) if not m: sys.exit(f"ERROR: {kind} {name} not found in {src}; the harness is out of date") decl = m.group(0) decl = re.sub(r"(?m)^(#\[derive[^\n]*\]\n)?(pub(?:\(crate\))? )?(%s %s)" % (kind, name), lambda mm: (mm.group(1) or "") + "pub " + mm.group(3), decl, count=1) out.append(decl + "\n") open(dest, "a", encoding="utf-8").write("\n".join(out)) PY # A new pure module added to cad/ but not to ENGINE_FILES would be # unmeasured and nobody would notice. Say so loudly. for f in "$CAD"/*.rs; do base="$(basename "$f")" # Membership by explicit loop: `${ENGINE_FILES[*]}` joins on the first # character of IFS, which this script sets to a newline, so the obvious # `case " ${ENGINE_FILES[*]} " in *" $base "*)` never matches. listed=0 for known in "${ENGINE_FILES[@]}"; do [[ "$known" == "$base" ]] && { listed=1; break; } done [[ "$listed" == "1" ]] && continue # Measured by CAD_BENCH=1 instead; see the note on ENGINE_FILES. [[ "$base" == "profile_benchmarks.rs" ]] && continue if ! grep -qE 'live_design!|impl Widget|&mut Cx|Live, LiveHook|use makepad_widgets::\*' "$f"; then echo "NOTE: $base has no widget markers but is not in ENGINE_FILES." >&2 echo " If it is pure, add it so it gets measured." >&2 fi done # --------------------------------------------------------------------------- # 4. Instrumented run: unit tests (in-file #[cfg(test)]) + integration suite # --------------------------------------------------------------------------- if [[ "$BENCH_MODE" == "1" ]]; then echo echo "=== CAD benchmarks (release, host-only). Compare against BENCH_BASELINE.md." echo "=== Absolute numbers are machine-dependent; the ratios are the signal." echo cargo test --release --manifest-path "$HARNESS/Cargo.toml" --lib -- \ --ignored --nocapture --test-threads=1 profile_benchmarks exit 0 fi cargo test --manifest-path "$HARNESS/Cargo.toml" --all-targets # --------------------------------------------------------------------------- # 5. Report # --------------------------------------------------------------------------- LLVM_BIN="$RUSTUP_HOME/toolchains/$TOOLCHAIN-$HOST_TRIPLE/lib/rustlib/$HOST_TRIPLE/bin" # A reused toolchain home might have been installed without the # coverage components. Say which component is missing rather than # failing on a "no such file" three lines later. if [[ ! -x "$LLVM_BIN/llvm-profdata" ]]; then echo "ERROR: llvm-tools-preview is missing from $RUSTUP_HOME." >&2 echo " rustup component add llvm-tools-preview --toolchain $TOOLCHAIN" >&2 exit 1 fi "$LLVM_BIN/llvm-profdata" merge -sparse "$WORK/profiles"/*.profraw \ -o "$WORK/coverage.profdata" mapfile -t BINS < <(find "$CARGO_TARGET_DIR/debug/deps" -maxdepth 1 -type f -executable \ \( -name 'nigig_build-*' -o -name 'cad_integration-*' \) ! -name '*.d') [[ ${#BINS[@]} -gt 0 ]] || { echo "no instrumented test binaries found"; exit 1; } OBJECTS=("${BINS[0]}") for b in "${BINS[@]:1}"; do OBJECTS+=(-object "$b"); done # Two independent exclusions, because either one alone leaks: # - the regex drops third-party/generated code wherever it was built # from (the Makepad checkout can live outside $WORK when # CAD_COV_MAKEPAD is set, so its real path is spliced in); # - the explicit source list means only the fourteen engine files and # the integration suite contribute to TOTAL, so the number cannot be # diluted by scaffolding that happens to sit under $WORK. MAKEPAD_ESC="$(printf '%s' "$MAKEPAD" | sed 's/[][\.^$*+?(){}|\/]/\\&/g')" IGNORE="(/cargo/registry|/cargo/git|/rustc/|$MAKEPAD_ESC|/shim/|/picker/|harness/src/lib\.rs)" SOURCES=() for f in "${ENGINE_FILES[@]}"; do SOURCES+=("$DEST/$f") done SOURCES+=("$HARNESS/tests/cad_integration.rs") "$LLVM_BIN/llvm-cov" report "${OBJECTS[@]}" \ -instr-profile="$WORK/coverage.profdata" -ignore-filename-regex="$IGNORE" \ "${SOURCES[@]}" if [[ "$KEEP_COVERAGE" == "1" ]]; then "$LLVM_BIN/llvm-cov" show "${OBJECTS[@]}" \ -instr-profile="$WORK/coverage.profdata" -ignore-filename-regex="$IGNORE" \ "${SOURCES[@]}" \ | grep -E '^ *[0-9]+\| *0\|' > "$WORK/uncovered-lines.txt" || true echo "uncovered line report: $WORK/uncovered-lines.txt" >&2 fi # --------------------------------------------------------------------------- # 6. Enforce the floors # --------------------------------------------------------------------------- "$LLVM_BIN/llvm-cov" export "${OBJECTS[@]}" \ -instr-profile="$WORK/coverage.profdata" -ignore-filename-regex="$IGNORE" \ "${SOURCES[@]}" > "$WORK/coverage.json" if [[ "${CAD_COVERAGE_REPORT_ONLY:-0}" == "1" ]]; then echo 'report-only mode: the floors were not enforced' exit 0 fi python3 - "$WORK/coverage.json" "$TOTAL_FLOOR" "$PER_FILE_FLOORS" <<'PY' import json, sys path, total_floor, per_file = sys.argv[1], float(sys.argv[2]), sys.argv[3] with open(path) as fh: data = json.load(fh) export = data["data"][0] total = export["totals"]["lines"]["percent"] measured = {f["filename"]: f["summary"]["lines"]["percent"] for f in export["files"]} failures = [] if total < total_floor: failures.append(f" total {total:.2f}% is below the floor of {total_floor:.2f}%") for line in per_file.split(): if not line.strip(): continue name, _, floor = line.rpartition(":") floor = float(floor) hits = [v for k, v in measured.items() if k.endswith("/cad/" + name)] if not hits: failures.append( f" {name} has a floor but was not measured -- was it renamed, " "deleted, or dropped from ENGINE_FILES? A floor on a file that " "is not measured silently protects nothing.") continue if hits[0] < floor: failures.append( f" {name} {hits[0]:.2f}% is below its floor of {floor:.2f}%") if failures: print("coverage floors not met:", file=sys.stderr) print("\n".join(failures), file=sys.stderr) print( "\nEither the change removed tested code, or it added untested code.\n" "Lowering a floor is a reviewable edit to tools/test-cad-coverage.sh,\n" "not something to do quietly.", file=sys.stderr, ) sys.exit(1) print(f"all coverage floors met (total {total:.2f}%)") PY