nigig-org/tools/test-pdf-coverage.sh

#!/usr/bin/env bash
# LLVM source-coverage for the PDF engine crates, with an enforced floor.
#
# Why a floor and not a report: coverage that is only ever *printed* drifts
# down. Every silently-empty bug this stack has shipped lived in a file
# nobody was watching - `image.rs` sat at 14% while the JPEG decoder was a
# stub returning black rectangles, and no build ever said so.
#
# The floor is deliberately set a little under today's measurement, so
# ordinary refactoring does not trip it but a real regression does. It is
# not a target: raising a number here is only meaningful when the tests
# that raised it assert values. See
# `REVIEWS/adr/0017-pdf-declared-versus-delivered.md`.
#
# Usage:
#   ./tools/test-pdf-coverage.sh              # enforce the floor
#   PDF_COVERAGE_REPORT_ONLY=1 ./tools/test-pdf-coverage.sh
#
# Runs entirely inside one temporary directory: its own rustup, cargo home
# and target dir, all removed on exit. It does not touch ~/.cargo, the
# workspace target directory or any source file.

set -Eeuo pipefail
IFS=$'\n\t'

ROOT="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
TOOLCHAIN="${RUST_TOOLCHAIN:-1.97.1}"

# The whole-stack floor. Measured at 83.9%; set below that so a routine
# change does not fail the build, while a real loss of coverage does.
TOTAL_FLOOR="${PDF_COVERAGE_TOTAL_FLOOR:-80}"

# Per-file floors for the files that have actually harboured the bugs.
#
# A single whole-stack number hides exactly the failure this is meant to
# catch: `image.rs` can fall from 33% to 5% while the total moves by less
# than a point, because the total is dominated by the files that were
# already well covered. Each entry is "path:floor", set a few points under
# the current measurement.
#
# These are floors on files that were once badly covered, not a list of
# files that matter. A file missing from here is still counted in the total.
PER_FILE_FLOORS="${PDF_COVERAGE_PER_FILE_FLOORS:-\
pdf-graphics/src/image.rs:42
pdf-graphics/src/jpeg.rs:74
pdf-graphics/src/font.rs:40
pdf-graphics/src/cmap.rs:30
pdf-graphics/src/content_writer.rs:50
pdf-graphics/src/graphics_state.rs:60
pdf-cos/src/filter.rs:70
pdf-cos/src/ccitt.rs:88
pdf-cos/src/jbig2.rs:90
pdf-cos/src/jpx.rs:88
pdf-graphics/src/stamp.rs:90
pdf-document/src/page.rs:88
pdf-document/src/document.rs:72
pdf-document/src/destinations.rs:94
pdf-document/src/annotations.rs:70}"

WORK="$(mktemp -d "${TMPDIR:-/tmp}/pdf-coverage.XXXXXXXX")"
cleanup() {
  local status=$?
  if [ "${KEEP_TEST_ENV:-0}" = "1" ]; then
    printf 'kept coverage environment at %s\n' "$WORK" >&2
  else
    chmod -R u+w "$WORK" 2>/dev/null || true
    rm -rf "$WORK"
  fi
  exit "$status"
}
trap cleanup EXIT HUP INT TERM

export RUSTUP_HOME="$WORK/rustup"
export CARGO_HOME="$WORK/cargo"
export CARGO_TARGET_DIR="$WORK/target"
export PATH="$CARGO_HOME/bin:$PATH"
export LLVM_PROFILE_FILE="$WORK/profiles/%p-%m.profraw"
export RUSTFLAGS="-C instrument-coverage -C codegen-units=1 -C opt-level=0"
mkdir -p "$WORK/profiles"

printf 'installing Rust %s with llvm-tools\n' "$TOOLCHAIN"
curl --fail --silent --show-error --location https://sh.rustup.rs \
  -o "$WORK/rustup-init"
chmod 700 "$WORK/rustup-init"
"$WORK/rustup-init" -y --profile minimal --default-toolchain "$TOOLCHAIN" \
  --component llvm-tools-preview --no-modify-path >/dev/null

# The three engine crates are wired by relative path dependencies, so they
# are copied together into a standalone workspace. pdf-makepad is excluded:
# it needs GUI system libraries and is not part of the engine's coverage.
SRC="$ROOT/crates/apps/pdf"
mkdir -p "$WORK/pdf"
cp -a "$SRC/pdf-cos" "$SRC/pdf-document" "$SRC/pdf-graphics" "$SRC/tests" \
  "$WORK/pdf/"
# The fuzz directory is its own workspace and breaks the copied one.
rm -rf "$WORK/pdf/pdf-cos/fuzz"

cat > "$WORK/pdf/Cargo.toml" <<'TOML'
[workspace]
members = ["pdf-cos", "pdf-document", "pdf-graphics"]
resolver = "2"
TOML

printf 'running the engine test suites under instrumentation\n'
cargo test --manifest-path "$WORK/pdf/Cargo.toml" --workspace --all-targets \
  >"$WORK/test.log" 2>&1 || { cat "$WORK/test.log"; exit 1; }
grep -E 'test result' "$WORK/test.log" | tail -20

HOST="$(rustc -vV | sed -n 's/^host: //p')"
LLVM_BIN="$RUSTUP_HOME/toolchains/$TOOLCHAIN-$HOST/lib/rustlib/$HOST/bin"

"$LLVM_BIN/llvm-profdata" merge -sparse "$WORK/profiles"/*.profraw \
  -o "$WORK/coverage.profdata"

# Every instrumented test binary contributes; a single one under-reports
# badly, because each crate's tests cover the crates beneath it.
mapfile -t BINARIES < <(
  find "$CARGO_TARGET_DIR/debug/deps" -maxdepth 1 -type f -executable \
    ! -name '*.d' ! -name '*.so' | sort
)
OBJECTS=()
for bin in "${BINARIES[@]}"; do
  OBJECTS+=(-object "$bin")
done
if [ "${#OBJECTS[@]}" -eq 0 ]; then
  echo 'no instrumented binaries were produced' >&2
  exit 1
fi

# Only the crates' own `src/` trees count. The work directory is named
# `pdf-coverage.*`, so it must not appear in this pattern: matching it once
# excluded every source file and reported a confident 0.00%.
IGNORE='(/cargo/registry|/cargo/git|/rustc/|/pdf/[^/]+/tests/|/pdf/tests/)'

printf '\n=== per-file coverage ===\n'
"$LLVM_BIN/llvm-cov" report "${OBJECTS[@]}" \
  -instr-profile="$WORK/coverage.profdata" \
  -ignore-filename-regex="$IGNORE" \
  | tee "$WORK/report.txt"

# `llvm-cov export` gives a machine-readable total rather than a parsed
# table, so the gate does not depend on column alignment.
"$LLVM_BIN/llvm-cov" export "${OBJECTS[@]}" \
  -instr-profile="$WORK/coverage.profdata" \
  -ignore-filename-regex="$IGNORE" \
  -summary-only > "$WORK/coverage.json"

TOTAL="$(python3 - "$WORK/coverage.json" <<'PY'
import json, sys
with open(sys.argv[1]) as fh:
    data = json.load(fh)
totals = data["data"][0]["totals"]["lines"]
print(f'{totals["percent"]:.2f}')
PY
)"

printf '\ntotal line coverage: %s%% (floor %s%%)\n' "$TOTAL" "$TOTAL_FLOOR"

if [ "${PDF_COVERAGE_REPORT_ONLY:-0}" = "1" ]; then
  echo 'report-only mode: the floor was not enforced'
  exit 0
fi

python3 - "$WORK/coverage.json" "$TOTAL_FLOOR" "$PER_FILE_FLOORS" <<'PY'
import json
import sys

path, total_floor, per_file = sys.argv[1], float(sys.argv[2]), sys.argv[3]
with open(path) as fh:
    data = json.load(fh)

failures = []

total = data["data"][0]["totals"]["lines"]["percent"]
if total < total_floor:
    failures.append(
        f"  total {total:.2f}% is below the floor of {total_floor:.2f}%"
    )

# Match on suffix: the report carries absolute paths inside the throwaway
# work directory, which differ every run.
measured = {}
for entry in data["data"][0]["files"]:
    measured[entry["filename"]] = entry["summary"]["lines"]["percent"]

for line in per_file.split():
    if not line.strip():
        continue
    name, _, floor = line.rpartition(":")
    floor = float(floor)
    matches = [v for k, v in measured.items() if k.endswith(name)]
    if not matches:
        failures.append(
            f"  {name} has a floor but was not measured - was it renamed or "
            "deleted? A floor on a file that no longer exists silently "
            "protects nothing."
        )
        continue
    got = matches[0]
    if got < floor:
        failures.append(f"  {name} {got:.2f}% is below its floor of {floor:.2f}%")

if failures:
    print("coverage floors not met:", file=sys.stderr)
    print("\n".join(failures), file=sys.stderr)
    print(
        "\nEither the change removed tested code, or it added untested code.\n"
        "Lowering a floor is a reviewable edit to tools/test-pdf-coverage.sh,\n"
        "not something to do quietly.",
        file=sys.stderr,
    )
    sys.exit(1)

print(f"all coverage floors met (total {total:.2f}%)")
PY