makepad/libs/sqlite_query/tests/crash.rs

//! Crash injection: kill the process at every durable write step of a commit,
//! reopen the database, and require that it is intact and shows either the old
//! state or the new one — never a mixture, never a torn page.
//!
//! The child process is this same test binary re-executed with
//! `MAKEPAD_SQLITE_CRASH_CHILD` set; the pager's crash hook aborts it (no
//! unwinding, no flushing) after the requested number of write steps.

mod common;

use common::*;
use makepad_sqlite::{Connection, Database, Value};
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::Duration;

const ROWS_BEFORE: i64 = 60;
const ROWS_ADDED: i64 = 40;

fn setup(path: &Path) {
    setup_mode(path, false)
}

fn setup_mode(path: &Path, wal: bool) {
    let mut db = Connection::open(path, Duration::from_secs(5)).expect("open");
    if wal {
        db.execute("PRAGMA journal_mode=WAL", &[]).expect("wal");
    }
    db.execute(
        "CREATE TABLE t(id INTEGER PRIMARY KEY, k TEXT, v BLOB)",
        &[],
    )
    .unwrap();
    db.execute("CREATE INDEX t_by_k ON t(k)", &[]).unwrap();
    db.execute("BEGIN IMMEDIATE", &[]).unwrap();
    for i in 1..=ROWS_BEFORE {
        db.execute(
            "INSERT INTO t(id, k, v) VALUES (?1, ?2, ?3)",
            &[
                Value::Integer(i),
                Value::text(format!("k{i:04}")),
                Value::Blob(vec![1u8; 120]),
            ],
        )
        .unwrap();
    }
    db.execute("COMMIT", &[]).unwrap();
}

/// The transaction the child runs; also used with no crash to count the steps.
fn mutate(path: &Path) {
    let mut db = Connection::open(path, Duration::from_secs(5)).expect("open");
    db.execute("BEGIN IMMEDIATE", &[]).unwrap();
    for i in ROWS_BEFORE + 1..=ROWS_BEFORE + ROWS_ADDED {
        db.execute(
            "INSERT INTO t(id, k, v) VALUES (?1, ?2, ?3)",
            &[
                Value::Integer(i),
                Value::text(format!("k{i:04}")),
                Value::Blob(vec![2u8; 900]),
            ],
        )
        .unwrap();
    }
    db.execute("DELETE FROM t WHERE id <= 20", &[]).unwrap();
    db.execute("UPDATE t SET k = 'updated' WHERE id BETWEEN 30 AND 40", &[])
        .unwrap();
    db.execute("COMMIT", &[]).unwrap();
}

/// Child mode: run the transaction with the crash countdown armed.
#[test]
fn crash_child() {
    let Ok(path) = std::env::var("MAKEPAD_SQLITE_CRASH_CHILD") else {
        return; // not the child: nothing to do
    };
    let at: u64 = std::env::var("MAKEPAD_SQLITE_CRASH_AT")
        .ok()
        .and_then(|v| v.parse().ok())
        .unwrap_or(0);
    makepad_sqlite::pager::set_crash_after(at);
    mutate(Path::new(&path));
    // Reaching this point means the crash point was past the end of the commit.
    println!("STEPS {}", makepad_sqlite::pager::write_steps());
}

fn run_child(db: &Path, crash_at: u64) -> (bool, String) {
    let exe = std::env::current_exe().expect("test binary path");
    let out = Command::new(exe)
        .args(["--exact", "crash_child", "--nocapture"])
        .env("MAKEPAD_SQLITE_CRASH_CHILD", db)
        .env("MAKEPAD_SQLITE_CRASH_AT", crash_at.to_string())
        .output()
        .expect("spawn child");
    (
        out.status.success(),
        String::from_utf8_lossy(&out.stdout).into_owned(),
    )
}

fn state(path: &Path) -> (i64, i64, i64) {
    let mut db = Database::open(path).expect("reopen");
    let rows = db
        .query("SELECT COUNT(*) FROM t", &[])
        .expect("count")
        .rows[0][0]
        .as_integer()
        .unwrap_or(-1);
    let updated = db
        .query("SELECT COUNT(*) FROM t WHERE k = 'updated'", &[])
        .expect("count updated")
        .rows[0][0]
        .as_integer()
        .unwrap_or(-1);
    let max = db
        .query("SELECT COALESCE(MAX(id), 0) FROM t", &[])
        .expect("max")
        .rows[0][0]
        .as_integer()
        .unwrap_or(-1);
    (rows, updated, max)
}

fn recover_and_check(path: &Path, scratch: &Scratch, tag: &str) -> (i64, i64, i64) {
    // Opening read-write is what performs hot-journal recovery.
    {
        let _conn = Connection::open(path, Duration::from_secs(5)).expect("recover");
    }
    let mut db = Database::open(path).expect("reopen");
    let (pager, schema) = db.parts();
    let report = makepad_sqlite::integrity::check(pager, schema, true).expect("integrity");
    if !report.ok() {
        let keep = scratch.path(&format!("broken-{tag}.db"));
        let _ = std::fs::copy(path, &keep);
        panic!(
            "after a crash at {tag} the database is damaged:\n{}",
            report.problems.join("\n")
        );
    }
    drop(db);
    if have_sqlite3() {
        let out = sqlite3(path, "PRAGMA integrity_check;\n");
        assert_eq!(out.trim(), "ok", "sqlite3 disagrees after a crash at {tag}");
    }
    state(path)
}

#[test]
fn every_crash_point_leaves_a_consistent_database() {
    if std::env::var("MAKEPAD_SQLITE_CRASH_CHILD").is_ok() {
        return; // this process is a child
    }
    let scratch = Scratch::new("crash");
    let pristine = scratch.path("pristine.db");
    setup(&pristine);
    let before = state(&pristine);
    assert_eq!(before.0, ROWS_BEFORE);

    // How many write steps a full commit takes.
    let work = scratch.path("count.db");
    std::fs::copy(&pristine, &work).unwrap();
    let (ok, out) = run_child(&work, 0);
    assert!(ok, "uncrashed child failed: {out}");
    let steps: u64 = out
        .lines()
        .find_map(|l| l.strip_prefix("STEPS "))
        .and_then(|v| v.trim().parse().ok())
        .expect("child reported no step count");
    assert!(steps > 5, "expected a multi-step commit, got {steps}");
    eprintln!("crash harness: {steps} durable write steps per commit");
    let after = state(&work);
    assert_eq!(after.0, ROWS_BEFORE + ROWS_ADDED - 20);
    assert_eq!(after.1, 11);

    // Crash at every single step and check what survives.
    let mut crashed = 0;
    let mut committed = 0;
    for at in 1..=steps {
        let path = scratch.path(&format!("crash-{at}.db"));
        std::fs::copy(&pristine, &path).unwrap();
        let (ok, _) = run_child(&path, at);
        if !ok {
            crashed += 1;
        }
        let recovered = recover_and_check(&path, &scratch, &at.to_string());
        // The only two acceptable outcomes: the old state or the new one.
        assert!(
            recovered == before || recovered == after,
            "crash at step {at} left a mixture: {recovered:?} (before {before:?}, after {after:?})"
        );
        if recovered == after {
            committed += 1;
        }
        let _ = std::fs::remove_file(&path);
        let _ = std::fs::remove_file(path.with_extension("db-journal"));
    }
    assert!(crashed > 0, "no crash point actually aborted the child");
    assert!(
        committed > 0,
        "no crash point landed after the commit; the harness is not covering the end"
    );
}

#[test]
fn every_crash_point_in_wal_mode_leaves_a_consistent_database() {
    if std::env::var("MAKEPAD_SQLITE_CRASH_CHILD").is_ok() {
        return;
    }
    let scratch = Scratch::new("crash-wal");
    let pristine = scratch.path("pristine.db");
    setup_mode(&pristine, true);
    let pristine_wal = scratch.path("pristine.db-wal");
    let before = state(&pristine);
    assert_eq!(before.0, ROWS_BEFORE);

    let copy_pair = |from: &Path, to: &Path| {
        std::fs::copy(from, to).expect("copy db");
        let from_wal = PathBuf::from(format!("{}-wal", from.display()));
        let to_wal = PathBuf::from(format!("{}-wal", to.display()));
        if from_wal.exists() {
            std::fs::copy(&from_wal, &to_wal).expect("copy wal");
        } else {
            let _ = std::fs::remove_file(&to_wal);
        }
    };
    assert!(pristine_wal.exists(), "the fixture is not in WAL mode");

    let work = scratch.path("count.db");
    copy_pair(&pristine, &work);
    let (ok, out) = run_child(&work, 0);
    assert!(ok, "uncrashed child failed: {out}");
    let steps: u64 = out
        .lines()
        .find_map(|l| l.strip_prefix("STEPS "))
        .and_then(|v| v.trim().parse().ok())
        .expect("child reported no step count");
    eprintln!("wal crash harness: {steps} durable write steps per commit");
    let after = state(&work);
    assert_eq!(after.0, ROWS_BEFORE + ROWS_ADDED - 20);

    let mut committed = 0;
    for at in 1..=steps {
        let path = scratch.path(&format!("wal-crash-{at}.db"));
        copy_pair(&pristine, &path);
        let (_ok, _) = run_child(&path, at);
        let recovered = recover_and_check(&path, &scratch, &format!("wal-{at}"));
        assert!(
            recovered == before || recovered == after,
            "crash at WAL step {at} left a mixture: {recovered:?} (before {before:?}, after {after:?})"
        );
        if recovered == after {
            committed += 1;
        }
        let _ = std::fs::remove_file(&path);
        let _ = std::fs::remove_file(PathBuf::from(format!("{}-wal", path.display())));
    }
    assert!(committed > 0, "no crash point landed after the commit");
}

/// Child mode for the checkpoint sweep: fold the log with the countdown armed.
#[test]
fn crash_child_checkpoint() {
    let Ok(path) = std::env::var("MAKEPAD_SQLITE_CRASH_CHILD_CKPT") else {
        return; // not the child: nothing to do
    };
    let at: u64 = std::env::var("MAKEPAD_SQLITE_CRASH_AT")
        .ok()
        .and_then(|v| v.parse().ok())
        .unwrap_or(0);
    let mut db = Connection::open(Path::new(&path), Duration::from_secs(5)).expect("open");
    makepad_sqlite::pager::set_crash_after(at);
    db.execute("PRAGMA wal_checkpoint", &[]).expect("checkpoint");
    println!("STEPS {}", makepad_sqlite::pager::write_steps());
}

fn run_checkpoint_child(db: &Path, crash_at: u64) -> (bool, String) {
    let exe = std::env::current_exe().expect("test binary path");
    let out = Command::new(exe)
        .args(["--exact", "crash_child_checkpoint", "--nocapture"])
        .env("MAKEPAD_SQLITE_CRASH_CHILD_CKPT", db)
        .env("MAKEPAD_SQLITE_CRASH_AT", crash_at.to_string())
        .output()
        .expect("spawn child");
    (
        out.status.success(),
        String::from_utf8_lossy(&out.stdout).into_owned(),
    )
}

/// CRASH SAFETY OF THE CHECKPOINT ITSELF: a writer killed at any durable
/// step of folding the log into the database leaves either the old WAL over
/// a (possibly torn) database — which recovery replays — or the folded
/// state; the logical content is identical at every point, and it is NEVER a
/// mixture. This is the property that lets a connection own its log for
/// life: nothing about crash recovery depends on idle-time lock courtesy.
/// (Every sweep iteration also re-acquires WAL ownership after an aborted
/// owner, which is the no-stale-lock property: POSIX locks die with the
/// process.)
#[test]
fn every_crash_point_in_a_wal_checkpoint_leaves_a_consistent_database() {
    if std::env::var("MAKEPAD_SQLITE_CRASH_CHILD").is_ok()
        || std::env::var("MAKEPAD_SQLITE_CRASH_CHILD_CKPT").is_ok()
    {
        return;
    }
    let scratch = Scratch::new("crash-ckpt");
    let pristine = scratch.path("pristine.db");
    setup_mode(&pristine, true);
    let pristine_wal = scratch.path("pristine.db-wal");
    assert!(pristine_wal.exists(), "the fixture is not in WAL mode");
    let before = state(&pristine);
    assert_eq!(before.0, ROWS_BEFORE);

    let copy_pair = |from: &Path, to: &Path| {
        std::fs::copy(from, to).expect("copy db");
        let from_wal = PathBuf::from(format!("{}-wal", from.display()));
        let to_wal = PathBuf::from(format!("{}-wal", to.display()));
        if from_wal.exists() {
            std::fs::copy(&from_wal, &to_wal).expect("copy wal");
        } else {
            let _ = std::fs::remove_file(&to_wal);
        }
    };

    let work = scratch.path("count.db");
    copy_pair(&pristine, &work);
    let (ok, out) = run_checkpoint_child(&work, 0);
    assert!(ok, "uncrashed checkpoint child failed: {out}");
    let steps: u64 = out
        .lines()
        .find_map(|l| l.strip_prefix("STEPS "))
        .and_then(|v| v.trim().parse().ok())
        .expect("child reported no step count");
    assert!(steps > 3, "expected a multi-step checkpoint, got {steps}");
    eprintln!("checkpoint crash harness: {steps} durable write steps");

    let mut crashed = 0;
    for at in 1..=steps {
        let path = scratch.path(&format!("ckpt-crash-{at}.db"));
        copy_pair(&pristine, &path);
        let (ok, _) = run_checkpoint_child(&path, at);
        if !ok {
            crashed += 1;
        }
        let recovered = recover_and_check(&path, &scratch, &format!("ckpt-{at}"));
        // A checkpoint changes no logical content: every crash point must
        // recover to exactly the pre-checkpoint state.
        assert_eq!(
            recovered, before,
            "crash at checkpoint step {at} changed the database's content"
        );
        let _ = std::fs::remove_file(&path);
        let _ = std::fs::remove_file(PathBuf::from(format!("{}-wal", path.display())));
    }
    assert!(crashed > 0, "no checkpoint crash point actually aborted the child");
}

#[test]
fn a_hot_journal_is_replayed_by_sqlite_too() {
    if std::env::var("MAKEPAD_SQLITE_CRASH_CHILD").is_ok() || !have_sqlite3() {
        return;
    }
    let scratch = Scratch::new("crash-cli");
    let pristine = scratch.path("p.db");
    setup(&pristine);

    // Find a crash point in the middle of writing the database file.
    let work = scratch.path("count.db");
    std::fs::copy(&pristine, &work).unwrap();
    let (_, out) = run_child(&work, 0);
    let steps: u64 = out
        .lines()
        .find_map(|l| l.strip_prefix("STEPS "))
        .and_then(|v| v.trim().parse().ok())
        .unwrap_or(10);

    let mid = (steps / 2).max(2);
    let path: PathBuf = scratch.path("hot.db");
    std::fs::copy(&pristine, &path).unwrap();
    let (ok, _) = run_child(&path, mid);
    assert!(!ok, "the child should have crashed at step {mid}");
    let journal = PathBuf::from(format!("{}-journal", path.display()));
    if journal.exists() {
        // SQLite must roll the journal back by itself and end up consistent.
        let out = sqlite3(&path, "PRAGMA integrity_check;\nSELECT COUNT(*) FROM t;\n");
        let mut lines = out.lines();
        assert_eq!(lines.next().unwrap_or("").trim(), "ok");
        let count: i64 = lines.next().unwrap_or("0").trim().parse().unwrap_or(-1);
        assert!(
            count == ROWS_BEFORE || count == ROWS_BEFORE + ROWS_ADDED - 20,
            "sqlite recovered a mixture: {count} rows"
        );
    }
}