nigig-org/crates/apps/pdf/pdf-graphics/tests/text_search.rs

//! Phase 8 exit criterion: search and selection integration tests.
//!
//! `search.rs`'s unit tests build runs directly. These start from a real
//! PDF and go through the whole pipeline — parse, interpret, extract,
//! index, search — because the defects this module fixes were all
//! *emergent*: each individual stage was doing what it said, and the
//! answer at the end was still wrong.
//!
//! The two that matter most here:
//!
//! - A word split across runs by a kerning adjustment was unfindable,
//!   while `plain_text()` showed it plainly on the page.
//! - A two-column page was read across rather than down, because the two
//!   columns share their baselines and line grouping came first.
//!
//! See `REVIEWS/adr/0034-pdf-text-search-and-layout.md`.

use std::path::PathBuf;

use nigig_pdf_document::PdfDocument;
use nigig_pdf_graphics::content::parse_content_stream;
use nigig_pdf_graphics::recording::RecordingDevice;
use nigig_pdf_graphics::search::{
    detect_columns_from_segments, detect_lines, layout_text, reading_order, search, SearchOptions,
};
use nigig_pdf_graphics::text::PageText;

fn corpus(relative: &str) -> Vec<u8> {
    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
        .join("../tests/corpus")
        .join(relative);
    std::fs::read(&path).unwrap_or_else(|e| panic!("missing fixture {}: {e}", path.display()))
}

/// Parse a fixture and extract page 0's text the way the application does.
///
/// The fonts matter: without a width table every advance is zero, every
/// run lands on top of the last, and the layout tests below would be
/// measuring nothing. Registering them is what makes this an integration
/// test rather than a longer unit test.
fn page_text(relative: &str) -> PageText {
    let data = corpus(relative);
    let mut doc = PdfDocument::parse(&data).expect("fixture parses");
    let page = doc.page(0).expect("page 0");

    let mut device = RecordingDevice::new();
    for (name, resource) in &page.fonts {
        // The width table matters: without one every advance is zero,
        // every run lands on top of the last, and the layout assertions
        // below would be measuring nothing. The fixtures use base-14
        // fonts, whose metrics are built in.
        // Keyed by the *resource* name (`/F1`), which is what the
        // interpreter looks up — `register_standard_font` keys by the base
        // font name, which never matches a `Tf` operand.
        device.register_font(
            name,
            nigig_pdf_graphics::font::GlyphWidths::from_standard_font(&resource.base_font),
        );
    }
    let ops = parse_content_stream(&page.content_data).expect("content parses");
    nigig_pdf_graphics::content::interpret_ops(&ops, &mut device).expect("interprets");
    PageText::from_commands(&device.into_commands())
}

// ------------------------------------------------------------- search

#[test]
fn a_word_split_by_kerning_is_found_in_a_real_document() {
    // The headline defect. `two_columns.pdf` emits "Hy" and "phen" as
    // separate runs at the same baseline, which is what a typesetter does
    // whenever it adjusts a pair.
    let page = page_text("basic/two_columns.pdf");

    assert!(
        page.find("Hyphen").is_empty(),
        "this test is meaningless if the old per-run search already found it"
    );

    let hits = search(&page, "Hyphen", SearchOptions::default());
    assert_eq!(hits.len(), 1, "the split word was not found");
    assert_eq!(hits[0].text, "Hyphen");
    assert!(hits[0].is_split(), "the hit must span the two runs");
    assert_eq!(
        hits[0].rects.len(),
        2,
        "one highlight rectangle per run, not one merged box"
    );
}

#[test]
fn a_split_words_highlight_covers_both_halves_and_nothing_else() {
    let page = page_text("basic/two_columns.pdf");
    let hits = search(&page, "Hyphen", SearchOptions::default());
    let rects = &hits[0].rects;

    // The runs are placed at x=50 and x=61.13 in the fixture, so the two
    // rectangles must start there and be adjacent.
    assert!(
        (rects[0][0] - 50.0).abs() < 0.5,
        "the first half should start at the run origin, got {rects:?}"
    );
    assert!(
        (rects[1][0] - 61.13).abs() < 0.5,
        "the second half should start at the second run, got {rects:?}"
    );
    // The second run starts slightly *before* the first one's advance
    // ends: that is the kerning adjustment that split the word in the
    // first place. What matters is that the halves are adjacent and in
    // order, not that they are disjoint to the point.
    assert!(
        rects[1][0] > rects[0][0] && rects[1][0] < rects[0][2] + 1.0,
        "the halves must be adjacent and in order: {rects:?}"
    );
    // Both on the same baseline, so the same vertical extent.
    assert!((rects[0][1] - rects[1][1]).abs() < 1e-6, "{rects:?}");
}

#[test]
fn search_finds_text_that_plain_extraction_shows() {
    // The invariant behind the whole module: anything the page displays is
    // findable. A search that misses a word the reader can see is the
    // defect, whatever the internal reason.
    let page = page_text("basic/two_columns.pdf");
    let text = layout_text(&page);

    for word in ["Annual", "Report", "Left", "Right", "Hyphen"] {
        assert!(
            text.contains(word),
            "extraction lost {word:?}; got {text:?}"
        );
        assert!(
            !search(&page, word, SearchOptions::default()).is_empty(),
            "{word:?} is in the extracted text but not findable"
        );
    }
}

#[test]
fn a_search_across_a_column_gutter_finds_nothing() {
    // "Left one Right one" is not text that appears on the page; it is an
    // artefact of reading across the gutter.
    let page = page_text("basic/two_columns.pdf");
    assert!(
        search(&page, "one Right", SearchOptions::default()).is_empty(),
        "text spanning the gutter must not match"
    );
}

#[test]
fn case_and_accent_options_survive_the_whole_pipeline() {
    let page = page_text("basic/two_columns.pdf");
    assert_eq!(
        search(&page, "annual report", SearchOptions::default()).len(),
        1,
        "the default search is case-insensitive"
    );
    let exact = SearchOptions {
        case_insensitive: false,
        ..SearchOptions::default()
    };
    assert!(
        search(&page, "annual report", exact).is_empty(),
        "case sensitivity must be honourable end to end"
    );
}

#[test]
fn every_hit_rectangle_lies_on_the_page() {
    // A highlight off the edge of the page is the visible symptom of an
    // offset computed against the wrong origin.
    let data = corpus("basic/two_columns.pdf");
    let mut doc = PdfDocument::parse(&data).expect("parses");
    let media = doc.page(0).expect("page 0").media_box;
    let page = page_text("basic/two_columns.pdf");

    for word in ["Annual", "Left", "Right", "Hyphen"] {
        for hit in search(&page, word, SearchOptions::default()) {
            for rect in &hit.rects {
                assert!(
                    rect[0] >= media[0] - 1.0 && rect[2] <= media[2] + 1.0,
                    "{word:?} highlight {rect:?} is outside the page {media:?}"
                );
                assert!(
                    rect[2] > rect[0],
                    "{word:?} highlight {rect:?} has no width"
                );
            }
        }
    }
}

#[test]
fn the_extracted_text_and_the_searched_text_agree() {
    // The invariant that ties the two halves of this module together.
    // `layout_text` and the search index each decide where a word ends,
    // and if they decide differently then a user searching for what they
    // can see gets nothing — which is the original defect wearing a
    // different hat.
    //
    // Every word of the extracted text must be findable, and no run of
    // words spanning a break may be.
    let page = page_text("basic/two_columns.pdf");
    let text = layout_text(&page);

    for line in text.lines().filter(|l| !l.trim().is_empty()) {
        assert!(
            !search(&page, line.trim(), SearchOptions::default()).is_empty(),
            "the extracted line {line:?} is not findable"
        );
    }

    // And the joins the extraction made are real: "Hyphen" is one word in
    // the output because the two runs abut, not because they were glued.
    assert!(
        text.contains("Hyphen"),
        "the kerning-split word must extract as one word, got {text:?}"
    );
    assert!(
        !text.contains("Hy phen"),
        "an abutting split must not become a space, got {text:?}"
    );
}

#[test]
fn a_real_page_survives_the_separator_rules() {
    // The three separator cases, driven from a document rather than from
    // hand-built runs: abutting runs join, spaced runs get a space, and a
    // new line or column gets a newline.
    let page = page_text("basic/two_columns.pdf");
    let index = nigig_pdf_graphics::search::SearchIndex::for_page(&page, SearchOptions::default());
    let flat = index.text();

    assert!(flat.contains("Hyphen"), "abutting runs must join: {flat:?}");
    assert!(
        flat.contains("Annual Report"),
        "a word space must survive: {flat:?}"
    );
    // Two runs on one line separated by a real word space. Unlike
    // "Annual Report", which is one run, this can only read correctly if
    // the *separator* rule inserted the space.
    assert!(
        flat.contains("alpha beta"),
        "a space between two runs on a line must survive: {flat:?}"
    );
    assert!(
        !flat.contains("alphabeta"),
        "separated runs must not be glued: {flat:?}"
    );
    assert!(
        flat.contains('\n'),
        "line and column breaks must be newlines: {flat:?}"
    );
    assert!(
        !flat.contains("one Right"),
        "no separator may let a query span a column: {flat:?}"
    );
}

#[test]
fn a_whole_word_search_on_a_real_page_rejects_substrings() {
    let page = page_text("basic/two_columns.pdf");
    // "one" appears in "Left one" and "Right one" as a whole word.
    let words = SearchOptions {
        whole_words: true,
        ..SearchOptions::default()
    };
    assert_eq!(search(&page, "one", words).len(), 2);
    // "ne" is a substring of both and a word in neither.
    assert!(search(&page, "ne", words).is_empty());
    assert_eq!(
        search(&page, "ne", SearchOptions::default()).len(),
        2,
        "without the option it is still a substring match"
    );

    // The page also carries "gamma" and "gam": as a substring "gam"
    // matches both, as a whole word only one.
    assert_eq!(
        search(&page, "gam", SearchOptions::default()).len(),
        2,
        "substring search must match inside gamma"
    );
    assert_eq!(
        search(&page, "gam", words).len(),
        1,
        "whole-word search must reject the gamma prefix"
    );
}

#[test]
fn an_overlapping_match_on_a_real_page_is_counted_once() {
    // The page carries "aaa". Advancing by one character after a hit
    // instead of past it reports two overlapping matches, and the two
    // highlights then paint over each other.
    let page = page_text("basic/two_columns.pdf");
    assert_eq!(
        search(&page, "aa", SearchOptions::default()).len(),
        1,
        "an overlapping match must be reported once"
    );
}

#[test]
fn a_column_gutter_on_a_real_page_is_wider_than_a_word_space() {
    // The threshold the whole layout analysis turns on. The fixture has
    // both: a 4-unit word space between "alpha" and "beta", and a 260-unit
    // gutter between the columns. A threshold that cannot tell them apart
    // either glues the columns or splits every line.
    let page = page_text("basic/two_columns.pdf");
    let columns = detect_columns_from_segments(page.segments());
    assert_eq!(
        columns.len(),
        2,
        "the page has exactly two columns, got {}",
        columns.len()
    );
    // "alpha" and "beta" must be in the same column despite their gap.
    let text = layout_text(&page);
    assert!(
        text.contains("alpha beta"),
        "a word space must not read as a column break: {text:?}"
    );
    // "wide" and "space" are further apart still — their boxes do not
    // overlap at all — and are on one line.
    //
    // Mutating the gutter threshold to zero does **not** break this page,
    // and that is worth stating rather than hiding: a longer line
    // elsewhere in the column ("aaa gamma gam") spans both runs and
    // merges the bands anyway, so band merging makes the threshold
    // non-load-bearing on a realistic page. The unit test
    // `a_line_split_into_runs_stays_one_line` isolates it, on a page with
    // nothing else to bridge the gap. Both tests are kept: one shows the
    // rule, the other shows the algorithm tolerates the rule being wrong.
    assert!(
        text.contains("wide space"),
        "a wide word space must not read as a column break: {text:?}"
    );
}

#[test]
fn repeated_text_on_a_real_page_is_found_once_per_occurrence() {
    let page = page_text("basic/two_columns.pdf");
    // "Right" begins both right-column lines.
    let hits = search(&page, "Right", SearchOptions::default());
    assert_eq!(hits.len(), 2, "both occurrences must be reported");
    // On different lines, so at different heights.
    assert!(
        (hits[0].rects[0][1] - hits[1].rects[0][1]).abs() > 1.0,
        "the two hits should be on different lines: {hits:?}"
    );
}

// ------------------------------------------------------------- layout

#[test]
fn a_two_column_page_is_read_down_each_column() {
    // The other headline defect. The fixture emits left, right, left,
    // right — as a typesetter does — and the columns share baselines, so
    // grouping into lines first produces "Left one Right one".
    let page = page_text("basic/two_columns.pdf");

    let naive = page.plain_text();
    assert!(
        naive.contains("Left one") && naive.contains("Right one"),
        "sanity: the runs are all present, got {naive:?}"
    );

    let text = layout_text(&page);
    let left = text.find("Left one").expect("left column");
    let hyphen = text.find("Hyphen").expect("second left line");
    let right = text.find("Right one").expect("right column");

    assert!(
        left < hyphen,
        "the left column must be read top to bottom, got {text:?}"
    );
    assert!(
        hyphen < right,
        "the whole left column must precede the right one, got {text:?}"
    );
}

#[test]
fn the_columns_of_a_real_page_are_detected_from_its_geometry() {
    let page = page_text("basic/two_columns.pdf");
    let columns = detect_columns_from_segments(page.segments());

    // The heading spans both columns, so it bridges them into one band —
    // which is right: a page with a full-width heading is not two
    // independent columns, and reading it top to bottom is what a reader
    // expects. What must not happen is losing text.
    let counted: usize = columns.iter().map(|c| c.segments.len()).sum();
    assert_eq!(
        counted,
        page.segments().len(),
        "every run must belong to exactly one column"
    );

    let mut seen: Vec<usize> = columns.iter().flat_map(|c| c.segments.clone()).collect();
    seen.sort_unstable();
    seen.dedup();
    assert_eq!(
        seen.len(),
        page.segments().len(),
        "a run was double-counted"
    );
}

#[test]
fn reading_order_is_a_permutation_of_the_runs() {
    // Whatever the layout analysis decides, it may not drop or duplicate a
    // run. Losing one silently is how text vanishes from a copy-paste.
    for fixture in [
        "basic/text.pdf",
        "basic/two_columns.pdf",
        "basic/multipage.pdf",
    ] {
        let page = page_text(fixture);
        let order = reading_order(page.segments());
        let mut sorted = order.clone();
        sorted.sort_unstable();
        let expected: Vec<usize> = (0..page.segments().len()).collect();
        assert_eq!(
            sorted, expected,
            "{fixture}: reading order is not a permutation of the runs"
        );
    }
}

#[test]
fn a_single_column_page_reads_in_document_order() {
    // `basic/text.pdf` is two lines, one under the other. The layout
    // analysis must not disturb the simple case.
    let page = page_text("basic/text.pdf");
    let text = layout_text(&page);
    assert_eq!(
        text, "Hello World\nSecond line",
        "a simple page must survive layout analysis unchanged"
    );
}

#[test]
fn lines_of_a_real_page_group_by_baseline() {
    let page = page_text("basic/text.pdf");
    let lines = detect_lines(page.segments());
    assert_eq!(lines.len(), 2, "two Td-separated lines");
    assert!(
        lines[0].baseline > lines[1].baseline,
        "lines are ordered top to bottom in PDF coordinates"
    );
}

// ---------------------------------------------------------- selection

#[test]
fn selecting_across_two_lines_takes_the_text_between_them() {
    let page = page_text("basic/text.pdf");
    let segments = page.segments();
    assert!(segments.len() >= 2, "the fixture has two runs");

    let first = &segments[0];
    let second = &segments[1];
    // From the middle of the first line to the middle of the second.
    let from = (first.origin[0] + first.advance / 2.0, first.origin[1] + 1.0);
    let to = (
        second.origin[0] + second.advance / 2.0,
        second.origin[1] + 1.0,
    );

    let selected = page.text_in_range(from, to);
    assert!(
        !selected.is_empty(),
        "a drag across two lines must select something"
    );
    assert!(
        selected.len() < page.plain_text().len(),
        "a partial drag must not select the whole page: {selected:?}"
    );
    // It must be a contiguous slice of the page's text, not a rearranged
    // one.
    let whole = page.plain_text().replace('\n', "");
    let flattened = selected.replace('\n', "");
    assert!(
        whole.contains(&flattened),
        "the selection {flattened:?} is not a substring of the page {whole:?}"
    );
}

#[test]
fn a_click_lands_on_the_run_under_it() {
    let page = page_text("basic/two_columns.pdf");
    let segments = page.segments();

    for (index, segment) in segments.iter().enumerate() {
        // A point inside this run's own box must resolve to this run.
        let point = (
            segment.origin[0] + segment.advance / 2.0,
            segment.origin[1] + 1.0,
        );
        let (found, _) = page
            .segment_at(point.0, point.1)
            .unwrap_or_else(|| panic!("run {index} ({:?}) is not hit-testable", segment.text));
        assert_eq!(
            found, index,
            "a click in the middle of {:?} resolved to {:?}",
            segment.text, segments[found].text
        );
    }
}

#[test]
fn a_click_in_the_gutter_hits_nothing() {
    // Between the columns there is no text, and reporting a hit there
    // would start a selection the user did not ask for.
    let page = page_text("basic/two_columns.pdf");
    // The fixture's columns are at x=50 and x=350; x=250 is empty.
    assert!(
        page.segment_at(250.0, 651.0).is_none(),
        "the gutter must not hit-test to a run"
    );
}