88048aa784
Add sandbox authoring effects and character/composite creation, improve creator submission handling, and extend publishing/search metadata. Update importer conversions and the asset UI integration. Validation: 31 store search tests, ten composite tests and two author-policy tests passed in release mode.
429 lines
18 KiB
Rust
429 lines
18 KiB
Rust
//! The alias-aware search index: alias terms and canonical-alias ordering,
|
|
//! transactional reindex on alias retarget, generation-carrying keyset
|
|
//! cursors that fail closed on stale and tampered bytes, and whole-reindex
|
|
//! rollback when a statement fails mid-transaction.
|
|
|
|
mod common;
|
|
use common::*;
|
|
use makepad_asset_store::{
|
|
AssetAnnotation, AssetServerCore, Budgets, SearchFilters, SearchQuery, SearchViewer,
|
|
ServerError, ViewerScope, Visibility,
|
|
};
|
|
use makepad_asset_data::{AssetId, AssetRevisionRef};
|
|
|
|
const ANYONE: SearchViewer<'static> = SearchViewer { principal: None, scope: ViewerScope::All };
|
|
|
|
fn ann(title: &str) -> AssetAnnotation {
|
|
AssetAnnotation {
|
|
title: title.into(),
|
|
description: String::new(),
|
|
kind: None,
|
|
categories: Vec::new(),
|
|
tags: Vec::new(),
|
|
creator: String::new(),
|
|
artist: String::new(),
|
|
artist_url: String::new(),
|
|
album: String::new(),
|
|
source_url: String::new(),
|
|
license: String::new(),
|
|
license_url: String::new(),
|
|
owner: None,
|
|
generator: String::new(),
|
|
backend: String::new(),
|
|
model: String::new(),
|
|
prompt: String::new(),
|
|
provenance: String::new(),
|
|
visibility: Visibility::Public,
|
|
}
|
|
}
|
|
|
|
fn q(text: &str) -> SearchQuery<'_> {
|
|
SearchQuery {
|
|
text,
|
|
filters: SearchFilters::default(),
|
|
expand: false,
|
|
page_size: 10,
|
|
newest: false,
|
|
facets: 0,
|
|
}
|
|
}
|
|
|
|
fn stale(err: ServerError) {
|
|
assert!(
|
|
matches!(err, ServerError::InvalidInput { what: "stale search cursor" }),
|
|
"expected stale cursor, got {err}"
|
|
);
|
|
}
|
|
|
|
fn malformed(err: ServerError) {
|
|
assert!(
|
|
matches!(err, ServerError::InvalidInput { what: "search cursor malformed" }),
|
|
"expected malformed cursor, got {err}"
|
|
);
|
|
}
|
|
|
|
fn tampered(err: ServerError) {
|
|
assert!(
|
|
matches!(err, ServerError::InvalidInput { what: "search cursor tampered" }),
|
|
"expected tampered cursor, got {err}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn alias_terms_index_and_retarget_reindexes_transactionally() {
|
|
let (_root, core) = open_core("alias_terms");
|
|
let search = core.search();
|
|
let (id_a, _rev_a) = publish_prop(&core, "rik2", 1, b"glb a", b"thumb a", NOW);
|
|
let (id_b, rev_b) = publish_prop(&core, "rik2", 2, b"glb b", b"thumb b", NOW);
|
|
search.set_annotation(&id_a, &ann("lantern alpha"), NOW).unwrap();
|
|
search.set_annotation(&id_b, &ann("lantern beta"), NOW).unwrap();
|
|
|
|
// No alias yet: alias-shaped terms match nothing, hits carry no alias.
|
|
assert_eq!(search.search(&q("lamp"), &ANYONE, None).unwrap().total, 0);
|
|
let page = search.search(&q("lantern"), &ANYONE, None).unwrap();
|
|
assert!(page.hits.iter().all(|h| h.alias.is_none()));
|
|
|
|
// An alias head indexes every tokenized segment, normalized exactly like
|
|
// annotation text (lowercase runs, `_` and `-` split, digits kept).
|
|
let alias = "rik2/props/old-lamp_3".parse().unwrap();
|
|
let rev_of = |id: AssetId, rev| AssetRevisionRef { asset_id: id, revision: rev };
|
|
core.catalog().set_asset_alias(&alias, &rev_of(id_a, _rev_a), NOW + 1).unwrap();
|
|
for term in ["lamp", "old", "props", "3", "LAMP", "rik2"] {
|
|
let page = search.search(&q(term), &ANYONE, None).unwrap();
|
|
assert_eq!((page.total, page.hits[0].asset_id), (1, id_a), "term {term}");
|
|
}
|
|
let page = search.search(&q("lamp"), &ANYONE, None).unwrap();
|
|
assert_eq!(page.hits[0].alias.as_deref(), Some("rik2/props/old-lamp_3"));
|
|
assert!(page.hits[0].live);
|
|
// Multi-term AND spans annotation and alias postings as one index.
|
|
let page = search.search(&q("lantern lamp"), &ANYONE, None).unwrap();
|
|
assert_eq!((page.total, page.hits[0].asset_id), (1, id_a));
|
|
// At equal score the unaliased asset sorts first ('' before any alias).
|
|
let page = search.search(&q("lantern"), &ANYONE, None).unwrap();
|
|
let got: Vec<(AssetId, Option<&str>)> =
|
|
page.hits.iter().map(|h| (h.asset_id, h.alias.as_deref())).collect();
|
|
assert_eq!(got, vec![(id_b, None), (id_a, Some("rik2/props/old-lamp_3"))]);
|
|
|
|
// Retargeting moves the terms, canonical alias and liveness in one
|
|
// transaction: the old target keeps nothing.
|
|
core.catalog().set_asset_alias(&alias, &rev_of(id_b, rev_b), NOW + 2).unwrap();
|
|
let page = search.search(&q("lamp"), &ANYONE, None).unwrap();
|
|
assert_eq!((page.total, page.hits[0].asset_id), (1, id_b));
|
|
let page = search.search(&q("lantern"), &ANYONE, None).unwrap();
|
|
let got: Vec<(AssetId, Option<&str>, bool)> =
|
|
page.hits.iter().map(|h| (h.asset_id, h.alias.as_deref(), h.live)).collect();
|
|
assert_eq!(
|
|
got,
|
|
vec![(id_a, None, false), (id_b, Some("rik2/props/old-lamp_3"), true)]
|
|
);
|
|
|
|
// With several heads the canonical alias is the smallest one; clearing it
|
|
// falls back to the next.
|
|
let alias2 = "rik2/props/beta-lamp".parse().unwrap();
|
|
core.catalog().set_asset_alias(&alias2, &rev_of(id_b, rev_b), NOW + 3).unwrap();
|
|
let page = search.search(&q("lamp"), &ANYONE, None).unwrap();
|
|
assert_eq!(page.hits[0].alias.as_deref(), Some("rik2/props/beta-lamp"));
|
|
// "beta" now reaches id_b through BOTH its title (100) and an alias
|
|
// segment (80): one hit, summed across the two posting tables.
|
|
let page = search.search(&q("beta"), &ANYONE, None).unwrap();
|
|
assert_eq!((page.total, page.hits[0].asset_id, page.hits[0].score), (1, id_b, 180));
|
|
assert!(core.catalog().clear_asset_alias(&alias2).unwrap());
|
|
let page = search.search(&q("lamp"), &ANYONE, None).unwrap();
|
|
assert_eq!(page.hits[0].alias.as_deref(), Some("rik2/props/old-lamp_3"));
|
|
|
|
// Clearing the last head removes the terms; annotations are untouched.
|
|
assert!(core.catalog().clear_asset_alias(&alias).unwrap());
|
|
assert_eq!(search.search(&q("lamp"), &ANYONE, None).unwrap().total, 0);
|
|
assert_eq!(search.search(&q("lantern"), &ANYONE, None).unwrap().total, 2);
|
|
|
|
// Quarantining the aliased revision tears the alias terms down too.
|
|
core.catalog().set_asset_alias(&alias, &rev_of(id_b, rev_b), NOW + 4).unwrap();
|
|
assert_eq!(search.search(&q("lamp"), &ANYONE, None).unwrap().total, 1);
|
|
core.catalog().quarantine_asset(&id_b, &rev_b, NOW + 5).unwrap();
|
|
assert_eq!(search.search(&q("lamp"), &ANYONE, None).unwrap().total, 0);
|
|
assert_eq!(core.catalog().resolve_asset_alias(&alias).unwrap(), None);
|
|
}
|
|
|
|
#[test]
|
|
fn canonical_alias_orders_equal_scores_and_pages_without_dups() {
|
|
let (_root, core) = open_core("canon_order");
|
|
let search = core.search();
|
|
// Four equal-score assets: two unaliased, two aliased. The alias segments
|
|
// deliberately avoid the query term so scores stay identical.
|
|
let id1 = asset_id_n(1);
|
|
core.catalog().register_asset(&id1, "rik2", NOW).unwrap();
|
|
let (id2, rev2) = publish_prop(&core, "rik2", 2, b"g2", b"t2", NOW);
|
|
let (id3, rev3) = publish_prop(&core, "rik2", 3, b"g3", b"t3", NOW);
|
|
let id4 = asset_id_n(4);
|
|
core.catalog().register_asset(&id4, "rik2", NOW).unwrap();
|
|
for id in [id1, id2, id3, id4] {
|
|
search.set_annotation(&id, &ann("beacon post"), NOW).unwrap();
|
|
}
|
|
core.catalog()
|
|
.set_asset_alias(
|
|
&"rik2/items/bb".parse().unwrap(),
|
|
&AssetRevisionRef { asset_id: id2, revision: rev2 },
|
|
NOW + 1,
|
|
)
|
|
.unwrap();
|
|
core.catalog()
|
|
.set_asset_alias(
|
|
&"rik2/items/aa".parse().unwrap(),
|
|
&AssetRevisionRef { asset_id: id3, revision: rev3 },
|
|
NOW + 2,
|
|
)
|
|
.unwrap();
|
|
|
|
// Total order: score DESC (all equal), canonical alias ASC with ''
|
|
// (unaliased) first, asset id ASC as the last key.
|
|
let page = search.search(&q("beacon"), &ANYONE, None).unwrap();
|
|
let full: Vec<(AssetId, Option<&str>)> =
|
|
page.hits.iter().map(|h| (h.asset_id, h.alias.as_deref())).collect();
|
|
assert_eq!(
|
|
full,
|
|
vec![
|
|
(id1, None),
|
|
(id4, None),
|
|
(id3, Some("rik2/items/aa")),
|
|
(id2, Some("rik2/items/bb")),
|
|
]
|
|
);
|
|
assert!(page.hits.windows(2).all(|w| w[0].score == w[1].score), "scores tied");
|
|
|
|
// A page-size-1 keyset walk over the tied set: no dup, no miss, same
|
|
// order as the unpaged query.
|
|
let mut one = q("beacon");
|
|
one.page_size = 1;
|
|
let mut seen = Vec::new();
|
|
let mut cursor: Option<Vec<u8>> = None;
|
|
loop {
|
|
let page = search.search(&one, &ANYONE, cursor.as_deref()).unwrap();
|
|
assert_eq!(page.total, 4, "total constant across the walk");
|
|
seen.extend(page.hits.iter().map(|h| h.asset_id));
|
|
match page.cursor {
|
|
Some(c) => cursor = Some(c),
|
|
None => break,
|
|
}
|
|
assert!(seen.len() <= 4, "walk must terminate");
|
|
}
|
|
assert_eq!(seen, vec![id1, id4, id3, id2]);
|
|
|
|
// Browse mode follows the same total order (every score is zero).
|
|
let mut browse = q("");
|
|
browse.filters = SearchFilters { namespace: Some("rik2"), ..Default::default() };
|
|
let page = search.search(&browse, &ANYONE, None).unwrap();
|
|
assert_eq!(
|
|
page.hits.iter().map(|h| h.asset_id).collect::<Vec<_>>(),
|
|
vec![id1, id4, id3, id2]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn index_mutations_invalidate_cursors_and_noops_do_not() {
|
|
let (_root, core) = open_core("generation");
|
|
let search = core.search();
|
|
let (id3, rev3) = publish_prop(&core, "rik2", 3, b"g3", b"t3", NOW);
|
|
let id1 = asset_id_n(1);
|
|
let id2 = asset_id_n(2);
|
|
core.catalog().register_asset(&id1, "rik2", NOW).unwrap();
|
|
core.catalog().register_asset(&id2, "rik2", NOW).unwrap();
|
|
let id4 = asset_id_n(4);
|
|
core.catalog().register_asset(&id4, "rik2", NOW).unwrap();
|
|
search.set_annotation(&id1, &ann("gizmo one"), NOW).unwrap();
|
|
search.set_annotation(&id2, &ann("gizmo two"), NOW).unwrap();
|
|
search.set_annotation(&id3, &ann("gizmo three"), NOW).unwrap();
|
|
|
|
let mut one = q("gizmo");
|
|
one.page_size = 1;
|
|
let mint = || search.search(&one, &ANYONE, None).unwrap().cursor.expect("more hits");
|
|
|
|
// Annotating ANY asset (even one not in this result set) retires the
|
|
// cursor: the total order may have shifted under it.
|
|
let c = mint();
|
|
search.set_annotation(&id2, &ann("gizmo two prime"), NOW + 1).unwrap();
|
|
stale(search.search(&one, &ANYONE, Some(&c)).unwrap_err());
|
|
|
|
// Alias-head set retires cursors.
|
|
let c = mint();
|
|
let alias = "rik2/misc/tool".parse().unwrap();
|
|
core.catalog()
|
|
.set_asset_alias(&alias, &AssetRevisionRef { asset_id: id3, revision: rev3 }, NOW + 2)
|
|
.unwrap();
|
|
stale(search.search(&one, &ANYONE, Some(&c)).unwrap_err());
|
|
|
|
// Clearing an annotation retires cursors.
|
|
let c = mint();
|
|
search.clear_annotation(&id1).unwrap();
|
|
stale(search.search(&one, &ANYONE, Some(&c)).unwrap_err());
|
|
|
|
// Pure no-ops do NOT: clearing a never-annotated asset or a nonexistent
|
|
// alias leaves the generation alone and the cursor keeps working.
|
|
let c = mint();
|
|
search.clear_annotation(&id4).unwrap();
|
|
assert!(!core.catalog().clear_asset_alias(&"rik2/misc/ghost".parse().unwrap()).unwrap());
|
|
let page = search.search(&one, &ANYONE, Some(&c)).unwrap();
|
|
assert_eq!(page.hits.len(), 1);
|
|
|
|
// Quarantine that drops an alias head retires cursors.
|
|
core.catalog().quarantine_asset(&id3, &rev3, NOW + 3).unwrap();
|
|
stale(search.search(&one, &ANYONE, Some(&c)).unwrap_err());
|
|
}
|
|
|
|
#[test]
|
|
fn cursor_tampering_and_hostile_bytes_fail_closed() {
|
|
let (_root, core) = open_core("tamper");
|
|
let search = core.search();
|
|
for n in 1..=2u8 {
|
|
let id = asset_id_n(n);
|
|
core.catalog().register_asset(&id, "rik2", NOW).unwrap();
|
|
search.set_annotation(&id, &ann(&format!("brass item {n}")), NOW).unwrap();
|
|
}
|
|
let mut one = q("brass");
|
|
one.page_size = 1;
|
|
let cursor = search.search(&one, &ANYONE, None).unwrap().cursor.expect("two hits");
|
|
|
|
// Any bit flip inside the checksummed body is tampering: generation,
|
|
// fingerprint, score, asset id and the check itself are all covered.
|
|
for idx in [1usize, 20, 45, cursor.len() - 17, cursor.len() - 1] {
|
|
let mut bad = cursor.clone();
|
|
bad[idx] ^= 1;
|
|
tampered(search.search(&one, &ANYONE, Some(&bad)).unwrap_err());
|
|
}
|
|
// Structural damage is malformed: wrong version, truncation, growth,
|
|
// emptiness, oversize garbage, forged alias length.
|
|
let mut bad = cursor.clone();
|
|
bad[0] = 9;
|
|
malformed(search.search(&one, &ANYONE, Some(&bad)).unwrap_err());
|
|
malformed(search.search(&one, &ANYONE, Some(&cursor[..cursor.len() - 1])).unwrap_err());
|
|
let mut bad = cursor.clone();
|
|
bad.push(0);
|
|
malformed(search.search(&one, &ANYONE, Some(&bad)).unwrap_err());
|
|
malformed(search.search(&one, &ANYONE, Some(b"")).unwrap_err());
|
|
malformed(search.search(&one, &ANYONE, Some(&vec![0u8; 10_000])).unwrap_err());
|
|
let mut bad = cursor.clone();
|
|
bad[57..59].copy_from_slice(&200u16.to_be_bytes());
|
|
malformed(search.search(&one, &ANYONE, Some(&bad)).unwrap_err());
|
|
let mut bad = cursor.clone();
|
|
bad[57..59].copy_from_slice(&5u16.to_be_bytes());
|
|
malformed(search.search(&one, &ANYONE, Some(&bad)).unwrap_err());
|
|
|
|
// The genuine cursor still works after all hostile traffic.
|
|
let page = search.search(&one, &ANYONE, Some(&cursor)).unwrap();
|
|
assert_eq!(page.hits.len(), 1);
|
|
|
|
// Same laws when the keyset position carries a real alias.
|
|
let (id6, rev6) = publish_prop(&core, "rik2", 6, b"g6", b"t6", NOW);
|
|
let (id7, rev7) = publish_prop(&core, "rik2", 7, b"g7", b"t7", NOW);
|
|
let id5 = asset_id_n(5);
|
|
core.catalog().register_asset(&id5, "rik2", NOW).unwrap();
|
|
for id in [id5, id6, id7] {
|
|
search.set_annotation(&id, &ann("copper beacon"), NOW).unwrap();
|
|
}
|
|
core.catalog()
|
|
.set_asset_alias(
|
|
&"rik2/set/aa".parse().unwrap(),
|
|
&AssetRevisionRef { asset_id: id6, revision: rev6 },
|
|
NOW + 1,
|
|
)
|
|
.unwrap();
|
|
core.catalog()
|
|
.set_asset_alias(
|
|
&"rik2/set/bb".parse().unwrap(),
|
|
&AssetRevisionRef { asset_id: id7, revision: rev7 },
|
|
NOW + 2,
|
|
)
|
|
.unwrap();
|
|
let mut two = q("copper");
|
|
two.page_size = 2;
|
|
let page = search.search(&two, &ANYONE, None).unwrap();
|
|
assert_eq!(
|
|
page.hits.iter().map(|h| h.alias.as_deref()).collect::<Vec<_>>(),
|
|
vec![None, Some("rik2/set/aa")]
|
|
);
|
|
let cursor = page.cursor.expect("third hit remains");
|
|
// Flip a byte inside the alias region.
|
|
let mut bad = cursor.clone();
|
|
bad[60] ^= 1;
|
|
tampered(search.search(&two, &ANYONE, Some(&bad)).unwrap_err());
|
|
// Forge the alias length on an alias-carrying cursor.
|
|
let mut bad = cursor.clone();
|
|
bad[57..59].copy_from_slice(&0u16.to_be_bytes());
|
|
malformed(search.search(&two, &ANYONE, Some(&bad)).unwrap_err());
|
|
let page = search.search(&two, &ANYONE, Some(&cursor)).unwrap();
|
|
assert_eq!(
|
|
page.hits.iter().map(|h| h.asset_id).collect::<Vec<_>>(),
|
|
vec![id7]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn mid_transaction_failure_rolls_back_the_whole_reindex() {
|
|
let root = test_root("rollback");
|
|
let mut budgets = Budgets::default_v1();
|
|
budgets.max_search_index_terms = 4;
|
|
let core = AssetServerCore::open(&root, budgets).unwrap();
|
|
let search = core.search();
|
|
let (id_a, rev_a) = publish_prop(&core, "rik2", 1, b"glb a", b"thumb a", NOW);
|
|
let mut aa = ann("alpha beta");
|
|
aa.tags = vec!["durable".into()];
|
|
search.set_annotation(&id_a, &aa, NOW).unwrap();
|
|
let alias = "rik2/props/lamp".parse().unwrap();
|
|
core.catalog()
|
|
.set_asset_alias(&alias, &AssetRevisionRef { asset_id: id_a, revision: rev_a }, NOW + 1)
|
|
.unwrap();
|
|
|
|
// Annotation path: term-budget admission fires after the annotation
|
|
// upsert and the delete-old-index statements in the same transaction.
|
|
// Failing it must roll all of them back.
|
|
let g0 = search.generation().unwrap();
|
|
let err = search
|
|
.set_annotation(&id_a, &ann("gamma delta epsilon zeta theta"), NOW + 2)
|
|
.unwrap_err();
|
|
assert!(matches!(err, ServerError::OverBudget { .. }), "{err}");
|
|
assert_eq!(search.generation().unwrap(), g0, "failed tx must not advance the generation");
|
|
assert_eq!(search.annotation(&id_a).unwrap().unwrap().title, "alpha beta");
|
|
assert_eq!(search.search(&q("alpha"), &ANYONE, None).unwrap().total, 1);
|
|
assert_eq!(search.search(&q("gamma"), &ANYONE, None).unwrap().total, 0);
|
|
let mut tagged = q("");
|
|
tagged.filters = SearchFilters { tag: Some("durable"), ..Default::default() };
|
|
assert_eq!(search.search(&tagged, &ANYONE, None).unwrap().total, 1);
|
|
let page = search.search(&q("lamp"), &ANYONE, None).unwrap();
|
|
assert_eq!((page.total, page.hits[0].live), (1, true), "alias index intact");
|
|
|
|
// A within-budget write lands, advancing the generation exactly once;
|
|
// alias postings survive an annotation rebuild.
|
|
search.set_annotation(&id_a, &ann("gamma delta epsilon zeta"), NOW + 3).unwrap();
|
|
assert_eq!(search.generation().unwrap(), g0 + 1);
|
|
assert_eq!(search.search(&q("gamma"), &ANYONE, None).unwrap().total, 1);
|
|
assert_eq!(search.search(&q("alpha"), &ANYONE, None).unwrap().total, 0);
|
|
assert_eq!(search.search(&q("lamp"), &ANYONE, None).unwrap().total, 1);
|
|
|
|
// Alias path: failing the alias-posting rebuild rolls back the CATALOG
|
|
// alias write too — the head, the live flag, the canonical alias, the
|
|
// terms and the generation all stay exactly as before.
|
|
let (id_b, rev_b) = publish_prop(&core, "rik2", 2, b"glb b", b"thumb b", NOW);
|
|
search.set_annotation(&id_b, &ann("omega"), NOW + 4).unwrap();
|
|
let g1 = search.generation().unwrap();
|
|
let alias_b = "rik2/props/large/brass/blamp".parse().unwrap();
|
|
let err = core
|
|
.catalog()
|
|
.set_asset_alias(&alias_b, &AssetRevisionRef { asset_id: id_b, revision: rev_b }, NOW + 5)
|
|
.unwrap_err();
|
|
assert!(matches!(err, ServerError::OverBudget { .. }), "{err}");
|
|
assert_eq!(core.catalog().resolve_asset_alias(&alias_b).unwrap(), None);
|
|
assert_eq!(search.generation().unwrap(), g1);
|
|
assert_eq!(search.search(&q("blamp"), &ANYONE, None).unwrap().total, 0);
|
|
let page = search.search(&q("omega"), &ANYONE, None).unwrap();
|
|
assert_eq!((page.hits[0].live, page.hits[0].alias.as_deref()), (false, None));
|
|
|
|
let alias_b = "rik2/props/blamp".parse().unwrap();
|
|
core.catalog()
|
|
.set_asset_alias(&alias_b, &AssetRevisionRef { asset_id: id_b, revision: rev_b }, NOW + 6)
|
|
.unwrap();
|
|
assert_eq!(search.generation().unwrap(), g1 + 1);
|
|
let page = search.search(&q("blamp"), &ANYONE, None).unwrap();
|
|
assert_eq!(
|
|
(page.total, page.hits[0].live, page.hits[0].alias.as_deref()),
|
|
(1, true, Some("rik2/props/blamp"))
|
|
);
|
|
}
|