d8d5ea8b6a
Restore missing required asset/flow packages from the pinned nigig revision; retain upstream script isolation, X11 hover, and widget-core fixes. Validated makepad-widgets data feature with Rust 1.97.1.
393 lines
14 KiB
Rust
393 lines
14 KiB
Rust
//! The fully-local content path, end to end, with no UI.
|
|
//!
|
|
//! This is the VJ's standalone mode reduced to its load-bearing parts, so it
|
|
//! can be RUN and BELIEVED on a machine with no window: host the Asset Server
|
|
//! in-process on loopback, walk a directory of media, catalogue every file
|
|
//! WITHOUT COPYING it, and then read the bytes back out of the store and
|
|
//! check them against the originals.
|
|
//!
|
|
//! It exists because "it compiles on Windows" is not evidence. Media
|
|
//! Foundation decoding a first frame, a loopback bind, SQLite under a WAL,
|
|
//! reference blobs hashing files in place — each of those is a place where a
|
|
//! platform can differ, and this walks all of them and prints what happened.
|
|
//!
|
|
//! ```text
|
|
//! cargo run --release --example local_media_import -- <media-dir> [store-root]
|
|
//! ```
|
|
//!
|
|
//! Exit code is 0 only if every published asset read back byte-identical to
|
|
//! the file on disk that it references.
|
|
|
|
use makepad_asset_client::{
|
|
ApiEndpoints, AssetClient, ClientConfig, ClientError, PublishBundle, PublishBundleFile,
|
|
PublishRights, PublishThumbnail,
|
|
};
|
|
use makepad_asset_data::{
|
|
AssetAlias, AssetKind, DerivativePolicy, FileRole, MediaType, Redistribution, Sha256,
|
|
ThumbnailMedia,
|
|
};
|
|
use makepad_asset_importer::thumbs;
|
|
use makepad_asset_importer::videothumb::probe_video;
|
|
use makepad_asset_store::{AssetServer, BlobRefPolicy, ServerConfig};
|
|
use std::net::SocketAddr;
|
|
use std::path::{Path, PathBuf};
|
|
use std::time::Instant;
|
|
|
|
const NS: &str = "vjmedia";
|
|
|
|
fn classify(ext: &str) -> Option<(AssetKind, FileRole, MediaType)> {
|
|
Some(match ext {
|
|
"mp4" | "mov" | "m4v" => (AssetKind::Video, FileRole::Video, MediaType::Mp4),
|
|
"png" => (AssetKind::Texture, FileRole::Texture, MediaType::Png),
|
|
"jpg" | "jpeg" => (AssetKind::Texture, FileRole::Texture, MediaType::Jpeg),
|
|
"wav" | "wave" => (AssetKind::Audio, FileRole::Audio, MediaType::Wav),
|
|
"mp3" => (AssetKind::Audio, FileRole::Audio, MediaType::Mp3),
|
|
"ogg" | "oga" => (AssetKind::Audio, FileRole::Audio, MediaType::Ogg),
|
|
_ => return None,
|
|
})
|
|
}
|
|
|
|
fn walk(dir: &Path, depth: usize, out: &mut Vec<PathBuf>) {
|
|
if depth > 8 || out.len() > 5_000 {
|
|
return;
|
|
}
|
|
let Ok(entries) = std::fs::read_dir(dir) else { return };
|
|
let mut paths: Vec<PathBuf> = entries.flatten().map(|e| e.path()).collect();
|
|
paths.sort();
|
|
for path in paths {
|
|
let Some(name) = path.file_name().and_then(|n| n.to_str()) else { continue };
|
|
if name.starts_with('.') {
|
|
continue;
|
|
}
|
|
if path.is_dir() {
|
|
walk(&path, depth + 1, out);
|
|
} else {
|
|
out.push(path);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn slug(text: &str, max: usize) -> String {
|
|
let mut out = String::new();
|
|
let mut dash = true;
|
|
for ch in text.chars() {
|
|
let c = ch.to_ascii_lowercase();
|
|
if c.is_ascii_alphanumeric() {
|
|
out.push(c);
|
|
dash = false;
|
|
} else if !dash && out.len() < max {
|
|
out.push('-');
|
|
dash = true;
|
|
}
|
|
if out.len() >= max {
|
|
break;
|
|
}
|
|
}
|
|
let t = out.trim_matches('-').to_string();
|
|
if t.is_empty() { "clip".to_string() } else { t }
|
|
}
|
|
|
|
fn hex8(bytes: &[u8]) -> String {
|
|
let mut h = Sha256::new();
|
|
h.update(bytes);
|
|
h.finalize()[..4].iter().map(|b| format!("{b:02x}")).collect()
|
|
}
|
|
|
|
fn rights() -> PublishRights {
|
|
PublishRights {
|
|
license: "LicenseRef-Personal-Library".to_string(),
|
|
license_revision: String::new(),
|
|
terms_digest: None,
|
|
terms_url: String::new(),
|
|
credits: String::new(),
|
|
source: "local import".to_string(),
|
|
source_archive: None,
|
|
redistribution: Redistribution::Forbidden,
|
|
derivatives: DerivativePolicy::Allowed,
|
|
}
|
|
}
|
|
|
|
fn main() {
|
|
let args: Vec<String> = std::env::args().skip(1).collect();
|
|
if args.is_empty() {
|
|
eprintln!("usage: local_media_import <media-dir> [store-root]");
|
|
std::process::exit(2);
|
|
}
|
|
let media_dir = PathBuf::from(&args[0]);
|
|
let root = args
|
|
.get(1)
|
|
.map(PathBuf::from)
|
|
.unwrap_or_else(|| std::env::temp_dir().join("vj-local-store-proof"));
|
|
|
|
println!("== FULLY-LOCAL CONTENT PATH ==");
|
|
println!("media dir : {}", media_dir.display());
|
|
println!("store root: {}", root.display());
|
|
|
|
// ---- 1. host the store, loopback only -------------------------------
|
|
let t0 = Instant::now();
|
|
std::fs::create_dir_all(&root).expect("create store root");
|
|
let mut cfg = ServerConfig::new(root.clone());
|
|
cfg.control_addr = SocketAddr::from(([127, 0, 0, 1], 0));
|
|
cfg.data_addr = SocketAddr::from(([127, 0, 0, 1], 0));
|
|
cfg.bootstrap_admin = true;
|
|
cfg.discovery = None;
|
|
cfg.blob_refs = BlobRefPolicy::local_host();
|
|
cfg.log = false;
|
|
let server = match AssetServer::start(cfg) {
|
|
Ok(s) => s,
|
|
Err(e) => {
|
|
eprintln!("FAIL: could not host the store: {e}");
|
|
std::process::exit(1);
|
|
}
|
|
};
|
|
let token = std::fs::read_to_string(root.join("admin-token"))
|
|
.expect("admin token")
|
|
.trim()
|
|
.to_string();
|
|
println!(
|
|
"[1] store up on control {} / data {} ({} ms)",
|
|
server.control_addr(),
|
|
server.data_addr(),
|
|
t0.elapsed().as_millis()
|
|
);
|
|
assert!(server.control_addr().ip().is_loopback(), "control plane must be loopback");
|
|
assert!(server.data_addr().ip().is_loopback(), "data plane must be loopback");
|
|
|
|
// ---- 2. connect a normal client -------------------------------------
|
|
let mut client_cfg = ClientConfig::new(root.join("proof-cache"));
|
|
client_cfg.token = Some(token);
|
|
let endpoints = ApiEndpoints { control: server.control_addr(), data: server.data_addr() };
|
|
let mut client = match AssetClient::connect(client_cfg, endpoints, Some(server.server_id())) {
|
|
Ok(c) => c,
|
|
Err(e) => {
|
|
eprintln!("FAIL: client could not attach: {e}");
|
|
std::process::exit(1);
|
|
}
|
|
};
|
|
println!("[2] client attached over HTTP");
|
|
|
|
// ---- 3. walk + import by reference -----------------------------------
|
|
let mut paths = Vec::new();
|
|
if media_dir.is_file() {
|
|
paths.push(media_dir.clone());
|
|
} else {
|
|
walk(&media_dir, 0, &mut paths);
|
|
}
|
|
let mut published = 0usize;
|
|
let mut present = 0usize;
|
|
let mut failed = 0usize;
|
|
let mut imported: Vec<(PathBuf, makepad_asset_data::BlobId, u64)> = Vec::new();
|
|
let rights = rights();
|
|
let t_import = Instant::now();
|
|
|
|
for path in &paths {
|
|
let ext = path
|
|
.extension()
|
|
.and_then(|e| e.to_str())
|
|
.unwrap_or_default()
|
|
.to_ascii_lowercase();
|
|
let Some((kind, role, media)) = classify(&ext) else { continue };
|
|
let abs = std::path::absolute(path).unwrap_or_else(|_| path.clone());
|
|
let stem = abs.file_stem().and_then(|s| s.to_str()).unwrap_or("clip").to_string();
|
|
let alias_text =
|
|
format!("{NS}/{}-{}", slug(&stem, 48), hex8(abs.to_string_lossy().as_bytes()));
|
|
let Ok(alias) = AssetAlias::new(alias_text) else {
|
|
failed += 1;
|
|
continue;
|
|
};
|
|
if !matches!(client.resolve_alias(&alias), Err(ClientError::NotFound { .. })) {
|
|
present += 1;
|
|
continue;
|
|
}
|
|
|
|
// The thumbnail is the one thing the store DOES own. For video it
|
|
// costs one hardware-decoded frame and no full read.
|
|
let (thumb, millis, dims) = match kind {
|
|
AssetKind::Video => match probe_video(&abs) {
|
|
Ok(p) => {
|
|
if !p.real_frame {
|
|
println!(" ! {}: no decodable first frame", stem);
|
|
}
|
|
(
|
|
PublishThumbnail::plain(
|
|
p.thumbnail_jpeg,
|
|
ThumbnailMedia::Jpeg,
|
|
thumbs::THUMB_DIM as u32,
|
|
thumbs::THUMB_DIM as u32,
|
|
),
|
|
p.duration_ms,
|
|
None,
|
|
)
|
|
}
|
|
Err(e) => {
|
|
println!(" x {}: video probe failed: {e}", stem);
|
|
failed += 1;
|
|
continue;
|
|
}
|
|
},
|
|
AssetKind::Texture => {
|
|
let bytes = std::fs::read(&abs).unwrap_or_default();
|
|
let d = thumbs::png_dims(&bytes).or_else(|| thumbs::jpeg_dims(&bytes));
|
|
match (makepad_asset_importer::import::usable_image_thumb(&bytes), d) {
|
|
(Some((t, m, w, h)), Some(d)) => {
|
|
(PublishThumbnail::plain(t, m, w, h), 0, Some(d))
|
|
}
|
|
_ => {
|
|
println!(" x {}: unusable image", stem);
|
|
failed += 1;
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
_ => {
|
|
let bytes = std::fs::read(&abs).unwrap_or_default();
|
|
match thumbs::decode_audio(&bytes, media)
|
|
.and_then(|pcm| thumbs::audio_thumbnail_jpeg(&pcm))
|
|
{
|
|
Ok(t) => (
|
|
PublishThumbnail {
|
|
bytes: t.bytes,
|
|
media: ThumbnailMedia::Jpeg,
|
|
width: t.width,
|
|
height: t.height,
|
|
views: t.views,
|
|
},
|
|
thumbs::audio_millis(&bytes, media).unwrap_or(0),
|
|
None,
|
|
),
|
|
Err(e) => {
|
|
println!(" x {}: audio decode failed: {e}", stem);
|
|
failed += 1;
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
let mut bundle = PublishBundle::new(
|
|
NS,
|
|
kind,
|
|
stem.clone(),
|
|
vec![PublishBundleFile::reference(role, media, abs.clone(), dims)],
|
|
thumb,
|
|
rights.clone(),
|
|
);
|
|
bundle.alias = Some(alias);
|
|
bundle.media_millis = millis;
|
|
bundle.categories = vec!["imported".to_string()];
|
|
match client.publish_bundle(&bundle) {
|
|
Ok(p) => {
|
|
published += 1;
|
|
imported.push((abs.clone(), p.files[0].blob, p.files[0].byte_len));
|
|
}
|
|
Err(e) => {
|
|
println!(" x {}: publish failed: {e}", stem);
|
|
failed += 1;
|
|
}
|
|
}
|
|
}
|
|
println!(
|
|
"[3] imported by reference: {published} published, {present} already present, {failed} failed ({} ms)",
|
|
t_import.elapsed().as_millis()
|
|
);
|
|
|
|
// ---- 4. the store did NOT copy the payloads --------------------------
|
|
let cas_bytes = dir_bytes(&root.join("cas"));
|
|
let source_bytes: u64 = imported.iter().map(|(_, _, n)| *n).sum();
|
|
println!(
|
|
"[4] cas holds {:.2} MB; the media it catalogues is {:.2} MB",
|
|
cas_bytes as f64 / 1e6,
|
|
source_bytes as f64 / 1e6
|
|
);
|
|
if source_bytes > 0 && cas_bytes >= source_bytes {
|
|
eprintln!("FAIL: the store copied the payloads (cas >= source bytes)");
|
|
std::process::exit(1);
|
|
}
|
|
|
|
// ---- 5. read every payload back and check it -------------------------
|
|
let mut verified = 0usize;
|
|
let mut bad = 0usize;
|
|
for (path, blob, len) in &imported {
|
|
let want = std::fs::read(path).unwrap_or_default();
|
|
match client.fetch_blob_bytes(blob, Some(*len)) {
|
|
Ok(got) if got == want => verified += 1,
|
|
Ok(_) => {
|
|
eprintln!(" MISMATCH {}", path.display());
|
|
bad += 1;
|
|
}
|
|
Err(e) => {
|
|
eprintln!(" UNREADABLE {}: {e}", path.display());
|
|
bad += 1;
|
|
}
|
|
}
|
|
}
|
|
println!("[5] served back and byte-compared: {verified} ok, {bad} bad");
|
|
|
|
// ---- 6. the catalog can be browsed like any other --------------------
|
|
match client.assets_page(Some(NS), None, 50) {
|
|
Ok(page) => println!("[6] catalog lists {} assets in {NS}", page.assets.len()),
|
|
Err(e) => println!("[6] catalog listing failed: {e}"),
|
|
}
|
|
|
|
// ---- 7. the reference re-scan sees them all as present ---------------
|
|
match client.blob_refs_page(None, 64) {
|
|
Ok(page) => {
|
|
let ok = page.refs.iter().filter(|r| r.ok).count();
|
|
println!("[7] {} reference blobs, {} present", page.total, ok);
|
|
for r in page.refs.iter().filter(|r| !r.ok) {
|
|
println!(" ! {} is {}", r.path, r.state);
|
|
}
|
|
}
|
|
Err(e) => println!("[7] rescan failed: {e}"),
|
|
}
|
|
|
|
// ---- 8. drift is caught, not served ----------------------------------
|
|
if let Some((path, blob, len)) = imported.first().cloned() {
|
|
let original = std::fs::read(&path).unwrap_or_default();
|
|
let mut tampered = original.clone();
|
|
if !tampered.is_empty() {
|
|
tampered[0] ^= 0xff;
|
|
std::fs::write(&path, &tampered).ok();
|
|
let mut fresh_cfg = ClientConfig::new(root.join("proof-cache-2"));
|
|
fresh_cfg.token = std::fs::read_to_string(root.join("admin-token"))
|
|
.ok()
|
|
.map(|t| t.trim().to_string());
|
|
let refused = match AssetClient::connect(fresh_cfg, endpoints, Some(server.server_id()))
|
|
{
|
|
Ok(mut c) => c.fetch_blob_bytes(&blob, Some(len)).is_err(),
|
|
Err(_) => false,
|
|
};
|
|
std::fs::write(&path, &original).ok();
|
|
println!(
|
|
"[8] a file changed underneath is {}",
|
|
if refused { "REFUSED (correct)" } else { "SERVED ANYWAY (WRONG)" }
|
|
);
|
|
if !refused {
|
|
std::process::exit(1);
|
|
}
|
|
}
|
|
}
|
|
|
|
if bad > 0 || (published == 0 && present == 0) {
|
|
eprintln!("RESULT: FAIL");
|
|
std::process::exit(1);
|
|
}
|
|
println!("RESULT: PASS ({} ms total)", t0.elapsed().as_millis());
|
|
}
|
|
|
|
fn dir_bytes(dir: &Path) -> u64 {
|
|
let mut total = 0u64;
|
|
let mut stack = vec![dir.to_path_buf()];
|
|
while let Some(d) = stack.pop() {
|
|
let Ok(entries) = std::fs::read_dir(&d) else { continue };
|
|
for e in entries.flatten() {
|
|
let p = e.path();
|
|
if p.is_dir() {
|
|
stack.push(p);
|
|
} else if let Ok(m) = e.metadata() {
|
|
total += m.len();
|
|
}
|
|
}
|
|
}
|
|
total
|
|
}
|