makepad/libs/system_speech/build.rs
Admin 97e9572f42 speech: one STT/TTS API on every platform, through the ai-hub
Apps ask the hub for a recognizer or a voice and get one; where it runs is
the hub's decision. AiHub::start_stt / start_tts return poll-driven
sessions shaped like the chat session. The Auto ladder is Whisper/Kokoro in
this process (weights present, machine election), on the machine node over
loopback, on a LAN node, else the OS engine; SpeechReach::Local is the
"don't reach out" knob. Audio always comes back as PCM: the app owns the
device.

Three layers:
- makepad-ai-speech is the whole speech model family, engines only.
  libs/voice (Whisper + Silero VAD) folds in as the `whisper` and `vad`
  modules next to kokoro and indextts, each a cargo feature; the Apple
  bridges and the Speaker/VoiceTranscriber selection leave it.
- makepad-system-speech (new) is the OS speech services as blocking fns:
  Apple SpeechAnalyzer/AVSpeechSynthesizer via Swift, Windows.Media.Speech*
  on the vendored bindings, Android SpeechRecognizer/TextToSpeech through
  MakepadSpeech.java (API 26 floor), espeak-ng on Linux. It models the two
  STT shapes honestly: PCM in (Whisper, Apple) versus an engine that owns
  the microphone (Android, Windows), with capabilities the caller reads.
- the hub grows speech sessions, in-process Whisper/Kokoro workers with the
  residency election, a `whisper` wire backend (stt domain, registry entry
  pinned to ggerganov/whisper.cpp) so a Mac can serve a Quest, and a
  `language` field on the generate request.

Consumers: the Window voice input runs on an STT session and switches to
engine-mic mode when the recognizer owns the microphone; converse's
SpeechOutput is a lazily started TTS session plus a pump thread; route
drops its private speech copy for converse; vj's lyrics fallback and the
alignment bakes call the engines directly.

Verified here: speech-roundtrip through the real sessions (Apple voice in,
in-process Whisper on Metal out, 4.3% WER); system-speech-test TTS->STT
verbatim; hub/converse/system-speech unit tests; msvc, aarch64-android and
linux-gnu cross-checks; Java against android-34. Windows, Android and Linux
bridges are compile-checked only.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-01 23:32:35 +02:00

198 lines
7.6 KiB
Rust

//! Builds the Apple bridge (`swift/stt_bridge.swift` + `swift/tts_bridge.swift`)
//! into one static library on Apple hosts targeting macOS/iOS. Without a Swift
//! toolchain, or off Apple, the crate compiles without the `apple_speech` cfg
//! and both engines report themselves unavailable.
use std::env;
use std::fs;
use std::process::Command;
/// SpeechAnalyzer (the STT half of the bridge) is iOS 26+.
const IOS_DEPLOYMENT_TARGET_DEFAULT: &str = "26.0";
fn main() {
println!("cargo:rustc-check-cfg=cfg(apple_speech)");
println!("cargo:rerun-if-changed=swift/stt_bridge.swift");
println!("cargo:rerun-if-changed=swift/tts_bridge.swift");
println!("cargo:rerun-if-env-changed=MAKEPAD_SYSTEM_SPEECH_NO_APPLE_BRIDGE");
println!("cargo:rerun-if-env-changed=IPHONEOS_DEPLOYMENT_TARGET");
println!("cargo:rerun-if-env-changed=IPHONESIMULATOR_DEPLOYMENT_TARGET");
let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap_or_default();
let is_apple_host = env::var("HOST").unwrap_or_default().contains("apple-darwin");
let is_apple_target = target_os == "macos" || target_os == "ios";
let disabled = env::var_os("MAKEPAD_SYSTEM_SPEECH_NO_APPLE_BRIDGE").is_some();
if is_apple_host && is_apple_target && !disabled && build_apple_bridge(&target_os) {
println!("cargo:rustc-cfg=apple_speech");
} else if is_apple_target && !disabled {
println!("cargo:warning=makepad-system-speech: Apple bridge not built; system STT/TTS unavailable");
}
}
fn build_apple_bridge(target_os: &str) -> bool {
let out_dir = env::var("OUT_DIR").unwrap();
let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
let module_cache = format!("{out_dir}/swift_module_cache");
let _ = fs::create_dir_all(&module_cache);
let mut args = vec![
"-emit-library".to_string(),
"-static".to_string(),
"-parse-as-library".to_string(),
"-module-name".to_string(),
"makepad_system_speech".to_string(),
"-module-cache-path".to_string(),
module_cache,
"-O".to_string(),
];
if target_os == "ios" {
if let Some((target, sdk)) = ios_target_and_sdk() {
args.push("-target".to_string());
args.push(target);
args.push("-sdk".to_string());
args.push(sdk);
}
}
args.push("-o".to_string());
args.push(format!("{out_dir}/libmakepad_system_speech.a"));
args.push(format!("{manifest_dir}/swift/stt_bridge.swift"));
args.push(format!("{manifest_dir}/swift/tts_bridge.swift"));
match Command::new("swiftc").args(&args).status() {
Ok(status) if status.success() => {}
Ok(_) => {
println!("cargo:warning=swiftc failed for the makepad-system-speech Apple bridge");
return false;
}
Err(err) => {
println!("cargo:warning=swiftc unavailable ({err}) for the makepad-system-speech Apple bridge");
return false;
}
}
// Must come BEFORE any other link-search line so the patched .tbd files win.
if target_os == "macos" {
fix_swift_rpath_tbds(&out_dir);
}
println!("cargo:rustc-link-search=native={out_dir}");
println!("cargo:rustc-link-lib=static=makepad_system_speech");
// The Swift objects were built for this deployment target; the Rust link
// step must agree or the async runtime's symbols go missing.
if target_os == "ios" {
let deployment = ios_deployment_target();
println!("cargo:rustc-link-arg=-miphoneos-version-min={deployment}");
}
println!("cargo:rustc-link-lib=framework=Speech");
println!("cargo:rustc-link-lib=framework=Foundation");
println!("cargo:rustc-link-lib=framework=AVFoundation");
println!("cargo:rustc-link-lib=framework=CoreMedia");
// Swift runtime search paths so the linker resolves the bridge's symbols.
if let Ok(output) = Command::new("swiftc").args(["-print-target-info"]).output() {
if output.status.success() {
let info = String::from_utf8_lossy(&output.stdout);
for line in info.lines() {
let path = line.trim().trim_matches('"').trim_end_matches(',');
if path.starts_with('/') && path.contains("lib/swift") {
println!("cargo:rustc-link-search=native={path}");
}
}
}
}
true
}
fn ios_is_simulator() -> bool {
let arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default();
let abi = env::var("CARGO_CFG_TARGET_ABI").unwrap_or_default();
abi == "sim" || arch == "x86_64"
}
fn ios_deployment_target() -> String {
let key = if ios_is_simulator() {
"IPHONESIMULATOR_DEPLOYMENT_TARGET"
} else {
"IPHONEOS_DEPLOYMENT_TARGET"
};
env::var(key).unwrap_or_else(|_| IOS_DEPLOYMENT_TARGET_DEFAULT.to_string())
}
fn ios_target_and_sdk() -> Option<(String, String)> {
let arch = env::var("CARGO_CFG_TARGET_ARCH").ok()?;
let swift_arch = match arch.as_str() {
"aarch64" => "arm64",
"x86_64" => "x86_64",
_ => return None,
};
let deployment = ios_deployment_target();
let (swift_target, sdk_name) = if ios_is_simulator() {
(format!("{swift_arch}-apple-ios{deployment}-simulator"), "iphonesimulator")
} else {
(format!("{swift_arch}-apple-ios{deployment}"), "iphoneos")
};
let sdk_path = Command::new("xcrun")
.args(["--sdk", sdk_name, "--show-sdk-path"])
.output()
.ok()
.filter(|out| out.status.success())
.map(|out| String::from_utf8_lossy(&out.stdout).trim().to_string())?;
Some((swift_target, sdk_path))
}
/// The macOS SDK's Swift runtime `.tbd` files carry `$ld$previous` entries that
/// make the linker record `@rpath/libswift_Concurrency.dylib` as the install
/// name whenever MACOSX_DEPLOYMENT_TARGET < 15 — and Rust defaults to 11.0. The
/// binary then dies at launch with "Library not loaded: @rpath/...". Patched
/// copies without those entries, searched first, make the linker use the
/// absolute `/usr/lib/swift/...` names instead.
fn fix_swift_rpath_tbds(out_dir: &str) {
let Some(sdk_path) = Command::new("xcrun")
.args(["--show-sdk-path"])
.output()
.ok()
.filter(|o| o.status.success())
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
else {
return;
};
let override_dir = format!("{out_dir}/swift_tbd_override");
if fs::create_dir_all(&override_dir).is_err() {
return;
}
let swift_tbd_dir = format!("{sdk_path}/usr/lib/swift");
for name in [
"libswift_Concurrency.tbd",
"libswiftCore.tbd",
"libswiftFoundation.tbd",
"libswift_StringProcessing.tbd",
"libswift_RegexParser.tbd",
] {
let Ok(content) = fs::read_to_string(format!("{swift_tbd_dir}/{name}")) else {
continue;
};
if !content.contains("$ld$previous$@rpath/") {
continue;
}
let _ = fs::write(format!("{override_dir}/{name}"), strip_ld_previous_rpath(&content));
}
println!("cargo:rustc-link-search=native={override_dir}");
}
fn strip_ld_previous_rpath(content: &str) -> String {
let mut result = content.to_string();
while let Some(start) = result.find("'$ld$previous$@rpath/") {
let Some(end_quote_offset) = result[start + 1..].find('\'') else {
break;
};
let end = start + 1 + end_quote_offset + 1;
let rest = &result[end..];
let trimmed = rest.trim_start_matches(|c: char| matches!(c, ',' | ' ' | '\n' | '\r'));
let skip = rest.len() - trimmed.len();
result = format!("{}{}", &result[..start], &result[end + skip..]);
}
result
}