97e9572f42
Apps ask the hub for a recognizer or a voice and get one; where it runs is the hub's decision. AiHub::start_stt / start_tts return poll-driven sessions shaped like the chat session. The Auto ladder is Whisper/Kokoro in this process (weights present, machine election), on the machine node over loopback, on a LAN node, else the OS engine; SpeechReach::Local is the "don't reach out" knob. Audio always comes back as PCM: the app owns the device. Three layers: - makepad-ai-speech is the whole speech model family, engines only. libs/voice (Whisper + Silero VAD) folds in as the `whisper` and `vad` modules next to kokoro and indextts, each a cargo feature; the Apple bridges and the Speaker/VoiceTranscriber selection leave it. - makepad-system-speech (new) is the OS speech services as blocking fns: Apple SpeechAnalyzer/AVSpeechSynthesizer via Swift, Windows.Media.Speech* on the vendored bindings, Android SpeechRecognizer/TextToSpeech through MakepadSpeech.java (API 26 floor), espeak-ng on Linux. It models the two STT shapes honestly: PCM in (Whisper, Apple) versus an engine that owns the microphone (Android, Windows), with capabilities the caller reads. - the hub grows speech sessions, in-process Whisper/Kokoro workers with the residency election, a `whisper` wire backend (stt domain, registry entry pinned to ggerganov/whisper.cpp) so a Mac can serve a Quest, and a `language` field on the generate request. Consumers: the Window voice input runs on an STT session and switches to engine-mic mode when the recognizer owns the microphone; converse's SpeechOutput is a lazily started TTS session plus a pump thread; route drops its private speech copy for converse; vj's lyrics fallback and the alignment bakes call the engines directly. Verified here: speech-roundtrip through the real sessions (Apple voice in, in-process Whisper on Metal out, 4.3% WER); system-speech-test TTS->STT verbatim; hub/converse/system-speech unit tests; msvc, aarch64-android and linux-gnu cross-checks; Java against android-34. Windows, Android and Linux bridges are compile-checked only. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
83 lines
2.2 KiB
TOML
83 lines
2.2 KiB
TOML
[package]
|
|
name = "makepad-ai-speech"
|
|
version = "0.1.0"
|
|
edition = "2021"
|
|
description = "Speech family: IndexTTS-2.5 + Kokoro speech synthesis engines (text in, PCM out). No platform code: OS voices are makepad-system-speech, engine choice is makepad-ai-hub."
|
|
license = "MIT"
|
|
|
|
[features]
|
|
default = ["whisper", "vad", "kokoro", "indextts"]
|
|
# Whisper speech-to-text (CPU SIMD / Metal / CUDA).
|
|
whisper = []
|
|
# Silero voice-activity detection.
|
|
vad = []
|
|
# Kokoro-82M text-to-speech (+ its G2P).
|
|
kokoro = []
|
|
# IndexTTS-2.5 character-voice text-to-speech.
|
|
indextts = []
|
|
# Precompile the Whisper ggml Metal library at build time (else runtime source compile).
|
|
metal-precompile = []
|
|
|
|
[dependencies]
|
|
# THE CUDA store (Whisper's CUDA backend). Must NOT be target-specific: cargo
|
|
# drops target-cfg deps when this crate is a path dep of another standalone
|
|
# `[workspace]`, and the `links = "makepad_ai_cuda"` handshake
|
|
# (DEP_MAKEPAD_AI_CUDA_KERNELS, read in build.rs) only arrives for an
|
|
# unconditional dependency. On macOS/iOS the crate compiles to empty stubs.
|
|
makepad-ai-cuda = { path = "../../cuda" }
|
|
makepad-ai-common = { path = "../common" }
|
|
makepad-ai-loader = { path = "../../loader" }
|
|
makepad-ai-sfx = { path = "../sfx" }
|
|
|
|
[[bin]]
|
|
name = "g2p_test"
|
|
path = "src/bin/g2p_test.rs"
|
|
required-features = ["kokoro"]
|
|
|
|
[[bin]]
|
|
name = "har_bisect"
|
|
path = "src/bin/har_bisect.rs"
|
|
required-features = ["kokoro"]
|
|
|
|
[[bin]]
|
|
name = "kokoro_probe"
|
|
path = "src/bin/kokoro_probe.rs"
|
|
required-features = ["kokoro"]
|
|
|
|
[[bin]]
|
|
name = "parity"
|
|
path = "src/bin/parity.rs"
|
|
required-features = ["kokoro"]
|
|
|
|
[[bin]]
|
|
name = "tts_test"
|
|
path = "src/bin/tts_test.rs"
|
|
required-features = ["kokoro"]
|
|
|
|
[target.'cfg(any(target_os = "macos", target_os = "ios"))'.dependencies]
|
|
makepad-objc-sys = { path = "../../../objc-sys", version = "1.0.0" }
|
|
|
|
[[bin]]
|
|
name = "whisper-test"
|
|
path = "src/bin/whisper_test.rs"
|
|
required-features = ["whisper"]
|
|
|
|
[[bin]]
|
|
name = "whisper-parity"
|
|
path = "src/bin/whisper_parity.rs"
|
|
required-features = ["whisper"]
|
|
|
|
[[bin]]
|
|
name = "vad-test"
|
|
path = "src/bin/vad_test.rs"
|
|
required-features = ["vad"]
|
|
|
|
[[bin]]
|
|
name = "metal-strip-unused"
|
|
path = "src/bin/metal_strip_unused.rs"
|
|
required-features = ["whisper"]
|
|
|
|
[[test]]
|
|
name = "silero_vad"
|
|
path = "tests/silero_vad.rs"
|
|
required-features = ["vad"]
|