97e9572f42
Apps ask the hub for a recognizer or a voice and get one; where it runs is the hub's decision. AiHub::start_stt / start_tts return poll-driven sessions shaped like the chat session. The Auto ladder is Whisper/Kokoro in this process (weights present, machine election), on the machine node over loopback, on a LAN node, else the OS engine; SpeechReach::Local is the "don't reach out" knob. Audio always comes back as PCM: the app owns the device. Three layers: - makepad-ai-speech is the whole speech model family, engines only. libs/voice (Whisper + Silero VAD) folds in as the `whisper` and `vad` modules next to kokoro and indextts, each a cargo feature; the Apple bridges and the Speaker/VoiceTranscriber selection leave it. - makepad-system-speech (new) is the OS speech services as blocking fns: Apple SpeechAnalyzer/AVSpeechSynthesizer via Swift, Windows.Media.Speech* on the vendored bindings, Android SpeechRecognizer/TextToSpeech through MakepadSpeech.java (API 26 floor), espeak-ng on Linux. It models the two STT shapes honestly: PCM in (Whisper, Apple) versus an engine that owns the microphone (Android, Windows), with capabilities the caller reads. - the hub grows speech sessions, in-process Whisper/Kokoro workers with the residency election, a `whisper` wire backend (stt domain, registry entry pinned to ggerganov/whisper.cpp) so a Mac can serve a Quest, and a `language` field on the generate request. Consumers: the Window voice input runs on an STT session and switches to engine-mic mode when the recognizer owns the microphone; converse's SpeechOutput is a lazily started TTS session plus a pump thread; route drops its private speech copy for converse; vj's lyrics fallback and the alignment bakes call the engines directly. Verified here: speech-roundtrip through the real sessions (Apple voice in, in-process Whisper on Metal out, 4.3% WER); system-speech-test TTS->STT verbatim; hub/converse/system-speech unit tests; msvc, aarch64-android and linux-gnu cross-checks; Java against android-34. Windows, Android and Linux bridges are compile-checked only. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
46 lines
1.6 KiB
TOML
46 lines
1.6 KiB
TOML
[package]
|
|
name = "makepad-system-speech"
|
|
version = "0.1.0"
|
|
edition = "2021"
|
|
description = "The operating system's own speech services as plain blocking functions: speech-to-text and text-to-speech through Apple Speech / AVSpeechSynthesizer, Windows.Media.SpeechRecognition / SpeechSynthesis, Android SpeechRecognizer / TextToSpeech, and espeak-ng on Linux. No models, no network, no hub — makepad-ai-hub publishes these as the `stt.system` / `tts.system` pipes."
|
|
license = "MIT"
|
|
|
|
[dependencies]
|
|
|
|
[target.'cfg(target_os = "android")'.dependencies]
|
|
## Same rule as makepad-platform: crates.io versions, not path deps, so exactly
|
|
## one copy of the JNI / activity state exists in the app binary.
|
|
makepad-jni-sys = { version = "0.4.0" }
|
|
makepad-android-state = { version = "0.1.0" }
|
|
|
|
[target.'cfg(windows)'.dependencies.windows-core]
|
|
path = "../windows/windows-core"
|
|
version = "0.62.2"
|
|
|
|
## `IAsyncOperation` / `IAsyncAction` are named directly by the blocking waits
|
|
## in `platform/windows.rs`; the `windows` crate returns them but re-exports
|
|
## neither.
|
|
[target.'cfg(windows)'.dependencies.windows-future]
|
|
path = "../windows/windows-future"
|
|
version = "0.3.2"
|
|
default-features = false
|
|
|
|
[target.'cfg(windows)'.dependencies.windows]
|
|
path = "../windows/windows-rs"
|
|
version = "0.62.2"
|
|
features = [
|
|
"Foundation",
|
|
"Foundation_Collections",
|
|
"Globalization",
|
|
## `SpeechSynthesisStream` is gated on Media_Core: it is an IMediaSource.
|
|
"Media_Core",
|
|
"Media_SpeechSynthesis",
|
|
"Media_SpeechRecognition",
|
|
"Storage_Streams",
|
|
"Win32_Foundation",
|
|
"Win32_System_WinRT",
|
|
]
|
|
|
|
[[bin]]
|
|
name = "system-speech-test"
|
|
path = "src/bin/system_speech_test.rs"
|