97e9572f42
Apps ask the hub for a recognizer or a voice and get one; where it runs is the hub's decision. AiHub::start_stt / start_tts return poll-driven sessions shaped like the chat session. The Auto ladder is Whisper/Kokoro in this process (weights present, machine election), on the machine node over loopback, on a LAN node, else the OS engine; SpeechReach::Local is the "don't reach out" knob. Audio always comes back as PCM: the app owns the device. Three layers: - makepad-ai-speech is the whole speech model family, engines only. libs/voice (Whisper + Silero VAD) folds in as the `whisper` and `vad` modules next to kokoro and indextts, each a cargo feature; the Apple bridges and the Speaker/VoiceTranscriber selection leave it. - makepad-system-speech (new) is the OS speech services as blocking fns: Apple SpeechAnalyzer/AVSpeechSynthesizer via Swift, Windows.Media.Speech* on the vendored bindings, Android SpeechRecognizer/TextToSpeech through MakepadSpeech.java (API 26 floor), espeak-ng on Linux. It models the two STT shapes honestly: PCM in (Whisper, Apple) versus an engine that owns the microphone (Android, Windows), with capabilities the caller reads. - the hub grows speech sessions, in-process Whisper/Kokoro workers with the residency election, a `whisper` wire backend (stt domain, registry entry pinned to ggerganov/whisper.cpp) so a Mac can serve a Quest, and a `language` field on the generate request. Consumers: the Window voice input runs on an STT session and switches to engine-mic mode when the recognizer owns the microphone; converse's SpeechOutput is a lazily started TTS session plus a pump thread; route drops its private speech copy for converse; vj's lyrics fallback and the alignment bakes call the engines directly. Verified here: speech-roundtrip through the real sessions (Apple voice in, in-process Whisper on Metal out, 4.3% WER); system-speech-test TTS->STT verbatim; hub/converse/system-speech unit tests; msvc, aarch64-android and linux-gnu cross-checks; Java against android-34. Windows, Android and Linux bridges are compile-checked only. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
26 lines
1.4 KiB
TOML
26 lines
1.4 KiB
TOML
[package]
|
|
name = "makepad-app-route"
|
|
version = "0.1.0"
|
|
edition = "2021"
|
|
description = "AI-driven voice route/trip planner on MapView (see route.md)"
|
|
license = "MIT OR Apache-2.0"
|
|
|
|
[dependencies]
|
|
makepad-widgets = { path = "../../widgets", version = "2.0.0", features = ["maps", "voice"] }
|
|
mp-theme = { path = "../../libs/mp_theme" }
|
|
makepad-map-nav = { path = "../../libs/map_nav" } # M1: route.* tools (search + A* graph)
|
|
makepad-map-build = { path = "../../libs/map_build", features = ["faces"] } # first run: download + bake a test map
|
|
makepad-geodata = { path = "../../libs/geodata" } # M1: geo.* corridor queries, rain radar
|
|
makepad-mbtile-reader = { path = "../../libs/mbtile_reader" } # terrain drape landcover source
|
|
serde_json = "1" # geodata FeatureHit attrs
|
|
|
|
makepad-ai-hub = { path = "../../libs/ai/hub", default-features = false, features = ["llm"] } # M2: local dispatcher LLM
|
|
|
|
makepad-strict-json = { path = "../../libs/strict_json" } # claude_agent tool schemas
|
|
makepad-converse = { path = "../../libs/converse", features = ["local-llm"] } # M3: attention-gate filter
|
|
|
|
# Planned wiring (see route.md milestones) — enable as each phase lands:
|
|
# makepad-tesla = { path = "../../libs/tesla" } # M5: vehicle SoC / charge state
|
|
|
|
[dev-dependencies]
|
|
makepad-test = { path = "../../libs/makepad_test", version = "0.1.0" }
|