Compare commits
461 commits
435135ed50
...
fe5b75d92d
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fe5b75d92d | ||
|
|
a75fe914ff | ||
|
|
b005c6e9b2 | ||
|
|
9e61553c96 | ||
|
|
09fa1f080c | ||
|
|
1801e38453 | ||
|
|
83a00d2801 | ||
|
|
75b295ce9a | ||
|
|
b49e2e5060 | ||
|
|
3ce27922a4 | ||
|
|
3dfe5aaed3 | ||
|
|
2f44d20b51 | ||
|
|
f8b9a670b7 | ||
|
|
88048aa784 | ||
|
|
97912796a8 | ||
|
|
1be1e2153c | ||
|
|
39a1b742ac | ||
|
|
6f2fdfa861 | ||
|
|
1181a3bae4 | ||
|
|
2b4bff9a38 | ||
|
|
30575f082d | ||
|
|
b0b0e41da0 | ||
|
|
f3b6c2f36c | ||
|
|
778dbb8fc7 | ||
|
|
8f51704697 | ||
|
|
f45793a8fa | ||
|
|
28016ca897 | ||
|
|
2a064d3928 | ||
|
|
acd23289c6 | ||
|
|
a6c5f15bf7 | ||
|
|
767e260b97 | ||
|
|
bf9418e2bc | ||
|
|
b37b1168fd | ||
|
|
a7774885a1 | ||
|
|
c71d6392f9 | ||
|
|
7e397cf31e | ||
|
|
9a31d2104a | ||
|
|
24c927ad7e | ||
|
|
d20a20871a | ||
|
|
30ef92597f | ||
|
|
5863c43eb1 | ||
|
|
cff15ac045 | ||
|
|
2a5172465c | ||
|
|
27f02f2443 | ||
|
|
c7241e00be | ||
|
|
c6890fd47a | ||
|
|
cf3b6488de | ||
|
|
f5ec84e671 | ||
|
|
8f23e3b6ee | ||
|
|
ad5e98bbad | ||
|
|
75c34413be | ||
|
|
c90df1fe0a | ||
|
|
5a9eed960c | ||
|
|
303458d07a | ||
|
|
4c370488e9 | ||
|
|
2e869f84ea | ||
|
|
8c40d28a47 | ||
|
|
73886d1664 | ||
|
|
3c2530d000 | ||
|
|
2ed495f2ed | ||
|
|
4781378985 | ||
|
|
b41fab482f | ||
|
|
6139a7386f | ||
|
|
b7d95d823e | ||
|
|
8ff1d1c41b | ||
|
|
f95e34b4c1 | ||
|
|
c79e84efca | ||
|
|
4aec005469 | ||
|
|
4ee4f4cb79 | ||
|
|
018ce7370d | ||
|
|
8eaa6ec3df | ||
|
|
8b05496651 | ||
|
|
2f3e393c59 | ||
|
|
ddc4709b32 | ||
|
|
4cd24c4905 | ||
|
|
bc6c620ede | ||
|
|
ca92ff9da4 | ||
|
|
dbb46be699 | ||
|
|
a53d9b6b40 | ||
|
|
43302a63ed | ||
|
|
dbc7838094 | ||
|
|
635c3a3063 | ||
|
|
aa271c36f6 | ||
|
|
d641cbf995 | ||
|
|
cd33943a7a | ||
|
|
01e77d99c0 | ||
|
|
b04333265e | ||
|
|
179124f085 | ||
|
|
e2a37c97f7 | ||
|
|
e689aea8f5 | ||
|
|
a50750fda6 | ||
|
|
199be9eb1d | ||
|
|
e48b056df1 | ||
|
|
df9f4498b6 | ||
|
|
2dbd1b7dca | ||
|
|
bb6d48fc6a | ||
|
|
1f244282b1 | ||
|
|
075e1a74d4 | ||
|
|
d852699d08 | ||
|
|
7db06e8c09 | ||
|
|
0d7ddee3f7 | ||
|
|
a8377920b7 | ||
|
|
fe23e067dd | ||
|
|
7f2cc52e57 | ||
|
|
bb49fe6989 | ||
|
|
64f6d41731 | ||
|
|
662a1216e5 | ||
|
|
a40ad74a63 | ||
|
|
15c3b11da1 | ||
|
|
4e8e4cd45f | ||
|
|
102ffc5fa5 | ||
|
|
14b194bed1 | ||
|
|
7b10d61fc7 | ||
|
|
f615054dab | ||
|
|
de9aba3d83 | ||
|
|
93e7dae15d | ||
|
|
976ea0ed04 | ||
|
|
263d6f0ae8 | ||
|
|
dd51a028aa | ||
|
|
ec40fdb050 | ||
|
|
32bea89a92 | ||
|
|
8d542feea7 | ||
|
|
ccf818313d | ||
|
|
e3abf4dbbf | ||
|
|
09e4bf1e1e | ||
|
|
a2e0c43e4c | ||
|
|
e878c4dbb6 | ||
|
|
0d3a9b84d6 | ||
|
|
c36f1e78bc | ||
|
|
1980c24973 | ||
|
|
58f0700df3 | ||
|
|
2a096e50c3 | ||
|
|
5a9fa62eed | ||
|
|
428d23e8ed | ||
|
|
098ba7737a | ||
|
|
a8b0382dea | ||
|
|
6691420e25 | ||
|
|
2aa0780d0a | ||
|
|
b3da0c5712 | ||
|
|
90c4f59357 | ||
|
|
639887dda0 | ||
|
|
2b98a1f048 | ||
|
|
d74875327f | ||
|
|
13f935911a | ||
|
|
42213f95b1 | ||
|
|
862f3bd298 | ||
|
|
2a902c646d | ||
|
|
285bc9866f | ||
|
|
cb498c9c5c | ||
|
|
9e2d65b50b | ||
|
|
90dfcb630b | ||
|
|
72ab9b0f31 | ||
|
|
697215e1d9 | ||
|
|
2d4037d141 | ||
|
|
b1a92b73c6 | ||
|
|
6dcca0088b | ||
|
|
78a1681a49 | ||
|
|
5184340565 | ||
|
|
c7639f0841 | ||
|
|
dc9cce6c94 | ||
|
|
cb75d367d6 | ||
|
|
35ac364eea | ||
|
|
71c0e1eedf | ||
|
|
4e61e44fca | ||
|
|
707d7029e9 | ||
|
|
acb8689ad7 | ||
|
|
b0ec2a4d0a | ||
|
|
569b4a7f7c | ||
|
|
44b3a7f02e | ||
|
|
83a8d4f69d | ||
|
|
c3d9314f8b | ||
|
|
daf88e10cb | ||
|
|
645de42e52 | ||
|
|
a3248d1c51 | ||
|
|
410acd0eaa | ||
|
|
58d3fd5d5d | ||
|
|
2db9d4812c | ||
|
|
8ae7c3c396 | ||
|
|
d6d324144e | ||
|
|
662d141d19 | ||
|
|
13c2500678 | ||
|
|
66617cdd7a | ||
|
|
aa907cab9c | ||
|
|
97e0701ff8 | ||
|
|
0ef0311812 | ||
|
|
1e2dcd0da8 | ||
|
|
b39d301ee2 | ||
|
|
fef80585cb | ||
|
|
7a5bf69272 | ||
|
|
fad1c49827 | ||
|
|
dc85eb0ed3 | ||
|
|
0fdfba76a3 | ||
|
|
eec516ee06 | ||
|
|
bc87a3d359 | ||
|
|
4f98c73be2 | ||
|
|
a2cbe38330 | ||
|
|
7a2c09a873 | ||
|
|
cc1f25622a | ||
|
|
afdffd71b9 | ||
|
|
7b33f7bec6 | ||
|
|
3b1a8c264f | ||
|
|
e7be0e31dc | ||
|
|
362ac59407 | ||
|
|
d8539d89e1 | ||
|
|
463de64b33 | ||
|
|
9b4a3cac3e | ||
|
|
7e66991246 | ||
|
|
62f38a5b33 | ||
|
|
30fc13bba0 | ||
|
|
cb572aee0f | ||
|
|
f987651778 | ||
|
|
5a5eda2979 | ||
|
|
5b3378164c | ||
|
|
55a38104aa | ||
|
|
99d2180aad | ||
|
|
5b49fee084 | ||
|
|
9d664ca967 | ||
|
|
ef25495540 | ||
|
|
02b58fc931 | ||
|
|
57995b6c92 | ||
|
|
9047600062 | ||
|
|
a6f495dfd3 | ||
|
|
72b60b6d0c | ||
|
|
2d4ff16b93 | ||
|
|
383b83e783 | ||
|
|
c2c7f51400 | ||
|
|
21fce80db6 | ||
|
|
5c78ac04a3 | ||
|
|
c527cd8e88 | ||
|
|
6fddf9e285 | ||
|
|
0b5bff02cc | ||
|
|
f35dd1dd8c | ||
|
|
ff6cfec0fc | ||
|
|
5c4076168c | ||
|
|
1dcca41ec4 | ||
|
|
453197da4d | ||
|
|
c1f7b53dca | ||
|
|
ffd62da242 | ||
|
|
c7fe851c2e | ||
|
|
a9b2185759 | ||
|
|
8625076661 | ||
|
|
53f0e965f8 | ||
|
|
40e483f1ff | ||
|
|
93232a2b2e | ||
|
|
64d2d3acee | ||
|
|
cc8ee5c9f5 | ||
|
|
04a36ded31 | ||
|
|
017dc69faf | ||
|
|
78bfd5c27c | ||
|
|
3691875775 | ||
|
|
0950ed7b4e | ||
|
|
4b529a1203 | ||
|
|
37686534b0 | ||
|
|
95fd7d6c82 | ||
|
|
0e5e007f03 | ||
|
|
5fdd5d5f47 | ||
|
|
190062e4a1 | ||
|
|
a772ec8f40 | ||
|
|
432121ebe5 | ||
|
|
73ffe87e8b | ||
|
|
d1910b8fe4 | ||
|
|
8b46ce0279 | ||
|
|
4ca36c1d92 | ||
|
|
82ac76882e | ||
|
|
6105b61e6e | ||
|
|
43c36c1a36 | ||
|
|
3c033977f8 | ||
|
|
7f1fefd56d | ||
|
|
4e58ab9dec | ||
|
|
f80895accd | ||
|
|
7deb052ad1 | ||
|
|
4317f58804 | ||
|
|
dbe43bd327 | ||
|
|
0b2be94917 | ||
|
|
e6a305c7e7 | ||
|
|
82def6b25b | ||
|
|
6d47c3f5ac | ||
|
|
341f66d10a | ||
|
|
d9f04fcabf | ||
|
|
d3cd233e23 | ||
|
|
6857d86461 | ||
|
|
94b8726ffb | ||
|
|
0684b4123d | ||
|
|
5d007e0373 | ||
|
|
0105fa5cd8 | ||
|
|
9630337538 | ||
|
|
511048ca85 | ||
|
|
412b3c920e | ||
|
|
106b38cede | ||
|
|
aa7345d8c5 | ||
|
|
6be1118b2d | ||
|
|
58ad22e94b | ||
|
|
7613f3ef2f | ||
|
|
6b77318209 | ||
|
|
6f8c08f4a6 | ||
|
|
640602cc84 | ||
|
|
6c32c88d1a | ||
|
|
4946c9eb0d | ||
|
|
9c9c72ba8d | ||
|
|
441d604e5f | ||
|
|
22faab2c8b | ||
|
|
cf2b8ca4bd | ||
|
|
d3472ebd35 | ||
|
|
faf8112816 | ||
|
|
82d0cfaab9 | ||
|
|
0f967ce382 | ||
|
|
942ff86df3 | ||
|
|
670cc76977 | ||
|
|
77d91385fc | ||
|
|
9942e26f48 | ||
|
|
372bfd7960 | ||
|
|
22b2bf9fc3 | ||
|
|
0ecb81cf56 | ||
|
|
f51ca0795e | ||
|
|
1b7089962a | ||
|
|
fd5d70c108 | ||
|
|
04a94ef780 | ||
|
|
6b661bf4c5 | ||
|
|
dfe2087399 | ||
|
|
fb1416d8c0 | ||
|
|
f6bbee90ef | ||
|
|
b0646d0714 | ||
|
|
232909d16a | ||
|
|
aa816ed8a6 | ||
|
|
2b035ff33c | ||
|
|
ff62db3c5e | ||
|
|
c24c2064fd | ||
|
|
e2a262b05a | ||
|
|
19cf377238 | ||
|
|
d76abf387e | ||
|
|
ab8febc66d | ||
|
|
d1e7a6fea8 | ||
|
|
05fd7e2dae | ||
|
|
611c9ebd8c | ||
|
|
2e48c5bfb7 | ||
|
|
acad401218 | ||
|
|
dcf7d21060 | ||
|
|
60ee97858c | ||
|
|
d6cc49ad22 | ||
|
|
c1febc7985 | ||
|
|
4081e48c35 | ||
|
|
e5d37e0b0b | ||
|
|
2f5a9849c6 | ||
|
|
de937ac01c | ||
|
|
84b46c11a8 | ||
|
|
17437a2f53 | ||
|
|
4d75e65b13 | ||
|
|
b0e34c2f57 | ||
|
|
1940f3da25 | ||
|
|
6981e9052b | ||
|
|
942476f68c | ||
|
|
f233756d22 | ||
|
|
89fe45355b | ||
|
|
c3f631d4e3 | ||
|
|
730b5b8e08 | ||
|
|
79e7526b1e | ||
|
|
a6d0338ad2 | ||
|
|
d931f9d764 | ||
|
|
52251b77c2 | ||
|
|
e21db96d50 | ||
|
|
81ef71c38c | ||
|
|
a7eae82e41 | ||
|
|
38fe712728 | ||
|
|
09b41a8dc9 | ||
|
|
e0c6e74bf2 | ||
|
|
7b9ed2c3de | ||
|
|
fde3f398d9 | ||
|
|
2308736a2a | ||
|
|
14d0723872 | ||
|
|
8ae0ffb951 | ||
|
|
5cdcaedb8e | ||
|
|
ba1010d7c3 | ||
|
|
2acb7983ab | ||
|
|
d292ec7fe4 | ||
|
|
126d8c9168 | ||
|
|
cf4c84c22f | ||
|
|
c55a315515 | ||
|
|
9633ded6f4 | ||
|
|
0fd1ea2e92 | ||
|
|
5d28ed551a | ||
|
|
4144dc414c | ||
|
|
9ae91aca3c | ||
|
|
fe920c8988 | ||
|
|
47ac158a42 | ||
|
|
68fb182bf4 | ||
|
|
3c7566a565 | ||
|
|
3389475d33 | ||
|
|
debd8c1075 | ||
|
|
a7af3ea99a | ||
|
|
0537687b96 | ||
|
|
3203e32f4a | ||
|
|
aa96dbd625 | ||
|
|
58e1f72d72 | ||
|
|
ad24534b94 | ||
|
|
13b4c485e5 | ||
|
|
0ccd384a73 | ||
|
|
1e93309385 | ||
|
|
daff390fc6 | ||
|
|
c17480faf1 | ||
|
|
5b309b311f | ||
|
|
83d8eac9cc | ||
|
|
ed5b2fa509 | ||
|
|
876e3c4e18 | ||
|
|
d7f9813443 | ||
|
|
b94bc58a7f | ||
|
|
adbb078b68 | ||
|
|
e18cc694fc | ||
|
|
1ef6f59c3e | ||
|
|
c86a750258 | ||
|
|
f2b02531b7 | ||
|
|
066c78292e | ||
|
|
79882b5fb6 | ||
|
|
8ea3684250 | ||
|
|
cb97fc569f | ||
|
|
120b7e23ae | ||
|
|
31e5faaff5 | ||
|
|
f4af6df5a0 | ||
|
|
c189b035af | ||
|
|
7ff875a33e | ||
|
|
8c568dfcb0 | ||
|
|
a648cf8808 | ||
|
|
62dff26092 | ||
|
|
c9e6d88bf1 | ||
|
|
0e94bbac42 | ||
|
|
525ba1c2f4 | ||
|
|
d006d0a1bc | ||
|
|
d53c77d4a5 | ||
|
|
a2aaa8f778 | ||
|
|
8964ba6313 | ||
|
|
a9ce596e07 | ||
|
|
7598346ff5 | ||
|
|
4be6d19695 | ||
|
|
45b5b98612 | ||
|
|
97e9572f42 | ||
|
|
0fd356dbf0 | ||
|
|
346f31f8c4 | ||
|
|
b22259b542 | ||
|
|
d78411a63b | ||
|
|
6a1c16b2cb | ||
|
|
9ff44e87a6 | ||
|
|
a6341981d8 | ||
|
|
66e5e2f117 | ||
|
|
69d842c400 | ||
|
|
9e343a8f20 | ||
|
|
8211ae6d85 | ||
|
|
f51b5f3bbf | ||
|
|
294cb9ea47 | ||
|
|
929f822ca5 | ||
|
|
b7fe32cf07 | ||
|
|
5692fabbdf | ||
|
|
3fcccf3b47 | ||
|
|
f7093bf82f | ||
|
|
914471fa31 | ||
|
|
895b9a71c3 | ||
|
|
ea50c77970 | ||
|
|
c18b7f4169 | ||
|
|
b0380bac34 | ||
|
|
0d7185e0de | ||
|
|
237da90a36 | ||
|
|
6251f7c615 | ||
|
|
176433a56e |
124
AGENTS.md
|
|
@ -7,6 +7,11 @@
|
|||
> studio websocket bridge for all agent work. Full spec: [App Remote Control](#app-remote-control---remote).
|
||||
|
||||
## Execution Policy
|
||||
|
||||
- **Designs stay local.** Design documents, plans, and reports are local files
|
||||
(`local/agent_state/<topic>/DESIGN.md`) that lanes read from disk. Never
|
||||
publish them to the web (no Artifacts, no hosted pages); summarize in the
|
||||
terminal instead.
|
||||
- Launch UI programs as standalone release binaries from this checkout. Do
|
||||
not use the Studio remote bridge, `ObserveMount`, `RunItem`, or any
|
||||
`cargo-makepad studio` websocket client.
|
||||
|
|
@ -31,8 +36,56 @@
|
|||
can be run directly in the shell.
|
||||
- A standalone app's built-in screenshot/capture hook is valid for visual
|
||||
inspection.
|
||||
- **System-level screenshots are FORBIDDEN.** Never run `screencapture`,
|
||||
`CGWindowListCreateImage`/`CGDisplayCreateImage` scripts, `xcap`,
|
||||
`import`, `scrot`, `grim`, `xwd`, PowerShell/Win32 screen grabs, or any
|
||||
other OS screen capture — not of the display, not of a window, not
|
||||
"just the caption". The user's screen is private. The only image of a
|
||||
running app you may ever take is the app's own `--remote` grab (`/g`,
|
||||
`/gq`, `/tweak/grab`), which renders the app's own drawable and nothing
|
||||
else. If something only shows in the OS layer (native caption buttons,
|
||||
other apps, the desktop), ask the user for a screenshot instead of
|
||||
taking one.
|
||||
- When adding a new example crate, update both the Cargo workspace and
|
||||
`makepad.splash`.
|
||||
- **Zero locking on the UI thread, one mechanism everywhere.** The UI
|
||||
thread never takes a `Mutex`/`RwLock`/`Condvar` that another thread can
|
||||
hold, and never blocks on a channel. UI → workers/audio is commands over
|
||||
a channel (bounded, non-blocking send; a full queue is reported and
|
||||
retried next frame). Workers/audio → UI is snapshots over atomics, a
|
||||
triple buffer, or a channel read with `try_recv`. Large payloads (PCM,
|
||||
stems, grids, images) travel as `Arc` through the channel; a replaced
|
||||
payload is handed back so the UI thread does the drop, never a realtime
|
||||
thread. A realtime callback (audio) owns its state, never takes a lock
|
||||
the UI can hold, never allocates on the hot path. This is ONE code path
|
||||
for native and wasm — no `cfg` fork where desktop keeps shared mutexes.
|
||||
On wasm both the browser UI thread and the AudioWorklet thread abort on
|
||||
`Atomics.wait`, and a spinning fallback against a busy audio callback is
|
||||
a 100 % CPU feedback loop that kills audio and frame rate together (DJ
|
||||
web, 2026-09-03). `lock_from_ui` is only acceptable on state provably
|
||||
touched by the UI thread alone.
|
||||
- **Standard operating flow — who does what.** The main session (Fable)
|
||||
designs, briefs, manages and reviews; it does not write the code itself
|
||||
except one-line fixes. **Codex writes the code**: every implementation lane
|
||||
is a Codex lane with a precise brief (observations, files, rules, the
|
||||
verification commands) launched through `local/tools/delegate` /
|
||||
`local/agent_state/webdemos/tasks/queue.sh` and landed through
|
||||
`local/tools/integrate`. **Grok does the tests and the token-heavy work**:
|
||||
test suites, audits, surveys, log reading, conflict resolution passes,
|
||||
reviews of large diffs (`delegate grok` / `research-grok` / `review-grok`).
|
||||
A Fable subagent is the exception, only for a design-level change the
|
||||
other two cannot carry (a new platform mechanism), and it stops as soon as
|
||||
the API is fixed so Codex can do the conversions. Keep at most six lanes
|
||||
per provider; land everything through the integrator; the user tries the
|
||||
result — no routine captures.
|
||||
- **No temporary threads — use the pool.** Never spawn a thread for one
|
||||
job (`std::thread::spawn` is unsupported on wasm anyway; the platform
|
||||
spawner works everywhere). Background work goes to the platform thread
|
||||
pool (`cx.thread_spawner()` / the pool `TaskHandle` API) or to a
|
||||
long-lived worker created once at start-up and fed over a channel. On
|
||||
the web a Web Worker takes hundreds of milliseconds to come up, so a
|
||||
per-job thread is a stall; on desktop it is still churn. One mechanism
|
||||
on both targets.
|
||||
|
||||
## Standalone Launch
|
||||
1. `cargo build --release -p <package>` from this checkout.
|
||||
|
|
@ -44,6 +97,22 @@
|
|||
|
||||
## App Remote Control (`--remote`)
|
||||
|
||||
> **Focus law.** A `--remote` app opens its window VISIBLE BUT UNFOCUSED and
|
||||
> stays that way: it never activates, never becomes key, and bridge clicks
|
||||
> never raise it. The user keeps typing wherever they were. Everything the
|
||||
> bridge does (grabs, `/m`, `/k`, `/t`, `/snap`) works without focus because
|
||||
> input is injected through the app's event loop, not the OS. Do not work
|
||||
> around this (`MAKEPAD_FOCUS=1` exists only for a run the user asks to see
|
||||
> in front); `MAKEPAD_NO_FOCUS=1` gives a non-remote launch the same manners.
|
||||
|
||||
> **Who may open a visible window.** Subagent/lane verification runs HIDDEN:
|
||||
> launch with `MAKEPAD_HIDE_WINDOWS=1 <bin> --remote` — the window never
|
||||
> appears, grabs (`/g`), `/snap`, `/m`, `/k`, `/t` all still work offscreen.
|
||||
> Only the integrating session opens the one visible, unfocused window the
|
||||
> user watches; several look-alike windows on screen made the user "go
|
||||
> insane" (2026-08-26).
|
||||
|
||||
|
||||
Any makepad app launched with `--remote` runs a localhost HTTP server inside
|
||||
the process and prints one line before the UI appears:
|
||||
|
||||
|
|
@ -111,6 +180,8 @@ this pattern as an executable end-to-end test across three example apps.
|
|||
`GET /gq` (or `/close` each window, then `/quit`). Never leave test windows
|
||||
on the user's screen, and never `pkill` when the protocol is available.
|
||||
- **Never touch an instance the user is running.** Launch your own.
|
||||
- **`/g` is the only camera.** No OS-level screen capture of any kind (see
|
||||
Execution Policy) — the remote grab is what you get.
|
||||
- **A vanished window or app with `[makepad-remote] user closed …` in the log
|
||||
means the human dismissed it — it was in their way.** Do **not** treat that
|
||||
as a crash and do **not** relaunch it. The app prints
|
||||
|
|
@ -151,6 +222,59 @@ this pattern as an executable end-to-end test across three example apps.
|
|||
- **Cost when idle is zero.** The event loop only upshifts its paint clock
|
||||
while a remote request is in flight.
|
||||
|
||||
### The TWEAKER (`/tweak/*`) — design feedback and live styling
|
||||
|
||||
Every `--remote` app carries a design-feedback overlay (plan of record:
|
||||
repo-root `tweaker.md`; implementation: `widgets/src/tweaker.rs`). Off it
|
||||
costs nothing. On, the person (or you) points at the UI: pointer events over
|
||||
the window body are swallowed before widget dispatch — **clicking a Button in
|
||||
tweak mode outlines it and never fires it** — and the window grows a property
|
||||
sidebar next to the (compressed) app UI. Shift+F10 toggles it in-app; every edit,
|
||||
theirs or yours, lands in one shared diff log.
|
||||
|
||||
| Route | Answer | Notes |
|
||||
|---|---|---|
|
||||
| `/tweak` `?on=1\|0&annotate=1\|0` | `{"on":1,"annotate":0}` | toggle the overlay / the freehand draw mode (Alt-drag draws too) |
|
||||
| `/tweak/state` | `{"on":1,"sel":{path,ty,r,band},"props":[{n,v,set}],"hover":…,"diff":[…],"ann":[…]}` | the STRUCTURE feedback: pinned selection, its real reflected properties (`set:1` = explicitly applied), the edit log, annotation strokes with the widget paths they touch |
|
||||
| `/tweak/apply` (POST) | `{"ok":1,"path":…,"changed":[{path,prop,old,new}]}` | body `{"path":"a.b.c","splash":"{padding: Inset{left: 20}}"}` or the one-property shorthand `{"path":…,"prop":"draw_bg.border_radius","value":"8"}`. Evaluates the chunk onto that ONE instance through the ordinary apply machinery (`+:` merge rules intact) and triggers a full relayout. Answers after the next drawn frame |
|
||||
| `/tweak/diff` | `{"diff":[{path,prop,old,new}…]}` | the raw edit log, in order |
|
||||
| `/tweak/clear` | `{"ok":1}` | reset diff + annotations |
|
||||
| `/tweak/final` | `{"final":[…coalesced…],"ann":[…],"drew":0\|1,"png":path?}` | **read this when tweaking is done**: per (path, prop) only the original and final value, churn collapsed. When the user drew, `png` is the composited screenshot — look at it, the strokes mean something |
|
||||
| `/tweak/grab` | like `/g` | the overlay (outlines, strokes, sidebar) draws in the window's own pass, so any grab is already composited |
|
||||
|
||||
`local/tools/tweak` wraps all of this:
|
||||
`tweak PORT on`, `tweak PORT state`, `tweak PORT apply PATH PROP VALUE`,
|
||||
`tweak PORT splash PATH 'CHUNK'`, `tweak PORT final`, …
|
||||
|
||||
**How to listen.** Sidebar edits push to you: each one emits a marked
|
||||
`TWEAK sidebar <path> <prop> <old> -> <new>` line into the app log — the
|
||||
`/log` tail is your ear; you never poll `/tweak/state` for changes. Talk back
|
||||
on `/tweak/apply` (values or whole shader chunks) to the same selected
|
||||
instance.
|
||||
|
||||
**Write-back (you do this part — the overlay never writes source).** When the
|
||||
session is done, take `/tweak/final` and edit the splash source:
|
||||
|
||||
1. Resolve each entry's widget path to its DSL site: the dotted path mirrors
|
||||
the `script_mod!` tree (`/d` shows the same ids). `-` segments are
|
||||
anonymous containers — skip them when searching the source.
|
||||
2. Write each property at the **most specific existing site** — the widget's
|
||||
own `name := Type{…}` block if it has one; create one only when none
|
||||
exists.
|
||||
3. Respect the merge law: a property inside a typed sub-struct goes through
|
||||
`+:` (`draw_bg +: { border_radius: 8 }`), never a replacing
|
||||
`draw_bg: {…}`. Plain walk/layout values (`padding`, `margin`, `width`)
|
||||
are set directly (`padding: Inset{left: 20}`).
|
||||
4. Values come back in source spelling (`#rrggbbaa` colors, plain numbers) —
|
||||
paste them as-is. Mind the Rust-tokenizer hex-`e` trap in `script_mod!`:
|
||||
`#1e1e2e` must be written `#x1e1e2e`.
|
||||
5. Rebuild and relaunch; verify the value survived with `/tweak/state` or
|
||||
`/snap` before calling it done.
|
||||
|
||||
Reflection truth: `props` come from the widget's live Rust fields plus the
|
||||
type's DSL-declared shader inputs (`instance()`/`uniform()`), so the list is
|
||||
what the widget actually exposes — there is no synthetic schema to drift.
|
||||
|
||||
### Studio remote bridge (the older path)
|
||||
|
||||
The studio (`studio/desktop` + `studio/hub`) drives a hosted app over a
|
||||
|
|
|
|||
52
Cargo.toml
|
|
@ -1,17 +1,20 @@
|
|||
workspace.members = [
|
||||
# === app ===
|
||||
"apps/mpbrowser",
|
||||
"apps/mpterm",
|
||||
"apps/mpwm",
|
||||
"apps/browser",
|
||||
"apps/terminal",
|
||||
"apps/wm",
|
||||
"apps/aichat",
|
||||
"apps/finance",
|
||||
"apps/mpsheets",
|
||||
"libs/mp_theme",
|
||||
"libs/mp_wm_api",
|
||||
"apps/mptask",
|
||||
"apps/mpimage",
|
||||
"apps/mpvideo",
|
||||
"apps/mppdf",
|
||||
"apps/mpfiles",
|
||||
"apps/sheets",
|
||||
"apps/photos",
|
||||
"libs/wm_theme",
|
||||
"libs/wm_api",
|
||||
"libs/app_module",
|
||||
"apps/task",
|
||||
"apps/image",
|
||||
"apps/video",
|
||||
"apps/pdf",
|
||||
"apps/files",
|
||||
"apps/route",
|
||||
# === arcade (game.md) — networked AI game sandbox ===
|
||||
"apps/arcade",
|
||||
|
|
@ -27,6 +30,10 @@ workspace.members = [
|
|||
"libs/game/assets",
|
||||
"apps/asset-ui",
|
||||
"apps/asset-server",
|
||||
"apps/flow-server",
|
||||
"apps/flow-ui",
|
||||
"libs/flowgraph",
|
||||
"libs/media_view",
|
||||
"apps/ai-hub",
|
||||
"apps/vj",
|
||||
"apps/mixer",
|
||||
|
|
@ -40,6 +47,8 @@ workspace.members = [
|
|||
"libs/asset/store",
|
||||
"libs/asset/chat",
|
||||
"libs/chat_ui",
|
||||
"libs/ai/hub_ui",
|
||||
"libs/ai/services",
|
||||
"libs/asset/annotate",
|
||||
"libs/render",
|
||||
"libs/raytrace",
|
||||
|
|
@ -50,6 +59,8 @@ workspace.members = [
|
|||
"libs/sim/math",
|
||||
"libs/show_control",
|
||||
"libs/asset/creator",
|
||||
"libs/bounded_http",
|
||||
"libs/flow",
|
||||
"libs/strict_json",
|
||||
# === remesh (FaithC port) / xatlas (jpcy port) ===
|
||||
"libs/remesh",
|
||||
|
|
@ -94,6 +105,9 @@ workspace.members = [
|
|||
"examples/render_to_texture",
|
||||
# === digital-fabrication product ===
|
||||
"apps/fab",
|
||||
"apps/fabric",
|
||||
"libs/fabric/measure",
|
||||
"libs/fabric/draft",
|
||||
# === xr app ===
|
||||
"xr",
|
||||
# === studio ===
|
||||
|
|
@ -106,6 +120,8 @@ workspace.members = [
|
|||
"libs/score_layout",
|
||||
"libs/score_play",
|
||||
"libs/score_render",
|
||||
"libs/score_view",
|
||||
"libs/score_view/tests/embed_app",
|
||||
"libs/score_ai",
|
||||
"libs/score_import",
|
||||
"libs/score_pdf",
|
||||
|
|
@ -114,6 +130,8 @@ workspace.members = [
|
|||
"libs/midi_file",
|
||||
"libs/soundfont",
|
||||
"libs/piano_model",
|
||||
"libs/drumkit",
|
||||
"libs/drumkit_phys",
|
||||
"libs/musicxml",
|
||||
# === own MP3 / Ogg Vorbis decoders ===
|
||||
"libs/audio_decode",
|
||||
|
|
@ -123,6 +141,7 @@ workspace.members = [
|
|||
"libs/audio_lyrics",
|
||||
# === stems + lyrics side-channel bake/publish (asset-ui + VJ) ===
|
||||
"libs/audio_sidechannels",
|
||||
"libs/vj_analysis",
|
||||
"libs/teamtalk",
|
||||
# === pictures of audio (spectrogram, wave strip, composite) ===
|
||||
"libs/audio_picture",
|
||||
|
|
@ -135,16 +154,23 @@ workspace.members = [
|
|||
"libs/frametween",
|
||||
# === archive.org search + download content input (VJ / asset-ui) ===
|
||||
"libs/archive_org",
|
||||
# === image tile wall: HEVC tape atlases + baker CLI + TileGrid widget
|
||||
# (the engine extracted from the Source Library picture wall) ===
|
||||
"libs/image_tiles",
|
||||
"examples/image_tiles",
|
||||
# === mp4 sample index for range-streaming playback ===
|
||||
"libs/mp4_index",
|
||||
# === necessary tools ===
|
||||
"platform/video",
|
||||
"tools/cargo_makepad",
|
||||
"tools/makepad_loader",
|
||||
# === OSM PBF -> tile archive + nav artifact bake passes (CLI + in-app) ===
|
||||
"libs/map_build",
|
||||
"tools/map_tiles",
|
||||
"tools/map_bake",
|
||||
"tools/remote",
|
||||
"tools/dj_pack",
|
||||
"libs/system_speech",
|
||||
# === tests ===
|
||||
"platform/script/test",
|
||||
]
|
||||
|
|
@ -163,7 +189,6 @@ workspace.exclude = [
|
|||
"libs/diffusion",
|
||||
# the AI model workspace (loader + cuda/metal stores + model crates) — aiarch.md
|
||||
"libs/ai",
|
||||
"libs/voice",
|
||||
"widgets/test",
|
||||
"libs/stitch",
|
||||
"libs/wasm_bridge/test",
|
||||
|
|
@ -280,6 +305,9 @@ strip = true
|
|||
inherits = "release"
|
||||
debug = true
|
||||
|
||||
[profile.test.package.makepad-stitch]
|
||||
opt-level = 1
|
||||
|
||||
#[profile.dev.package.makepad-live-tokenizer]
|
||||
#opt-level = 3
|
||||
#[profile.dev.package.makepad-live-compiler]
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ makepad-ai-hub = { path = "../../libs/ai/hub", default-features = false }
|
|||
# Feature passthrough: the headless node exposes exactly the library's
|
||||
# feature set, so box launch scripts keep their --features flags working.
|
||||
[features]
|
||||
default = ["flux", "paint", "paint-cuda", "llm", "tts", "indextts", "video", "interpolate", "audio", "mesh", "matte-native", "depth-native", "segment-native", "upscale-native", "motion-native", "rig-native", "splat-native"]
|
||||
default = ["flux", "paint", "paint-cuda", "llm", "tts", "indextts", "video", "interpolate", "audio", "mesh", "matte-native", "depth-native", "segment-native", "body-native", "upscale-native", "motion-native", "rig-native", "splat-native", "stems-native"]
|
||||
python-backends = ["makepad-ai-hub/python-backends"]
|
||||
flux = ["makepad-ai-hub/flux"]
|
||||
paint = ["makepad-ai-hub/paint"]
|
||||
|
|
@ -29,7 +29,9 @@ mesh = ["makepad-ai-hub/mesh"]
|
|||
matte-native = ["makepad-ai-hub/matte-native"]
|
||||
depth-native = ["makepad-ai-hub/depth-native"]
|
||||
segment-native = ["makepad-ai-hub/segment-native"]
|
||||
body-native = ["makepad-ai-hub/body-native"]
|
||||
upscale-native = ["makepad-ai-hub/upscale-native"]
|
||||
motion-native = ["makepad-ai-hub/motion-native"]
|
||||
splat-native = ["makepad-ai-hub/splat-native"]
|
||||
rig-native = ["makepad-ai-hub/rig-native"]
|
||||
stems-native = ["makepad-ai-hub/stems-native"]
|
||||
|
|
|
|||
|
|
@ -36,10 +36,14 @@ fn run() -> Result<(), AssetAiError> {
|
|||
let mut cache_dir: Option<PathBuf> = None;
|
||||
let mut registry_path: Option<PathBuf> = None;
|
||||
let mut machine = false;
|
||||
let mut activity_probe = None;
|
||||
|
||||
let mut args = std::env::args().skip(1);
|
||||
while let Some(arg) = args.next() {
|
||||
match arg.as_str() {
|
||||
"--activity-probe" => {
|
||||
activity_probe = Some(args.next().ok_or_else(|| AssetAiError::Io("--activity-probe needs seconds (1..3600)".into()))?.parse::<u64>().map_err(|_| AssetAiError::Io("invalid activity probe seconds".into()))?);
|
||||
}
|
||||
"--port" => {
|
||||
let value = args
|
||||
.next()
|
||||
|
|
@ -78,7 +82,7 @@ fn run() -> Result<(), AssetAiError> {
|
|||
}
|
||||
"--help" | "-h" => {
|
||||
println!(
|
||||
"{SERVICE_NAME} {SERVICE_VERSION}\nusage: {SERVICE_NAME} [--port N] [--host ADDR] [--fleet NAME] [--cache-dir PATH] [--registry PATH] [--machine]"
|
||||
"{SERVICE_NAME} {SERVICE_VERSION}\nusage: {SERVICE_NAME} [--port N] [--host ADDR] [--fleet NAME] [--cache-dir PATH] [--registry PATH] [--machine] [--activity-probe SECONDS]"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
|
@ -88,6 +92,10 @@ fn run() -> Result<(), AssetAiError> {
|
|||
}
|
||||
}
|
||||
|
||||
if let Some(seconds) = activity_probe {
|
||||
return makepad_ai_hub::activity::run_probe(seconds);
|
||||
}
|
||||
|
||||
let port = match port {
|
||||
Some(port) => port,
|
||||
None => match std::env::var("MAKEPAD_ASSET_AI_PORT") {
|
||||
|
|
|
|||
46
apps/aichat/Cargo.toml
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
# aichat — the assistant, as an app.
|
||||
#
|
||||
# One conversation that drives every application through the AI services
|
||||
# layer. It is hosted three ways with one code base: under the window
|
||||
# manager as a special child seated in the pane slot (this binary,
|
||||
# `--stdin-loop`), standalone as its own window (this binary), and inside
|
||||
# any app's Window as the F10 overlay or inside the mobile/web superbuild
|
||||
# (this crate's lib, `makepad_aichat::script_mod` + `AiChatPanel{}`).
|
||||
#
|
||||
# The panel widget OWNS the engine: the service registry, the model, the
|
||||
# transcript. Hosts only feed it links (in-process) or bus frames (the
|
||||
# window manager's studio Custom frames) and give it a place to draw.
|
||||
#
|
||||
# Plan of record: repo-root aicontrol.md.
|
||||
|
||||
[package]
|
||||
name = "makepad-aichat"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
default-run = "aichat"
|
||||
|
||||
[lib]
|
||||
name = "makepad_aichat"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "aichat"
|
||||
path = "src/main.rs"
|
||||
|
||||
[features]
|
||||
default = ["engine"]
|
||||
# The real models (the hub's local runtime, the cloud providers). Off for a
|
||||
# build without a model runtime — the web page answers with NoModel and the
|
||||
# tool console still works.
|
||||
engine = ["makepad-ai-services/engine", "dep:makepad-asset-creator"]
|
||||
|
||||
[dependencies]
|
||||
makepad-widgets = { path = "../../widgets" }
|
||||
makepad-wm-theme = { path = "../../libs/wm_theme" }
|
||||
makepad-wm-api = { path = "../../libs/wm_api" }
|
||||
makepad-ai-services = { path = "../../libs/ai/services", default-features = false }
|
||||
makepad-strict-json = { path = "../../libs/strict_json" }
|
||||
# The generative pipelines behind the assistant's own `gen` service
|
||||
# (src/gen.rs): the creator runner picks a fleet node and brings the
|
||||
# picture back. Native only, with the engine.
|
||||
makepad-asset-creator = { path = "../../libs/asset/creator", optional = true }
|
||||
213
apps/aichat/src/bus.rs
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
//! The client half of the window manager's service bus.
|
||||
//!
|
||||
//! Under the WM the other apps are not in this process. The WM forwards
|
||||
//! their up-frames to the aichat child as studio `Custom` frames, each
|
||||
//! stamped with the endpoint the WM issued to the sender, and forwards
|
||||
//! the aichat child's down-frames (which name their target endpoint) back
|
||||
//! to the right client. This adapter turns those frames into ordinary
|
||||
//! [`ServiceLink`]s in the panel's registry, so the engine never knows
|
||||
//! whether a service is a channel away or a process away.
|
||||
//!
|
||||
//! One link per endpoint. A `Register` from an endpoint the registry does
|
||||
//! not know creates the link and registers it under the WM's endpoint id
|
||||
//! (`register_as`); a later `Register` from the same endpoint is just the
|
||||
//! manifest going down the existing link, where the registry answers it.
|
||||
//! The WM tells us about a dead client by sending `Unregister` on its
|
||||
//! behalf. Everything the registry sends down a bus link is drained here
|
||||
//! and put on the wire to the WM.
|
||||
|
||||
use makepad_ai_services::engine::ServiceRegistry;
|
||||
use makepad_ai_services::port::{ServiceLink, ServiceLinkHost};
|
||||
use makepad_ai_services::wire::*;
|
||||
use makepad_widgets::makepad_platform::studio::AppToStudio;
|
||||
use makepad_widgets::*;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct ServiceBus {
|
||||
hosts: HashMap<EndpointId, ServiceLinkHost>,
|
||||
}
|
||||
|
||||
impl ServiceBus {
|
||||
/// An up-frame from the WM. `None` for frames that are not the bus's.
|
||||
pub fn on_custom(&mut self, registry: &ServiceRegistry, json: &str) -> bool {
|
||||
let Some(frame) = HostedUp::parse(json) else { return false };
|
||||
let Some(from) = frame.from.clone() else { return true };
|
||||
match (&frame.msg, self.hosts.get(&from)) {
|
||||
(ServiceUp::Register { manifest, .. }, None) => {
|
||||
let (link, host) = ServiceLink::pair(manifest.clone());
|
||||
if registry.register_as(link, from.clone(), "", None).is_ok() {
|
||||
let _ = host.up.send(frame);
|
||||
self.hosts.insert(from, host);
|
||||
}
|
||||
}
|
||||
(_, Some(host)) => {
|
||||
let _ = host.up.send(frame);
|
||||
}
|
||||
(_, None) => {}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Put every frame the registry sent down a bus link on the wire.
|
||||
/// Links whose registry side is gone are dropped.
|
||||
pub fn relay_down(&mut self, registry: &ServiceRegistry) {
|
||||
for frame in self.drain_down(registry) {
|
||||
Cx::send_studio_message(AppToStudio::Custom(frame.to_json()));
|
||||
}
|
||||
}
|
||||
|
||||
fn drain_down(&mut self, registry: &ServiceRegistry) -> Vec<HostedDown> {
|
||||
let mut gone: Vec<EndpointId> = Vec::new();
|
||||
let mut frames = Vec::new();
|
||||
for (endpoint, host) in &self.hosts {
|
||||
loop {
|
||||
match host.down.try_recv() {
|
||||
Ok(mut frame) => {
|
||||
// `Registered` travels without a target; the WM
|
||||
// routes it by the endpoint we register as.
|
||||
if frame.to.is_none() {
|
||||
frame.to = Some(endpoint.clone());
|
||||
}
|
||||
frames.push(frame);
|
||||
}
|
||||
Err(std::sync::mpsc::TryRecvError::Empty) => break,
|
||||
Err(std::sync::mpsc::TryRecvError::Disconnected) => {
|
||||
gone.push(endpoint.clone());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for endpoint in gone {
|
||||
self.hosts.remove(&endpoint);
|
||||
registry.unregister(&endpoint);
|
||||
}
|
||||
frames
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.hosts.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.hosts.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use makepad_ai_services::engine::{EngineCore, NoModelWithReason, RegistryUp};
|
||||
|
||||
#[test]
|
||||
fn hosted_subscriptions_and_messages_cross_the_adapter_unchanged() {
|
||||
let registry = ServiceRegistry::new();
|
||||
let mut bus = ServiceBus::default();
|
||||
let endpoint = EndpointId("w4".into());
|
||||
let manifest = ServiceManifest::new("flow", "Flow", "A flow.")
|
||||
.with_topic(TopicDef::new("run", "Run events."));
|
||||
let register = HostedUp {
|
||||
from: Some(endpoint.clone()),
|
||||
msg: ServiceUp::Register { manifest, port_tag: 4 },
|
||||
};
|
||||
assert!(bus.on_custom(®istry, ®ister.to_json()));
|
||||
registry.pump();
|
||||
let host = bus.hosts.get(&endpoint).unwrap();
|
||||
let _registered = host.down.try_recv().unwrap();
|
||||
assert!(registry.send(
|
||||
&endpoint,
|
||||
ServiceDown::Subscribe {
|
||||
sub_id: "s1".into(),
|
||||
topic: "run".into(),
|
||||
filter: None,
|
||||
},
|
||||
));
|
||||
assert!(matches!(
|
||||
host.down.try_recv().unwrap().msg,
|
||||
ServiceDown::Subscribe { sub_id, topic, filter: None }
|
||||
if sub_id == "s1" && topic == "run"
|
||||
));
|
||||
let message = HostedUp {
|
||||
from: Some(endpoint.clone()),
|
||||
msg: ServiceUp::Message {
|
||||
sub_id: "s1".into(),
|
||||
topic: "run".into(),
|
||||
text: "finished".into(),
|
||||
data: None,
|
||||
final_: true,
|
||||
},
|
||||
};
|
||||
assert!(bus.on_custom(®istry, &message.to_json()));
|
||||
assert!(matches!(
|
||||
registry.pump().as_slice(),
|
||||
[RegistryUp::Message { endpoint: from, sub_id, message }]
|
||||
if from == &endpoint && sub_id == "s1" && message.final_
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_shutdown_is_flushed_through_the_hosted_service_bus() {
|
||||
let registry = ServiceRegistry::new();
|
||||
let mut bus = ServiceBus::default();
|
||||
let endpoint = EndpointId("w4".into());
|
||||
let manifest = ServiceManifest::new("flow", "Flow", "A flow.")
|
||||
.with_tool(ToolDef::new(
|
||||
"watch",
|
||||
"Watch a run.",
|
||||
r#"{"type":"object","properties":{}}"#,
|
||||
Risk::Read,
|
||||
))
|
||||
.with_topic(TopicDef::new("run", "Run events."));
|
||||
assert!(bus.on_custom(
|
||||
®istry,
|
||||
&HostedUp {
|
||||
from: Some(endpoint.clone()),
|
||||
msg: ServiceUp::Register { manifest, port_tag: 4 },
|
||||
}
|
||||
.to_json(),
|
||||
));
|
||||
let mut engine = EngineCore::new(
|
||||
registry.clone(),
|
||||
Box::new(NoModelWithReason::new("not used by the tool console")),
|
||||
None,
|
||||
0x44,
|
||||
);
|
||||
engine.send("/flow.watch {}", 0.0);
|
||||
let call_id = bus
|
||||
.drain_down(®istry)
|
||||
.into_iter()
|
||||
.find_map(|frame| match frame.msg {
|
||||
ServiceDown::Call(call) => Some(call.call_id),
|
||||
_ => None,
|
||||
})
|
||||
.expect("the hosted service receives the call");
|
||||
assert!(bus.on_custom(
|
||||
®istry,
|
||||
&HostedUp {
|
||||
from: Some(endpoint.clone()),
|
||||
msg: ServiceUp::Result(
|
||||
ToolResult::ok(call_id, "watching", "")
|
||||
.with_subscription(SubscriptionRequest::new("run")),
|
||||
),
|
||||
}
|
||||
.to_json(),
|
||||
));
|
||||
engine.pump(0.1);
|
||||
let sub_id = bus
|
||||
.drain_down(®istry)
|
||||
.into_iter()
|
||||
.find_map(|frame| match frame.msg {
|
||||
ServiceDown::Subscribe { sub_id, .. } => Some(sub_id),
|
||||
_ => None,
|
||||
})
|
||||
.expect("the hosted service receives the subscription");
|
||||
assert_eq!(sub_id, "l44-s1");
|
||||
engine.shutdown();
|
||||
assert!(matches!(
|
||||
bus.drain_down(®istry).as_slice(),
|
||||
[HostedDown { to: Some(to), msg: ServiceDown::Unsubscribe { sub_id: ended } }]
|
||||
if to == &endpoint && ended == &sub_id
|
||||
));
|
||||
}
|
||||
}
|
||||
297
apps/aichat/src/gen.rs
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
//! The assistant's own generative service: `gen.image{prompt}`.
|
||||
//!
|
||||
//! The hub's pipelines are not an app, so no app registers them; the
|
||||
//! panel registers this service in-process beside whatever apps join.
|
||||
//! An `image` call runs the creator pipeline on a worker thread (the
|
||||
//! runner is blocking: node pick over the LAN fleet, request, poll,
|
||||
//! fetch), streams the node's progress into the card, writes the picture
|
||||
//! under the makepad home's `gen` folder and answers with the path — the
|
||||
//! model then hands that path to `photos.add`, which puts it on the wall.
|
||||
//! Nothing goes through the asset store. Without the `engine` feature (the
|
||||
//! web page) the service still exists and says it cannot.
|
||||
|
||||
use makepad_ai_services::engine::ServiceRegistry;
|
||||
use makepad_ai_services::port::{AiServicePort, PortEvent};
|
||||
use makepad_ai_services::wire::{Risk, ServiceCall, ServiceManifest, ToolDef, ToolResult};
|
||||
use makepad_widgets::*;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::mpsc::{Receiver, TryRecvError};
|
||||
use std::sync::Arc;
|
||||
|
||||
/// The service id on the bus.
|
||||
pub const SERVICE_ID: &str = "gen";
|
||||
|
||||
pub fn manifest() -> ServiceManifest {
|
||||
ServiceManifest::new(
|
||||
SERVICE_ID,
|
||||
"Generate",
|
||||
"The machine's generative pipelines, on the fleet's GPU nodes: a \
|
||||
picture from a prompt. The picture is saved on this machine under \
|
||||
the makepad home's gen folder and the answer carries its path. To \
|
||||
SHOW it, call photos.add with that path (launch Photos first with \
|
||||
os.launch if it is not running) — the wall then glides onto it. A \
|
||||
generation takes half a minute on a warm node, longer when a node \
|
||||
must load the model.",
|
||||
)
|
||||
.with_tool(ToolDef::new(
|
||||
"image",
|
||||
"Generate one picture from a text prompt on a fleet image node; saves it under the makepad home's gen folder and returns the path.",
|
||||
r#"{"type":"object","properties":{"prompt":{"type":"string","description":"what the picture shows, in plain words"},"width":{"type":"integer","description":"pixels, optional (default 1024)"},"height":{"type":"integer","description":"pixels, optional (default 1024)"}},"required":["prompt"]}"#,
|
||||
Risk::Act,
|
||||
))
|
||||
}
|
||||
|
||||
/// What the worker reports back.
|
||||
enum GenMsg {
|
||||
Progress(String, u16),
|
||||
Done(Result<GenDone, String>),
|
||||
}
|
||||
|
||||
struct GenDone {
|
||||
path: PathBuf,
|
||||
node: String,
|
||||
}
|
||||
|
||||
struct Job {
|
||||
call_id: String,
|
||||
cancel: Arc<AtomicBool>,
|
||||
rx: Receiver<GenMsg>,
|
||||
}
|
||||
|
||||
/// The in-process port plus the jobs in flight.
|
||||
pub struct GenService {
|
||||
port: AiServicePort,
|
||||
jobs: Vec<Job>,
|
||||
}
|
||||
|
||||
impl GenService {
|
||||
/// Open the service and register it in the panel's registry. `None`
|
||||
/// only when the manifest does not validate (a programming error).
|
||||
pub fn open(registry: &ServiceRegistry) -> Option<GenService> {
|
||||
let (port, link) = AiServicePort::in_process(manifest()).ok()?;
|
||||
registry.register(link, "built in", None).ok()?;
|
||||
Some(GenService { port, jobs: Vec::new() })
|
||||
}
|
||||
|
||||
/// Drain the port and the workers; called on every panel event.
|
||||
pub fn handle_event(&mut self, cx: &mut Cx, event: &Event) {
|
||||
for ev in self.port.handle_event(cx, event) {
|
||||
match ev {
|
||||
PortEvent::Call(call) => self.start(call),
|
||||
PortEvent::Cancel { call_id } => {
|
||||
if let Some(job) = self.jobs.iter().find(|j| j.call_id == call_id) {
|
||||
job.cancel.store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
PortEvent::Registered(_)
|
||||
| PortEvent::ChatOpen { .. }
|
||||
| PortEvent::Subscribe { .. }
|
||||
| PortEvent::Unsubscribe { .. } => {}
|
||||
}
|
||||
}
|
||||
self.poll();
|
||||
}
|
||||
|
||||
fn start(&mut self, call: ServiceCall) {
|
||||
let id = call.call_id.clone();
|
||||
if call.tool != "image" {
|
||||
self.port.reply(ToolResult::refused(&id, format!("gen has no tool `{}`; it has image", call.tool)));
|
||||
return;
|
||||
}
|
||||
let args = match parse_image_args(&call.args) {
|
||||
Ok(a) => a,
|
||||
Err(why) => {
|
||||
self.port.reply(ToolResult::refused(&id, why));
|
||||
return;
|
||||
}
|
||||
};
|
||||
#[cfg(not(feature = "engine"))]
|
||||
{
|
||||
let _ = args;
|
||||
self.port.reply(ToolResult::unavailable(&id, "this build has no pipeline runtime; pictures need the native app"));
|
||||
}
|
||||
#[cfg(feature = "engine")]
|
||||
{
|
||||
use makepad_widgets::makepad_platform::thread::SignalToUI;
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let worker_cancel = cancel.clone();
|
||||
let spawned = std::thread::Builder::new().name("gen-image".into()).spawn(move || {
|
||||
let progress_tx = tx.clone();
|
||||
let mut progress = |note: &str, permille: u16| {
|
||||
let _ = progress_tx.send(GenMsg::Progress(note.to_string(), permille));
|
||||
SignalToUI::set_ui_signal();
|
||||
};
|
||||
let result = run_image(&args, &worker_cancel, &mut progress);
|
||||
let _ = tx.send(GenMsg::Done(result));
|
||||
SignalToUI::set_ui_signal();
|
||||
});
|
||||
match spawned {
|
||||
Ok(_) => self.jobs.push(Job { call_id: id, cancel, rx }),
|
||||
Err(e) => self.port.reply(ToolResult::failed(&id, format!("could not start the generation: {e}"))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn poll(&mut self) {
|
||||
let mut done: Vec<usize> = Vec::new();
|
||||
for (i, job) in self.jobs.iter().enumerate() {
|
||||
loop {
|
||||
match job.rx.try_recv() {
|
||||
Ok(GenMsg::Progress(note, permille)) => self.port.progress(&job.call_id, ¬e, permille),
|
||||
Ok(GenMsg::Done(Ok(out))) => {
|
||||
let path = out.path.to_string_lossy().to_string();
|
||||
self.port.reply(
|
||||
ToolResult::ok(
|
||||
&job.call_id,
|
||||
format!("saved {path} (made on {}). Show it on the wall with photos.add {{\"path\":\"{path}\"}}.", out.node),
|
||||
"saved",
|
||||
)
|
||||
.with_data(format!("{{\"path\":{},\"node\":{}}}", json_string(&path), json_string(&out.node))),
|
||||
);
|
||||
done.push(i);
|
||||
break;
|
||||
}
|
||||
Ok(GenMsg::Done(Err(e))) => {
|
||||
let result = if e == "cancelled" { ToolResult::cancelled(&job.call_id) } else { ToolResult::failed(&job.call_id, e) };
|
||||
self.port.reply(result);
|
||||
done.push(i);
|
||||
break;
|
||||
}
|
||||
Err(TryRecvError::Empty) => break,
|
||||
Err(TryRecvError::Disconnected) => {
|
||||
self.port.reply(ToolResult::failed(&job.call_id, "the generation worker died"));
|
||||
done.push(i);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for i in done.into_iter().rev() {
|
||||
self.jobs.remove(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The arguments of one `image` call, checked.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ImageArgs {
|
||||
pub prompt: String,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
}
|
||||
|
||||
/// Sizes are clamped to what the image nodes serve; a prompt is required.
|
||||
pub fn parse_image_args(args: &str) -> Result<ImageArgs, String> {
|
||||
use makepad_strict_json as json;
|
||||
let fields = match json::parse(args.as_bytes()) {
|
||||
Ok(json::Value::Obj(fields)) => fields,
|
||||
_ => return Err("image needs a JSON object with a `prompt`".to_string()),
|
||||
};
|
||||
let get = |key: &str| fields.iter().find(|(k, _)| k == key).map(|(_, v)| v.clone());
|
||||
let prompt = get("prompt").and_then(|v| v.as_str().map(|s| s.trim().to_string())).unwrap_or_default();
|
||||
if prompt.is_empty() {
|
||||
return Err("image needs a `prompt`".to_string());
|
||||
}
|
||||
if prompt.len() > 4000 {
|
||||
return Err("the prompt is longer than 4000 bytes".to_string());
|
||||
}
|
||||
let dim = |key: &str| -> Result<u32, String> {
|
||||
match get(key) {
|
||||
None => Ok(1024),
|
||||
Some(json::Value::Int(i)) if (256..=2048).contains(&i) => Ok(i as u32),
|
||||
Some(_) => Err(format!("`{key}` must be an integer from 256 to 2048")),
|
||||
}
|
||||
};
|
||||
Ok(ImageArgs { prompt, width: dim("width")?, height: dim("height")? })
|
||||
}
|
||||
|
||||
/// A short file-name-safe slug of the prompt.
|
||||
pub fn slug(prompt: &str) -> String {
|
||||
let mut out = String::new();
|
||||
for ch in prompt.chars().flat_map(|c| c.to_lowercase()) {
|
||||
if ch.is_ascii_alphanumeric() {
|
||||
out.push(ch);
|
||||
} else if !out.ends_with('-') && !out.is_empty() {
|
||||
out.push('-');
|
||||
}
|
||||
if out.len() >= 40 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let out = out.trim_matches('-').to_string();
|
||||
if out.is_empty() { "picture".to_string() } else { out }
|
||||
}
|
||||
|
||||
/// The file extension for a content type the nodes produce.
|
||||
pub fn extension_for(content_type: &str) -> &'static str {
|
||||
match content_type.split(';').next().unwrap_or("").trim() {
|
||||
"image/jpeg" => "jpg",
|
||||
"image/webp" => "webp",
|
||||
"image/gif" => "gif",
|
||||
_ => "png",
|
||||
}
|
||||
}
|
||||
|
||||
fn json_string(s: &str) -> String {
|
||||
makepad_strict_json::s(s.to_string()).to_json()
|
||||
}
|
||||
|
||||
#[cfg(feature = "engine")]
|
||||
fn run_image(args: &ImageArgs, cancel: &Arc<AtomicBool>, progress: &mut dyn FnMut(&str, u16)) -> Result<GenDone, String> {
|
||||
use makepad_asset_creator::makepad_ai_hub::home::makepad_home;
|
||||
use makepad_asset_creator::makepad_strict_json as json;
|
||||
use makepad_asset_creator::runner::generate_bytes;
|
||||
let body = json::obj(vec![
|
||||
("prompt", json::s(args.prompt.clone())),
|
||||
("width", json::Value::Int(args.width as i64)),
|
||||
("height", json::Value::Int(args.height as i64)),
|
||||
]);
|
||||
let seed = (Cx::time_now().max(0.0) * 1_000_000_000.0) as u64;
|
||||
progress("finding an image node", 0);
|
||||
let generated = generate_bytes("image.generate", &body, seed, cancel, progress)?;
|
||||
let artifact = generated.artifact.ok_or("the node returned no picture")?;
|
||||
let dir = makepad_home().join("gen");
|
||||
std::fs::create_dir_all(&dir).map_err(|e| format!("cannot make {}: {e}", dir.display()))?;
|
||||
let stamp = Cx::time_now().max(0.0) as u64;
|
||||
let path = dir.join(format!("{stamp}-{}.{}", slug(&args.prompt), extension_for(&artifact.content_type)));
|
||||
std::fs::write(&path, &artifact.bytes).map_err(|e| format!("cannot write {}: {e}", path.display()))?;
|
||||
Ok(GenDone { path, node: generated.node })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_manifest_validates_and_the_tool_acts() {
|
||||
let m = manifest();
|
||||
m.validate().expect("a manifest the wire accepts");
|
||||
assert_eq!(m.id, "gen");
|
||||
assert_eq!(m.tools.len(), 1);
|
||||
assert_eq!(m.tools[0].name, "image");
|
||||
assert_eq!(m.tools[0].risk, Risk::Act);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_args_need_a_prompt_and_sane_sizes() {
|
||||
let a = parse_image_args(r#"{"prompt":"a red bicycle on the moon"}"#).unwrap();
|
||||
assert_eq!((a.width, a.height), (1024, 1024));
|
||||
let b = parse_image_args(r#"{"prompt":"x","width":512,"height":768}"#).unwrap();
|
||||
assert_eq!((b.width, b.height), (512, 768));
|
||||
assert!(parse_image_args(r#"{"width":512}"#).unwrap_err().contains("prompt"));
|
||||
assert!(parse_image_args(r#"{"prompt":"x","width":16}"#).unwrap_err().contains("256"));
|
||||
assert!(parse_image_args("nope").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_names_are_slugs_with_the_right_extension() {
|
||||
assert_eq!(slug("A red bicycle, on the Moon!"), "a-red-bicycle-on-the-moon");
|
||||
assert_eq!(slug(" "), "picture");
|
||||
assert!(slug(&"word ".repeat(30)).len() <= 41);
|
||||
assert_eq!(extension_for("image/png"), "png");
|
||||
assert_eq!(extension_for("image/jpeg; charset=binary"), "jpg");
|
||||
assert_eq!(extension_for("application/octet-stream"), "png");
|
||||
}
|
||||
}
|
||||
89
apps/aichat/src/lib.rs
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
//! aichat as a library: the panel widget that owns the engine, the WM-bus
|
||||
//! client half, the settings, and the module root. A host links this,
|
||||
//! calls [`script_mod`] after the widgets' own, and puts `AiChatPanel{}`
|
||||
//! where the chat goes — the standalone binary and the WM pane child do
|
||||
//! that themselves; a plain `Window` does it for free through its F10
|
||||
//! slot, which instantiates `mod.widgets.AiChatOverlay{}` by name; the
|
||||
//! superbuild seats the same overlay in its pane.
|
||||
//!
|
||||
//! Linking this crate also gives the bridge its `/ai` routes: `script_mod`
|
||||
//! installs `Cx::ai_callback`, the way the widgets crate installs the
|
||||
//! tweaker's, so `/ai?on=1`, `/ai?say=…` and `/ai/transcript` drive and
|
||||
//! read the chat in a hidden instance.
|
||||
|
||||
pub use makepad_widgets;
|
||||
use makepad_widgets::ai_slot::AiSlotRequests;
|
||||
use makepad_widgets::makepad_platform::ScriptVmCx;
|
||||
use makepad_widgets::*;
|
||||
|
||||
pub mod bus;
|
||||
pub mod gen;
|
||||
pub mod overlay;
|
||||
pub mod panel;
|
||||
pub mod settings;
|
||||
|
||||
pub use bus::ServiceBus;
|
||||
pub use overlay::{AiChatOverlay, AiTranscript};
|
||||
pub use panel::{AiChatPanel, AiChatPanelAction};
|
||||
pub use settings::AiSettings;
|
||||
|
||||
/// Register the panel and the overlay, and give the bridge its `/ai`
|
||||
/// routes. Call once after `makepad_widgets::script_mod`.
|
||||
pub fn script_mod(vm: &mut ScriptVm) {
|
||||
crate::panel::script_mod(vm);
|
||||
crate::overlay::script_mod(vm);
|
||||
vm.cx_mut().ai_callback = Some(ai_callback);
|
||||
}
|
||||
|
||||
fn arg<'a>(args: &'a [(String, String)], keys: &[&str]) -> Option<&'a str> {
|
||||
for key in keys {
|
||||
if let Some((_, value)) = args.iter().find(|(k, _)| k == key) {
|
||||
return Some(value.as_str());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// The bridge's `/ai` dispatcher. `toggle` (`on=1|0`, or flip) and `say`
|
||||
/// (`say=TEXT`, opening the overlay if it is closed) are requests the
|
||||
/// slot and the overlay take on their next event; `transcript` is what
|
||||
/// the overlay last published. Never borrows a widget.
|
||||
fn ai_callback(cx: &mut Cx, op: &str, args: &[(String, String)]) -> Result<String, String> {
|
||||
match op {
|
||||
"toggle" => {
|
||||
let is_open = cx.global::<AiSlotRequests>().is_open;
|
||||
let on = match arg(args, &["on"]) {
|
||||
Some(value) => !matches!(value, "0" | "false" | "off" | "no"),
|
||||
None => !is_open,
|
||||
};
|
||||
cx.global::<AiSlotRequests>().open = Some(on);
|
||||
// The slot takes the request on its next event: make one.
|
||||
cx.new_next_frame();
|
||||
cx.redraw_all();
|
||||
Ok(format!("{{\"on\":{}}}", on as u8))
|
||||
}
|
||||
"say" => {
|
||||
let text = arg(args, &["say", "t"]).unwrap_or("").trim().to_string();
|
||||
if text.is_empty() {
|
||||
return Err("need say=TEXT".into());
|
||||
}
|
||||
let req = cx.global::<AiSlotRequests>();
|
||||
if !req.is_open {
|
||||
req.open = Some(true);
|
||||
}
|
||||
req.say.push(text);
|
||||
cx.new_next_frame();
|
||||
cx.redraw_all();
|
||||
Ok("{\"ok\":1}".into())
|
||||
}
|
||||
"transcript" => {
|
||||
let json = cx.global::<AiTranscript>().json.clone();
|
||||
if json.is_empty() {
|
||||
Ok("{\"status\":\"closed\",\"provider\":\"\",\"apps\":[],\"entries\":[],\"generation\":0}".into())
|
||||
} else {
|
||||
Ok(json)
|
||||
}
|
||||
}
|
||||
other => Err(format!("no ai op `{other}`; there are toggle, say, transcript")),
|
||||
}
|
||||
}
|
||||
79
apps/aichat/src/main.rs
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
//! aichat: the assistant as its own window, and as the window manager's
|
||||
//! special child. Both are this binary: standalone it is a window with
|
||||
//! the panel in it and whatever services are linked in-process (none,
|
||||
//! until an app embeds it); under the WM (`--stdin-loop`) the WM seats it
|
||||
//! in the pane slot and every other app's service reaches it over the bus
|
||||
//! as studio `Custom` frames, which the panel turns into registry links.
|
||||
|
||||
pub use makepad_widgets;
|
||||
use makepad_aichat::AiChatPanelAction;
|
||||
use makepad_widgets::*;
|
||||
|
||||
app_main!(App);
|
||||
|
||||
script_mod! {
|
||||
use mod.prelude.widgets.*
|
||||
use mod.widgets.*
|
||||
|
||||
startup() do #(App::script_component(vm)){
|
||||
ui: Root{
|
||||
main_window := Window{
|
||||
window.inner_size: vec2(460, 760)
|
||||
window.title: "AI"
|
||||
pass +: { clear_color: theme.color_bg_app }
|
||||
body +: {
|
||||
panel := AiChatPanel{
|
||||
width: Fill
|
||||
height: Fill
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Script, ScriptHook)]
|
||||
pub struct App {
|
||||
#[live]
|
||||
ui: WidgetRef,
|
||||
}
|
||||
|
||||
impl MatchEvent for App {
|
||||
fn handle_startup(&mut self, cx: &mut Cx) {
|
||||
makepad_wm_api::set_title(cx, "AI");
|
||||
}
|
||||
|
||||
fn handle_actions(&mut self, cx: &mut Cx, actions: &Actions) {
|
||||
for action in actions {
|
||||
if let Some(widget_action) = action.as_widget_action() {
|
||||
if let AiChatPanelAction::Close = widget_action.cast() {
|
||||
// Under the WM the pane hides; standalone the window stays.
|
||||
let _ = makepad_wm_api::send(cx, &makepad_wm_api::WmRequest::Close);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AppMain for App {
|
||||
fn script_mod(vm: &mut ScriptVm) -> ScriptValue {
|
||||
crate::makepad_widgets::script_mod(vm);
|
||||
makepad_wm_theme::apply(vm);
|
||||
makepad_aichat::script_mod(vm);
|
||||
self::script_mod(vm)
|
||||
}
|
||||
|
||||
fn handle_event(&mut self, cx: &mut Cx, event: &Event) {
|
||||
// Bus frames reach the panel through its own handle_event; of the
|
||||
// WM's own frames only the polite close is ours (the pane hides us
|
||||
// by leaving us running; this is the desktop going down).
|
||||
if let Event::Custom(json) = event {
|
||||
if let Some(makepad_wm_api::WmEvent::CloseRequested) = makepad_wm_api::WmEvent::parse(json) {
|
||||
cx.quit();
|
||||
return;
|
||||
}
|
||||
}
|
||||
self.match_event(cx, event);
|
||||
self.ui.handle_event(cx, event, &mut Scope::empty());
|
||||
}
|
||||
}
|
||||
261
apps/aichat/src/overlay.rs
Normal file
|
|
@ -0,0 +1,261 @@
|
|||
//! The chat's module root: what a host seats in-process.
|
||||
//!
|
||||
//! `mod.widgets.AiChatOverlay{}` is what the Window's AI slot
|
||||
//! (`widgets/src/ai_slot.rs`) instantiates by name on F10, and what the
|
||||
//! superbuild seats in its pane. It is a `View` with the panel in it and
|
||||
//! three duties around it: adopt the in-process service links the apps
|
||||
//! parked on `Cx` ([`PendingServiceLinks`]) into the panel's registry,
|
||||
//! send the lines the bridge asked to say ([`AiSlotRequests::say`]),
|
||||
//! and publish the transcript as JSON ([`AiTranscript`]) after each draw
|
||||
//! so `/ai/transcript` answers without touching a widget. The panel's
|
||||
//! Escape (an idle, empty composer) asks the slot to close.
|
||||
|
||||
use crate::panel::{AiChatPanel, AiChatPanelAction};
|
||||
use makepad_ai_services::port::PendingServiceLinks;
|
||||
use makepad_ai_services::state::{Entry, EngineState, Status, ToolStatus};
|
||||
use makepad_widgets::ai_slot::AiSlotRequests;
|
||||
use makepad_widgets::makepad_micro_serde::*;
|
||||
use makepad_widgets::*;
|
||||
|
||||
script_mod! {
|
||||
use mod.prelude.widgets_internal.*
|
||||
use mod.widgets.*
|
||||
|
||||
mod.widgets.AiChatOverlayBase = #(AiChatOverlay::register_widget(vm))
|
||||
mod.widgets.AiChatOverlay = set_type_default() do mod.widgets.AiChatOverlayBase{
|
||||
width: Fill
|
||||
height: Fill
|
||||
panel := AiChatPanel{
|
||||
width: Fill
|
||||
height: Fill
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The transcript as the bridge reads it: a `Cx` global the overlay
|
||||
/// rewrites after every draw of the panel.
|
||||
#[derive(Default)]
|
||||
pub struct AiTranscript {
|
||||
pub json: String,
|
||||
}
|
||||
|
||||
#[derive(SerJson)]
|
||||
struct TranscriptRow {
|
||||
kind: String,
|
||||
text: String,
|
||||
title: String,
|
||||
status: String,
|
||||
note: String,
|
||||
}
|
||||
|
||||
#[derive(SerJson)]
|
||||
struct Transcript {
|
||||
status: String,
|
||||
provider: String,
|
||||
apps: Vec<String>,
|
||||
entries: Vec<TranscriptRow>,
|
||||
generation: u64,
|
||||
}
|
||||
|
||||
/// The transcript JSON for a state: `status`, `provider`, the connected
|
||||
/// apps, and one row per entry with what its card says.
|
||||
pub fn transcript_json(state: &EngineState) -> String {
|
||||
let status = match &state.status {
|
||||
Status::Idle => "idle".to_string(),
|
||||
Status::Loading { phase, fraction } => format!("loading {phase} {:.0}%", fraction * 100.0),
|
||||
Status::Thinking => "thinking".to_string(),
|
||||
Status::Streaming => "streaming".to_string(),
|
||||
Status::WaitingForTool => "waiting_for_tool".to_string(),
|
||||
Status::Error(e) => format!("error: {e}"),
|
||||
};
|
||||
let entries = state
|
||||
.entries
|
||||
.iter()
|
||||
.map(|entry| match entry {
|
||||
Entry::User { text } => TranscriptRow {
|
||||
kind: "user".into(),
|
||||
text: text.clone(),
|
||||
title: String::new(),
|
||||
status: String::new(),
|
||||
note: String::new(),
|
||||
},
|
||||
Entry::Event(event) => TranscriptRow {
|
||||
kind: "event".into(),
|
||||
text: event.text.clone(),
|
||||
title: format!("{} · {}", event.service_label, event.topic),
|
||||
status: if event.final_ { "final".into() } else { "message".into() },
|
||||
note: if event.dropped == 0 {
|
||||
format!("sub_id: {}", event.sub_id)
|
||||
} else {
|
||||
format!("sub_id: {} · dropped: {}", event.sub_id, event.dropped)
|
||||
},
|
||||
},
|
||||
Entry::Assistant { text, streaming } => TranscriptRow {
|
||||
kind: "assistant".into(),
|
||||
text: text.clone(),
|
||||
title: String::new(),
|
||||
status: if *streaming { "streaming".into() } else { "done".into() },
|
||||
note: String::new(),
|
||||
},
|
||||
Entry::Tool(t) => {
|
||||
let (status, note, text) = match &t.status {
|
||||
ToolStatus::Confirm => ("confirm".to_string(), String::new(), String::new()),
|
||||
ToolStatus::Running { note, permille } => ("running".to_string(), format!("{note} {permille}‰"), String::new()),
|
||||
ToolStatus::Done { outcome, note, text } => (outcome.slug().to_string(), note.clone(), text.clone()),
|
||||
};
|
||||
TranscriptRow { kind: "tool".into(), text, title: t.title.clone(), status, note }
|
||||
}
|
||||
Entry::System { text } => TranscriptRow {
|
||||
kind: "system".into(),
|
||||
text: text.clone(),
|
||||
title: String::new(),
|
||||
status: String::new(),
|
||||
note: String::new(),
|
||||
},
|
||||
})
|
||||
.collect();
|
||||
Transcript {
|
||||
status,
|
||||
provider: state.provider_label.clone(),
|
||||
apps: state.services.iter().filter(|s| s.connected).map(|s| s.label.clone()).collect(),
|
||||
entries,
|
||||
generation: state.generation,
|
||||
}
|
||||
.serialize_json()
|
||||
}
|
||||
|
||||
#[derive(Script, ScriptHook, Widget)]
|
||||
pub struct AiChatOverlay {
|
||||
#[source]
|
||||
source: ScriptObjectRef,
|
||||
#[deref]
|
||||
view: View,
|
||||
/// The state generation last published as JSON.
|
||||
#[rust]
|
||||
published: Option<u64>,
|
||||
}
|
||||
|
||||
impl AiChatOverlay {
|
||||
/// The transcript JSON, rewritten when the engine's state moved — on
|
||||
/// every event as well as every draw, so a closed slot (no draws) still
|
||||
/// answers `/ai/transcript` with the turn that finished behind it.
|
||||
fn publish_transcript(&mut self, cx: &mut Cx) {
|
||||
let json = self
|
||||
.view
|
||||
.widget(cx, ids!(panel))
|
||||
.borrow::<AiChatPanel>()
|
||||
.and_then(|panel| {
|
||||
let state = panel.state()?;
|
||||
if self.published == Some(state.generation) {
|
||||
return None;
|
||||
}
|
||||
Some((state.generation, transcript_json(state)))
|
||||
});
|
||||
if let Some((generation, json)) = json {
|
||||
self.published = Some(generation);
|
||||
cx.global::<AiTranscript>().json = json;
|
||||
}
|
||||
}
|
||||
|
||||
/// The links the apps parked and the lines the bridge asked for, into
|
||||
/// the panel.
|
||||
fn adopt_requests(&mut self, cx: &mut Cx) {
|
||||
let links = cx.global::<PendingServiceLinks>().take();
|
||||
let says = std::mem::take(&mut cx.global::<AiSlotRequests>().say);
|
||||
if links.is_empty() && says.is_empty() {
|
||||
return;
|
||||
}
|
||||
let panel = self.view.widget(cx, ids!(panel));
|
||||
let Some(mut panel) = panel.borrow_mut::<AiChatPanel>() else {
|
||||
return;
|
||||
};
|
||||
for link in links {
|
||||
let label = link.manifest.label.clone();
|
||||
match panel.registry().register(link, "in this window", None) {
|
||||
Ok(endpoint) => log!("aichat: {label} joined in-process as {}", endpoint.as_str()),
|
||||
Err(e) => log!("aichat: {label} refused: {e}"),
|
||||
}
|
||||
}
|
||||
for text in says {
|
||||
panel.say(cx, text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Widget for AiChatOverlay {
|
||||
fn handle_event(&mut self, cx: &mut Cx, event: &Event, scope: &mut Scope) {
|
||||
self.adopt_requests(cx);
|
||||
self.view.handle_event(cx, event, scope);
|
||||
self.publish_transcript(cx);
|
||||
if let Event::Actions(actions) = event {
|
||||
for action in actions {
|
||||
if let Some(widget_action) = action.as_widget_action() {
|
||||
if let AiChatPanelAction::Close = widget_action.cast() {
|
||||
cx.global::<AiSlotRequests>().open = Some(false);
|
||||
cx.redraw_all();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn draw_walk(&mut self, cx: &mut Cx2d, scope: &mut Scope, walk: Walk) -> DrawStep {
|
||||
let step = self.view.draw_walk(cx, scope, walk);
|
||||
if step.is_done() {
|
||||
self.publish_transcript(cx);
|
||||
}
|
||||
step
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use makepad_ai_services::state::{EventEntry, ServiceInfo, ToolEntry};
|
||||
use makepad_ai_services::wire::ToolOutcome;
|
||||
|
||||
#[test]
|
||||
fn the_transcript_reads_as_json_rows() {
|
||||
let mut state = EngineState::default();
|
||||
state.provider_label = "No model".into();
|
||||
state.services.push(ServiceInfo {
|
||||
id: "sheets".into(),
|
||||
endpoint: "e1".into(),
|
||||
label: "Sheets".into(),
|
||||
parent: None,
|
||||
location: "in this window".into(),
|
||||
connected: true,
|
||||
launchable: false,
|
||||
tool_count: 1,
|
||||
});
|
||||
state.entries.push(Entry::User { text: "/sheets.summary {}".into() });
|
||||
state.entries.push(Entry::Event(EventEntry {
|
||||
sub_id: "s1".into(),
|
||||
service_label: "Sheets".into(),
|
||||
topic: "changes".into(),
|
||||
text: "A1 changed".into(),
|
||||
data: None,
|
||||
dropped: 2,
|
||||
final_: false,
|
||||
}));
|
||||
state.entries.push(Entry::Tool(ToolEntry {
|
||||
call_id: "c1".into(),
|
||||
service: "sheets".into(),
|
||||
service_label: "Sheets".into(),
|
||||
tool: "summary".into(),
|
||||
title: "Sheets · summary".into(),
|
||||
args: "{}".into(),
|
||||
status: ToolStatus::Done { outcome: ToolOutcome::Ok, note: "3 × 4".into(), text: "Sheet1, 3 rows".into() },
|
||||
preview: false,
|
||||
expanded: false,
|
||||
}));
|
||||
state.generation = 7;
|
||||
let json = transcript_json(&state);
|
||||
assert!(json.contains(r#""status":"idle""#), "{json}");
|
||||
assert!(json.contains(r#""apps":["Sheets"]"#), "{json}");
|
||||
assert!(json.contains(r#""kind":"user""#) && json.contains(r#""kind":"tool""#), "{json}");
|
||||
assert!(json.contains(r#""kind":"event""#) && json.contains("sub_id: s1") && json.contains("dropped: 2"), "{json}");
|
||||
assert!(json.contains(r#""title":"Sheets · summary""#) && json.contains(r#""status":"ok""#), "{json}");
|
||||
assert!(json.contains(r#""generation":7"#), "{json}");
|
||||
}
|
||||
}
|
||||
589
apps/aichat/src/panel.rs
Normal file
|
|
@ -0,0 +1,589 @@
|
|||
//! The chat panel: the one widget every host shows. It owns the engine —
|
||||
//! the service registry, the model, the transcript — and draws
|
||||
//! `EngineState` as a transcript of user lines, assistant text, tool cards
|
||||
//! (running, done, waiting for a confirm), and system lines, over a
|
||||
//! composer.
|
||||
//!
|
||||
//! Hosts talk to it in two ways: they hand it service links
|
||||
//! (`registry()`) or bus frames (`on_custom`), and they listen for
|
||||
//! [`AiChatPanelAction`]s. The engine runs on the panel's own events: every
|
||||
//! event pumps it, and while a turn is in flight the panel asks for the
|
||||
//! next frame so streaming, deadlines and the cloud provider's polling all
|
||||
//! advance without a host timer.
|
||||
|
||||
use crate::bus::ServiceBus;
|
||||
use crate::gen::GenService;
|
||||
use crate::settings::AiSettings;
|
||||
#[cfg(feature = "engine")]
|
||||
use makepad_ai_services::engine::models::{build_model, provider_rows};
|
||||
use makepad_ai_services::engine::NoModelWithReason;
|
||||
use makepad_ai_services::engine::{EngineCore, EngineEvent, ServiceRegistry};
|
||||
use makepad_ai_services::state::*;
|
||||
use makepad_ai_services::wire::ToolOutcome;
|
||||
use makepad_widgets::*;
|
||||
|
||||
script_mod! {
|
||||
use mod.prelude.widgets_internal.*
|
||||
use mod.widgets.*
|
||||
|
||||
mod.widgets.AiChatPanelBase = #(AiChatPanel::register_widget(vm))
|
||||
|
||||
let Line = Label{
|
||||
width: Fill
|
||||
height: Fit
|
||||
draw_text +: {
|
||||
color: theme.color_text
|
||||
text_style: theme.font_regular{font_size: 9.5}
|
||||
}
|
||||
}
|
||||
|
||||
let Row = View{
|
||||
width: Fill
|
||||
height: Fit
|
||||
flow: Down
|
||||
padding: Inset{left: 14 right: 14 top: 4 bottom: 4}
|
||||
}
|
||||
|
||||
mod.widgets.AiChatPanel = set_type_default() do mod.widgets.AiChatPanelBase{
|
||||
width: Fill
|
||||
height: Fill
|
||||
flow: Down
|
||||
draw_bg +: { color: theme.color_bg_app }
|
||||
|
||||
header := SolidView{
|
||||
width: Fill
|
||||
height: 36
|
||||
flow: Right
|
||||
spacing: 10
|
||||
padding: Inset{left: 14 right: 8}
|
||||
align: Align{y: 0.5}
|
||||
draw_bg +: { color: theme.color_bg_container }
|
||||
title := Label{
|
||||
text: "AI"
|
||||
draw_text +: {
|
||||
color: theme.color_text
|
||||
text_style: theme.font_bold{font_size: 10.5}
|
||||
}
|
||||
}
|
||||
provider := Label{
|
||||
width: Fill
|
||||
text: ""
|
||||
draw_text +: {
|
||||
color: theme.color_text_meta
|
||||
text_style: theme.font_regular{font_size: 8.5}
|
||||
}
|
||||
}
|
||||
clear_button := ButtonFlatter{ text: "Clear" }
|
||||
}
|
||||
|
||||
apps_row := Label{
|
||||
width: Fill
|
||||
height: Fit
|
||||
padding: Inset{left: 14 right: 14 top: 6 bottom: 2}
|
||||
max_lines: 2
|
||||
text: ""
|
||||
draw_text +: {
|
||||
color: theme.color_text_meta
|
||||
text_style: theme.font_regular{font_size: 8.5}
|
||||
}
|
||||
}
|
||||
|
||||
transcript := PortalList{
|
||||
width: Fill
|
||||
height: Fill
|
||||
auto_tail: true
|
||||
|
||||
UserRow := Row{
|
||||
padding: Inset{left: 14 right: 14 top: 10 bottom: 4}
|
||||
user_text := Line{
|
||||
draw_text +: {
|
||||
color: theme.color_text
|
||||
text_style: theme.font_bold{font_size: 9.5}
|
||||
}
|
||||
}
|
||||
}
|
||||
EventRow := Row{
|
||||
margin: Inset{left: 10 right: 10 top: 5 bottom: 5}
|
||||
padding: Inset{left: 10 right: 10 top: 7 bottom: 7}
|
||||
draw_bg +: { color: theme.color_bg_container }
|
||||
event_title := Line{
|
||||
draw_text +: {
|
||||
color: theme.color_text_hl
|
||||
text_style: theme.font_bold{font_size: 8.5}
|
||||
}
|
||||
}
|
||||
event_text := Line{}
|
||||
event_meta := Line{
|
||||
draw_text +: {
|
||||
color: theme.color_text_meta
|
||||
text_style: theme.font_regular{font_size: 8.0}
|
||||
}
|
||||
}
|
||||
}
|
||||
AssistantRow := Row{
|
||||
assistant_md := Markdown{
|
||||
width: Fill
|
||||
height: Fit
|
||||
body: ""
|
||||
}
|
||||
}
|
||||
StreamRow := Row{
|
||||
stream_text := Line{}
|
||||
}
|
||||
ToolRow := Row{
|
||||
padding: Inset{left: 22 right: 14 top: 3 bottom: 3}
|
||||
tool_head := View{
|
||||
width: Fill
|
||||
height: Fit
|
||||
cursor: MouseCursor.Hand
|
||||
tool_title := Line{
|
||||
draw_text +: {
|
||||
color: theme.color_text_meta
|
||||
text_style: theme.font_regular{font_size: 8.5}
|
||||
}
|
||||
}
|
||||
}
|
||||
tool_note := Line{
|
||||
draw_text +: {
|
||||
color: theme.color_text_meta
|
||||
text_style: theme.font_regular{font_size: 8.5}
|
||||
}
|
||||
}
|
||||
tool_bar := SolidView{
|
||||
width: Fill
|
||||
height: 2
|
||||
margin: Inset{top: 3}
|
||||
draw_bg +: { color: theme.color_text_hl }
|
||||
}
|
||||
tool_detail := Line{
|
||||
visible: false
|
||||
draw_text +: {
|
||||
color: theme.color_text_meta
|
||||
text_style: theme.font_regular{font_size: 8.0}
|
||||
}
|
||||
}
|
||||
}
|
||||
ConfirmRow := Row{
|
||||
padding: Inset{left: 22 right: 14 top: 6 bottom: 6}
|
||||
confirm_title := Line{
|
||||
draw_text +: {
|
||||
color: theme.color_text
|
||||
text_style: theme.font_regular{font_size: 9.0}
|
||||
}
|
||||
}
|
||||
confirm_buttons := View{
|
||||
width: Fill
|
||||
height: Fit
|
||||
flow: Right
|
||||
spacing: 8
|
||||
margin: Inset{top: 4}
|
||||
run_button := Button{ text: "Run" }
|
||||
deny_button := ButtonFlat{ text: "Cancel" }
|
||||
}
|
||||
}
|
||||
SystemRow := Row{
|
||||
system_text := Line{
|
||||
draw_text +: {
|
||||
color: theme.color_text_hl
|
||||
text_style: theme.font_regular{font_size: 8.5}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
status := Label{
|
||||
width: Fill
|
||||
height: Fit
|
||||
padding: Inset{left: 14 right: 14 top: 4 bottom: 2}
|
||||
max_lines: 2
|
||||
text: ""
|
||||
draw_text +: {
|
||||
color: theme.color_text_meta
|
||||
text_style: theme.font_regular{font_size: 8.5}
|
||||
}
|
||||
}
|
||||
|
||||
composer := View{
|
||||
width: Fill
|
||||
height: Fit
|
||||
flow: Right
|
||||
spacing: 8
|
||||
padding: Inset{left: 12 right: 12 top: 6 bottom: 10}
|
||||
align: Align{y: 0.5}
|
||||
input := TextInput{
|
||||
width: Fill
|
||||
height: Fit
|
||||
empty_text: "Ask AI"
|
||||
// The prompt is a hint, not text: a dark grey in every state,
|
||||
// never the typed colour (the composer is always focused).
|
||||
draw_text +: {
|
||||
color_empty: #666666
|
||||
color_empty_hover: #777777
|
||||
color_empty_focus: #666666
|
||||
}
|
||||
}
|
||||
send_button := Button{ text: "Send" }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What the panel tells its host.
|
||||
#[derive(Clone, Debug, PartialEq, Default)]
|
||||
pub enum AiChatPanelAction {
|
||||
/// Esc with an empty composer and no turn in flight: the host may hide
|
||||
/// the pane.
|
||||
Close,
|
||||
#[default]
|
||||
None,
|
||||
}
|
||||
|
||||
#[derive(Script, ScriptHook, Widget)]
|
||||
pub struct AiChatPanel {
|
||||
#[source]
|
||||
source: ScriptObjectRef,
|
||||
#[deref]
|
||||
view: View,
|
||||
#[rust]
|
||||
engine: Option<EngineCore>,
|
||||
#[rust]
|
||||
registry: ServiceRegistry,
|
||||
#[rust]
|
||||
bus: ServiceBus,
|
||||
/// The assistant's own `gen` service (pictures from the fleet), joined
|
||||
/// to the registry with the engine.
|
||||
#[rust]
|
||||
gen: Option<GenService>,
|
||||
#[rust]
|
||||
settings: Option<AiSettings>,
|
||||
#[rust]
|
||||
drawn_generation: u64,
|
||||
#[rust]
|
||||
next_frame: NextFrame,
|
||||
/// The composer took the keyboard once it existed on screen — a
|
||||
/// focus set before the first draw lands on no area at all.
|
||||
#[rust]
|
||||
composer_focused: bool,
|
||||
}
|
||||
|
||||
impl AiChatPanel {
|
||||
/// The registry a host plugs in-process links into.
|
||||
pub fn registry(&self) -> &ServiceRegistry {
|
||||
&self.registry
|
||||
}
|
||||
|
||||
/// A studio `Custom` frame that may be a bus frame from the WM.
|
||||
pub fn on_custom(&mut self, json: &str) -> bool {
|
||||
self.bus.on_custom(&self.registry, json)
|
||||
}
|
||||
|
||||
pub fn state(&self) -> Option<&EngineState> {
|
||||
self.engine.as_ref().map(|e| e.state())
|
||||
}
|
||||
|
||||
fn settings(&mut self) -> &AiSettings {
|
||||
if self.settings.is_none() {
|
||||
self.settings = Some(AiSettings::load());
|
||||
}
|
||||
self.settings.as_ref().unwrap()
|
||||
}
|
||||
|
||||
/// The engine comes up on first use — a local model is a long load
|
||||
/// and no host pays it before the person opens the pane.
|
||||
fn ensure_engine(&mut self) -> &mut EngineCore {
|
||||
if self.engine.is_none() {
|
||||
let lease_id = self.widget_uid().0;
|
||||
let settings = self.settings().clone();
|
||||
// The real models ride the `engine` feature; a build without a
|
||||
// runtime (the web page) says so and keeps the tool console.
|
||||
#[cfg(feature = "engine")]
|
||||
let (model, rows): (Box<dyn makepad_ai_services::Model>, Vec<ProviderRow>) = (
|
||||
match build_model(&settings.provider, settings.local_only) {
|
||||
Ok(m) => m,
|
||||
Err(reason) => Box::new(NoModelWithReason::new(reason)),
|
||||
},
|
||||
provider_rows(settings.local_only),
|
||||
);
|
||||
#[cfg(not(feature = "engine"))]
|
||||
let (model, rows): (Box<dyn makepad_ai_services::Model>, Vec<ProviderRow>) =
|
||||
(Box::new(NoModelWithReason::new("this build has no model runtime")), Vec::new());
|
||||
let mut core = EngineCore::new(self.registry.clone(), model, None, lease_id);
|
||||
// The state is the panel's window into the core; the core owns
|
||||
// it, so provider facts go in through the core.
|
||||
core.set_provider_facts(settings.provider.clone(), rows, settings.local_only);
|
||||
self.engine = Some(core);
|
||||
self.gen = GenService::open(&self.registry);
|
||||
}
|
||||
self.engine.as_mut().unwrap()
|
||||
}
|
||||
|
||||
fn now(cx: &Cx) -> f64 {
|
||||
cx.seconds_since_app_start()
|
||||
}
|
||||
|
||||
/// A line as if typed and sent — the bridge's `/ai?say=`.
|
||||
pub fn say(&mut self, cx: &mut Cx, text: String) {
|
||||
self.send(cx, text);
|
||||
}
|
||||
|
||||
fn send(&mut self, cx: &mut Cx, text: String) {
|
||||
let now = Self::now(cx);
|
||||
self.ensure_engine().send(&text, now);
|
||||
let input = self.view.text_input(cx, ids!(input));
|
||||
input.set_text(cx, "");
|
||||
// The widget drops the keyboard on submit; a chat composer keeps
|
||||
// it, so the next line can be typed straight away.
|
||||
input.take_key_focus(cx);
|
||||
self.view.redraw(cx);
|
||||
}
|
||||
|
||||
fn apps_line(&self, state: &EngineState) -> String {
|
||||
if state.services.is_empty() {
|
||||
return "No apps connected.".into();
|
||||
}
|
||||
state
|
||||
.services
|
||||
.iter()
|
||||
.map(|s| if s.connected { s.label.clone() } else { format!("{} (not running)", s.label) })
|
||||
.collect::<Vec<_>>()
|
||||
.join(" · ")
|
||||
}
|
||||
|
||||
fn status_line(state: &EngineState) -> String {
|
||||
let rate = state.rate.map(|r| format!(" · {r:.0} tok/s")).unwrap_or_default();
|
||||
match &state.status {
|
||||
Status::Idle => rate.trim_start_matches(" · ").to_string(),
|
||||
Status::Loading { phase, fraction } => format!("loading {phase} {:.0}%", fraction * 100.0),
|
||||
Status::Thinking => {
|
||||
if state.thinking.is_empty() {
|
||||
"thinking…".to_string()
|
||||
} else {
|
||||
let tail: String = state.thinking.chars().rev().take(120).collect::<Vec<_>>().into_iter().rev().collect();
|
||||
format!("thinking… {tail}")
|
||||
}
|
||||
}
|
||||
Status::Streaming => format!("writing{rate}"),
|
||||
Status::WaitingForTool => "waiting for the app…".to_string(),
|
||||
Status::Error(e) => e.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for AiChatPanel {
|
||||
fn drop(&mut self) {
|
||||
// The engine only enqueues Unsubscribe frames. Flush them while the
|
||||
// hosted bus and registry still exist; field destruction is too late.
|
||||
if let Some(engine) = self.engine.as_mut() {
|
||||
engine.shutdown();
|
||||
}
|
||||
self.bus.relay_down(&self.registry);
|
||||
}
|
||||
}
|
||||
|
||||
impl Widget for AiChatPanel {
|
||||
fn handle_event(&mut self, cx: &mut Cx, event: &Event, scope: &mut Scope) {
|
||||
if let Event::Custom(json) = event {
|
||||
self.on_custom(json);
|
||||
}
|
||||
// Esc: stop a running turn; with nothing running and an empty
|
||||
// composer, ask the host to hide the pane.
|
||||
if let Event::KeyDown(ke) = event {
|
||||
if ke.key_code == KeyCode::Escape {
|
||||
let busy = self.engine.as_ref().map(|e| e.state().status.is_busy()).unwrap_or(false);
|
||||
let empty = self.view.text_input(cx, ids!(input)).text().trim().is_empty();
|
||||
if busy {
|
||||
let now = Self::now(cx);
|
||||
self.ensure_engine().cancel(now);
|
||||
self.view.redraw(cx);
|
||||
} else if empty {
|
||||
cx.widget_action(self.widget_uid(), AiChatPanelAction::Close);
|
||||
}
|
||||
}
|
||||
}
|
||||
// The built-in services answer before the engine pumps, so a
|
||||
// finished picture lands in this same event.
|
||||
if let Some(gen) = self.gen.as_mut() {
|
||||
gen.handle_event(cx, event);
|
||||
}
|
||||
// Drive the engine on every event; the bus relays what it sent.
|
||||
let now = Self::now(cx);
|
||||
let mut changed = false;
|
||||
let mut busy = false;
|
||||
if let Some(engine) = self.engine.as_mut() {
|
||||
for ev in engine.pump(now) {
|
||||
match ev {
|
||||
EngineEvent::Changed => changed = true,
|
||||
EngineEvent::Confirm { .. } => changed = true,
|
||||
}
|
||||
}
|
||||
busy = engine.needs_pump()
|
||||
|| engine.state().status.is_busy()
|
||||
|| matches!(engine.state().status, Status::Loading { .. });
|
||||
}
|
||||
self.bus.relay_down(&self.registry);
|
||||
if changed {
|
||||
self.view.redraw(cx);
|
||||
}
|
||||
if busy {
|
||||
self.next_frame = cx.new_next_frame();
|
||||
}
|
||||
self.view.handle_event(cx, event, scope);
|
||||
self.widget_match_event(cx, event, scope);
|
||||
}
|
||||
|
||||
fn draw_walk(&mut self, cx: &mut Cx2d, scope: &mut Scope, walk: Walk) -> DrawStep {
|
||||
// The engine exists once the panel is on screen.
|
||||
let _ = self.ensure_engine();
|
||||
let state = self.engine.as_ref().unwrap().state().clone();
|
||||
self.drawn_generation = state.generation;
|
||||
self.view.label(cx, ids!(provider)).set_text(cx, &format!("{}{}", state.provider_label, if state.local_only { " · local only" } else { "" }));
|
||||
self.view.label(cx, ids!(apps_row)).set_text(cx, &self.apps_line(&state));
|
||||
self.view.label(cx, ids!(status)).set_text(cx, &Self::status_line(&state));
|
||||
while let Some(item) = self.view.draw_walk(cx, scope, walk).step() {
|
||||
let Some(mut list) = item.borrow_mut::<PortalList>() else { continue };
|
||||
let total = state.entries.len();
|
||||
list.set_item_range(cx, 0, total);
|
||||
while let Some(index) = list.next_visible_item(cx) {
|
||||
if index >= total {
|
||||
continue;
|
||||
}
|
||||
let entry = &state.entries[index];
|
||||
let row = match entry {
|
||||
Entry::User { text } => {
|
||||
let row = list.item(cx, index, id!(UserRow));
|
||||
row.label(cx, ids!(user_text)).set_text(cx, text);
|
||||
row
|
||||
}
|
||||
Entry::Event(event) => {
|
||||
let row = list.item(cx, index, id!(EventRow));
|
||||
row.label(cx, ids!(event_title))
|
||||
.set_text(cx, &format!("→ {} · {}", event.service_label, event.topic));
|
||||
row.label(cx, ids!(event_text)).set_text(cx, &event.text);
|
||||
let mut meta = format!("sub_id: {}", event.sub_id);
|
||||
if event.dropped != 0 {
|
||||
meta.push_str(&format!(" · dropped: {}", event.dropped));
|
||||
}
|
||||
if event.final_ {
|
||||
meta.push_str(" · final");
|
||||
}
|
||||
if let Some(data) = &event.data {
|
||||
meta.push_str(&format!("\n{data}"));
|
||||
}
|
||||
row.label(cx, ids!(event_meta)).set_text(cx, &meta);
|
||||
row
|
||||
}
|
||||
Entry::Assistant { text, streaming: false } => {
|
||||
let row = list.item(cx, index, id!(AssistantRow));
|
||||
// No text, no row: a blank block above a card is a gap.
|
||||
row.set_visible(cx, !text.trim().is_empty());
|
||||
if let Some(mut md) = row.widget(cx, ids!(assistant_md)).borrow_mut::<Markdown>() {
|
||||
md.set_text(cx, text);
|
||||
}
|
||||
row
|
||||
}
|
||||
Entry::Assistant { text, streaming: true } => {
|
||||
let row = list.item(cx, index, id!(StreamRow));
|
||||
row.set_visible(cx, !text.trim().is_empty());
|
||||
row.label(cx, ids!(stream_text)).set_text(cx, text);
|
||||
row
|
||||
}
|
||||
Entry::Tool(t) if matches!(t.status, ToolStatus::Confirm) => {
|
||||
let row = list.item(cx, index, id!(ConfirmRow));
|
||||
row.label(cx, ids!(confirm_title)).set_text(cx, &format!("{} — this changes things outside the app. Run it?", t.title));
|
||||
row
|
||||
}
|
||||
Entry::Tool(t) => {
|
||||
let row = list.item(cx, index, id!(ToolRow));
|
||||
let (glyph, note, permille, detail) = match &t.status {
|
||||
ToolStatus::Running { note, permille } => ("›", note.clone(), *permille, String::new()),
|
||||
ToolStatus::Done { outcome, note, text } => {
|
||||
let glyph = if outcome.is_ok() { "✓" } else { "✗" };
|
||||
let note = if note.is_empty() { outcome.slug().to_string() } else { note.clone() };
|
||||
(glyph, note, 1000, text.clone())
|
||||
}
|
||||
ToolStatus::Confirm => ("?", String::new(), 0, String::new()),
|
||||
};
|
||||
row.label(cx, ids!(tool_title)).set_text(cx, &format!("{glyph} {}", t.title));
|
||||
row.label(cx, ids!(tool_note)).set_text(cx, ¬e);
|
||||
let bar_visible = matches!(t.status, ToolStatus::Running { .. });
|
||||
row.view(cx, ids!(tool_bar)).set_visible(cx, bar_visible);
|
||||
if bar_visible {
|
||||
// The bar is a fraction of the row's inner width
|
||||
// (the row's rect is last frame's; the first
|
||||
// frame of a card draws it at a token width).
|
||||
let inner = (row.area().rect(cx).size.x - 36.0).max(24.0);
|
||||
let px = inner * (permille as f64 / 1000.0).max(0.05);
|
||||
let mut bar = row.view(cx, ids!(tool_bar));
|
||||
script_apply_eval!(cx, bar, { width: #(px) });
|
||||
}
|
||||
let detail_label = row.label(cx, ids!(tool_detail));
|
||||
detail_label.set_visible(cx, t.expanded && !detail.is_empty());
|
||||
if t.expanded {
|
||||
detail_label.set_text(cx, &format!("{}\n{}", t.args, detail));
|
||||
}
|
||||
let _ = matches!(t.status, ToolStatus::Done { outcome: ToolOutcome::Ok, .. });
|
||||
row
|
||||
}
|
||||
Entry::System { text } => {
|
||||
let row = list.item(cx, index, id!(SystemRow));
|
||||
row.label(cx, ids!(system_text)).set_text(cx, text);
|
||||
row
|
||||
}
|
||||
};
|
||||
row.draw_all(cx, &mut Scope::empty());
|
||||
}
|
||||
}
|
||||
if !self.composer_focused {
|
||||
self.composer_focused = true;
|
||||
self.view.text_input(cx, ids!(input)).take_key_focus(cx);
|
||||
}
|
||||
DrawStep::done()
|
||||
}
|
||||
}
|
||||
|
||||
impl WidgetMatchEvent for AiChatPanel {
|
||||
fn handle_actions(&mut self, cx: &mut Cx, actions: &Actions, scope: &mut Scope) {
|
||||
let input = self.view.text_input(cx, ids!(input));
|
||||
let returned = input.returned(actions).map(|(text, _)| text);
|
||||
if let Some(text) = returned {
|
||||
self.send(cx, text);
|
||||
} else if self.view.button(cx, ids!(send_button)).clicked(actions) {
|
||||
let text = input.text();
|
||||
self.send(cx, text);
|
||||
}
|
||||
if self.view.button(cx, ids!(clear_button)).clicked(actions) {
|
||||
let now = Self::now(cx);
|
||||
self.ensure_engine().clear(now);
|
||||
self.view.redraw(cx);
|
||||
}
|
||||
// Cards: confirm buttons and click-to-expand.
|
||||
let list = self.view.portal_list(cx, ids!(transcript));
|
||||
let items = list.items_with_actions(actions);
|
||||
if !items.is_empty() {
|
||||
let now = Self::now(cx);
|
||||
let call_ids: Vec<Option<String>> = {
|
||||
let state = self.ensure_engine().state();
|
||||
items
|
||||
.iter()
|
||||
.map(|(index, _)| match state.entries.get(*index) {
|
||||
Some(Entry::Tool(t)) => Some(t.call_id.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
for ((_, item), call_id) in items.iter().zip(call_ids) {
|
||||
let Some(call_id) = call_id else { continue };
|
||||
if item.button(cx, ids!(run_button)).clicked(actions) {
|
||||
self.ensure_engine().confirm(&call_id, true, now);
|
||||
self.view.redraw(cx);
|
||||
} else if item.button(cx, ids!(deny_button)).clicked(actions) {
|
||||
self.ensure_engine().confirm(&call_id, false, now);
|
||||
self.view.redraw(cx);
|
||||
} else if item.view(cx, ids!(tool_head)).finger_up(actions).is_some() {
|
||||
self.ensure_engine().toggle_tool(&call_id);
|
||||
self.view.redraw(cx);
|
||||
}
|
||||
}
|
||||
}
|
||||
let _ = scope;
|
||||
}
|
||||
}
|
||||
101
apps/aichat/src/settings.rs
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
//! What the person chose: which model answers, and whether cloud models
|
||||
//! are locked out. Two lines in `~/.makepad/aichat/settings`, the lock a
|
||||
//! promise kept in code (a cloud choice under the lock normalises to
|
||||
//! local at load and is refused at set), never a menu filter.
|
||||
|
||||
use makepad_ai_services::state::ProviderChoice;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct AiSettings {
|
||||
pub provider: ProviderChoice,
|
||||
/// Default ON: the person must switch it off before any cloud model
|
||||
/// can be picked.
|
||||
pub local_only: bool,
|
||||
}
|
||||
|
||||
impl Default for AiSettings {
|
||||
fn default() -> Self {
|
||||
AiSettings { provider: ProviderChoice::Local, local_only: true }
|
||||
}
|
||||
}
|
||||
|
||||
impl AiSettings {
|
||||
/// The lock, applied: a cloud provider under the lock becomes local.
|
||||
/// "none" (no model) is always allowed — it reaches nothing.
|
||||
pub fn normalized(mut self) -> Self {
|
||||
if self.local_only {
|
||||
if let ProviderChoice::Cloud(slug) = &self.provider {
|
||||
if slug != "none" {
|
||||
self.provider = ProviderChoice::Local;
|
||||
}
|
||||
}
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Can this choice be made under the current lock?
|
||||
pub fn allows(&self, choice: &ProviderChoice) -> Result<(), String> {
|
||||
match choice {
|
||||
ProviderChoice::Cloud(slug) if self.local_only && slug != "none" => Err("Local AI only is on".into()),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(text: &str) -> AiSettings {
|
||||
let mut s = AiSettings::default();
|
||||
for line in text.lines() {
|
||||
let Some((k, v)) = line.split_once('=') else { continue };
|
||||
match k.trim() {
|
||||
"provider" => s.provider = ProviderChoice::from_slug(v.trim()),
|
||||
"local_only" => s.local_only = v.trim() != "false",
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
s.normalized()
|
||||
}
|
||||
|
||||
pub fn render(&self) -> String {
|
||||
format!("provider={}\nlocal_only={}\n", self.provider.slug(), self.local_only)
|
||||
}
|
||||
|
||||
pub fn path() -> Option<PathBuf> {
|
||||
if cfg!(target_arch = "wasm32") {
|
||||
return None;
|
||||
}
|
||||
let home = std::env::var_os("HOME").or_else(|| std::env::var_os("USERPROFILE"))?;
|
||||
Some(PathBuf::from(home).join(".makepad").join("aichat").join("settings"))
|
||||
}
|
||||
|
||||
pub fn load() -> AiSettings {
|
||||
match Self::path().and_then(|p| std::fs::read_to_string(p).ok()) {
|
||||
Some(text) => Self::parse(&text),
|
||||
None => AiSettings::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn save(&self) -> Result<(), String> {
|
||||
let Some(path) = Self::path() else { return Ok(()) };
|
||||
if let Some(dir) = path.parent() {
|
||||
std::fs::create_dir_all(dir).map_err(|e| e.to_string())?;
|
||||
}
|
||||
std::fs::write(&path, self.render()).map_err(|e| e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_lock_is_kept_at_load_and_at_set() {
|
||||
let s = AiSettings::parse("provider=claude-api\nlocal_only=true\n");
|
||||
assert_eq!(s.provider, ProviderChoice::Local, "a cloud pref under the lock cannot resurrect a cloud model");
|
||||
assert!(s.allows(&ProviderChoice::Cloud("claude-api".into())).is_err());
|
||||
assert!(s.allows(&ProviderChoice::Cloud("none".into())).is_ok());
|
||||
let open = AiSettings::parse("provider=claude-api\nlocal_only=false\n");
|
||||
assert_eq!(open.provider, ProviderChoice::Cloud("claude-api".into()));
|
||||
assert_eq!(AiSettings::parse(&open.render()), open);
|
||||
assert_eq!(AiSettings::parse("garbage"), AiSettings::default());
|
||||
}
|
||||
}
|
||||
|
|
@ -19,6 +19,7 @@ makepad-render = { path = "../../libs/render" }
|
|||
makepad-gltf = { path = "../../libs/gltf" }
|
||||
# Splat viewer: ViewSplat + XrSceneView desktop host.
|
||||
makepad-xr = { path = "../../xr" }
|
||||
makepad-media-view = { path = "../../libs/media_view" }
|
||||
# REAL Asset Server frontend: shared session lifecycle (discovery/auth/
|
||||
# retry), catalog runtimes, committed-event subscriber, verified cache. The
|
||||
# VJ worker owns these crates; this app consumes the public API only.
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ use makepad_audio_lyrics::bake::LyricsBaker;
|
|||
use makepad_audio_lyrics::TrackLyrics;
|
||||
use makepad_audio_sidechannels::{encode_stem_oggs, publish_side_channels};
|
||||
use makepad_widgets::log;
|
||||
use makepad_widgets::makepad_platform::thread::SignalToUI;
|
||||
use makepad_widgets::makepad_platform::thread::{SignalToUI, ThreadOptions, ThreadSpawner};
|
||||
use std::collections::HashSet;
|
||||
use std::path::PathBuf;
|
||||
use std::str::FromStr;
|
||||
|
|
@ -545,14 +545,8 @@ pub struct AnalysisQueue {
|
|||
fetch_generation: u64,
|
||||
}
|
||||
|
||||
impl Default for AnalysisQueue {
|
||||
fn default() -> Self {
|
||||
AnalysisQueue::start()
|
||||
}
|
||||
}
|
||||
|
||||
impl AnalysisQueue {
|
||||
pub fn start() -> AnalysisQueue {
|
||||
pub fn start(spawner: ThreadSpawner) -> AnalysisQueue {
|
||||
let (bake_tx, bake_requests) = channel::<BakeRequest>();
|
||||
let (bake_done, bake_rx) = channel::<BakeMsg>();
|
||||
let (fetch_tx, fetch_requests) = channel::<FetchRequest>();
|
||||
|
|
@ -561,12 +555,26 @@ impl AnalysisQueue {
|
|||
let worker_batch = Arc::clone(&batch);
|
||||
// A failed spawn is not fatal: the queue simply never runs, and
|
||||
// every enqueue reports it instead of pretending to work.
|
||||
let _ = std::thread::Builder::new()
|
||||
.name("asset-ui-stem-bake".into())
|
||||
.spawn(move || bake_loop(bake_requests, bake_done, worker_batch));
|
||||
let _ = std::thread::Builder::new()
|
||||
.name("asset-ui-stem-fetch".into())
|
||||
.spawn(move || fetch_loop(fetch_requests, fetch_done));
|
||||
match spawner.spawn_worker(
|
||||
ThreadOptions {
|
||||
name: Some("asset-ui-stem-bake".into()),
|
||||
..Default::default()
|
||||
},
|
||||
move || bake_loop(bake_requests, bake_done, worker_batch),
|
||||
) {
|
||||
Ok(handle) => handle.detach(),
|
||||
Err(error) => log!("analysis bake worker unavailable: {error}"),
|
||||
}
|
||||
match spawner.spawn_worker(
|
||||
ThreadOptions {
|
||||
name: Some("asset-ui-stem-fetch".into()),
|
||||
..Default::default()
|
||||
},
|
||||
move || fetch_loop(fetch_requests, fetch_done),
|
||||
) {
|
||||
Ok(handle) => handle.detach(),
|
||||
Err(error) => log!("analysis fetch worker unavailable: {error}"),
|
||||
}
|
||||
AnalysisQueue {
|
||||
bake_tx,
|
||||
bake_rx,
|
||||
|
|
@ -1473,7 +1481,8 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn the_queue_counts_a_batch_and_keeps_its_verdict() {
|
||||
let mut queue = AnalysisQueue::start();
|
||||
let cx = makepad_widgets::Cx::new(Box::new(|_, _| {}));
|
||||
let mut queue = AnalysisQueue::start(cx.thread_spawner());
|
||||
assert!(!queue.busy());
|
||||
assert_eq!(queue.status_line(), "");
|
||||
assert_eq!(queue.progress_fraction(), 0.0);
|
||||
|
|
|
|||
|
|
@ -13,12 +13,11 @@
|
|||
//! - gallery preview decodes run on a small worker pool (2..=8 threads)
|
||||
//! pulling a last-in-first-out stack, capped so old off-screen work drops.
|
||||
|
||||
use makepad_widgets::makepad_platform::thread::SignalToUI;
|
||||
use makepad_widgets::makepad_platform::thread::{SignalToUI, ThreadOptions, ThreadSpawner};
|
||||
use makepad_widgets::{decode_image_from_data, ImageBuffer};
|
||||
use std::collections::HashSet;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::mpsc::{channel, Receiver, Sender};
|
||||
use std::sync::{Arc, Condvar, Mutex};
|
||||
|
||||
/// What one background read is for.
|
||||
pub enum IoPurpose {
|
||||
|
|
@ -164,35 +163,39 @@ pub enum IoDone {
|
|||
pub struct ArtifactIo {
|
||||
tx: Sender<IoRequest>,
|
||||
rx: Receiver<IoDone>,
|
||||
gallery: Arc<GalleryStack>,
|
||||
}
|
||||
|
||||
impl ArtifactIo {
|
||||
pub fn start() -> Self {
|
||||
pub fn start(spawner: ThreadSpawner) -> Self {
|
||||
let (request_tx, request_rx) = channel::<IoRequest>();
|
||||
let (done_tx, done_rx) = channel::<IoDone>();
|
||||
let gallery = Arc::new(GalleryStack::new());
|
||||
std::thread::Builder::new()
|
||||
.name("asset-ui-artifact-io".into())
|
||||
.spawn({
|
||||
let (gallery_tx, gallery_rx) = channel::<IoRequest>();
|
||||
spawner
|
||||
.spawn_worker(
|
||||
ThreadOptions {
|
||||
name: Some("asset-ui-artifact-io".into()),
|
||||
..Default::default()
|
||||
},
|
||||
{
|
||||
let done_tx = done_tx.clone();
|
||||
let gallery = Arc::clone(&gallery);
|
||||
move || dispatch_loop(request_rx, done_tx, gallery)
|
||||
})
|
||||
.expect("artifact io dispatcher");
|
||||
let n = gallery_worker_count();
|
||||
for i in 0..n {
|
||||
let done_tx = done_tx.clone();
|
||||
let gallery = Arc::clone(&gallery);
|
||||
std::thread::Builder::new()
|
||||
.name(format!("asset-ui-preview-{i}"))
|
||||
.spawn(move || gallery_loop(gallery, done_tx))
|
||||
.expect("gallery decode worker");
|
||||
}
|
||||
move || dispatch_loop(request_rx, done_tx, gallery_tx)
|
||||
},
|
||||
)
|
||||
.expect("artifact io dispatcher")
|
||||
.detach();
|
||||
spawner
|
||||
.spawn_worker(
|
||||
ThreadOptions {
|
||||
name: Some("asset-ui-preview".into()),
|
||||
..Default::default()
|
||||
},
|
||||
move || gallery_loop(gallery_rx, done_tx),
|
||||
)
|
||||
.expect("gallery decode worker")
|
||||
.detach();
|
||||
Self {
|
||||
tx: request_tx,
|
||||
rx: done_rx,
|
||||
gallery,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -206,12 +209,6 @@ impl ArtifactIo {
|
|||
}
|
||||
}
|
||||
|
||||
impl Drop for ArtifactIo {
|
||||
fn drop(&mut self) {
|
||||
self.gallery.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
fn is_gallery(purpose: &IoPurpose) -> bool {
|
||||
matches!(
|
||||
purpose,
|
||||
|
|
@ -221,13 +218,13 @@ fn is_gallery(purpose: &IoPurpose) -> bool {
|
|||
)
|
||||
}
|
||||
|
||||
fn dispatch_loop(rx: Receiver<IoRequest>, tx: Sender<IoDone>, gallery: Arc<GalleryStack>) {
|
||||
fn dispatch_loop(rx: Receiver<IoRequest>, tx: Sender<IoDone>, gallery: Sender<IoRequest>) {
|
||||
// One connected client per session, reused across opens: against the
|
||||
// local server a fresh connect costs more than the payload.
|
||||
let mut store: Option<(crate::import::ServerSession, makepad_asset_client::AssetClient)> = None;
|
||||
while let Ok(request) = rx.recv() {
|
||||
if is_gallery(&request.purpose) {
|
||||
gallery.push_latest(request);
|
||||
let _ = gallery.send(request);
|
||||
continue;
|
||||
}
|
||||
let done = process_with_store(request, &mut store);
|
||||
|
|
@ -236,10 +233,15 @@ fn dispatch_loop(rx: Receiver<IoRequest>, tx: Sender<IoDone>, gallery: Arc<Galle
|
|||
}
|
||||
SignalToUI::set_ui_signal();
|
||||
}
|
||||
gallery.shutdown();
|
||||
}
|
||||
|
||||
fn gallery_loop(gallery: Arc<GalleryStack>, tx: Sender<IoDone>) {
|
||||
fn gallery_loop(rx: Receiver<IoRequest>, tx: Sender<IoDone>) {
|
||||
let mut gallery = GalleryStack::new();
|
||||
while let Ok(request) = rx.recv() {
|
||||
gallery.push_latest(request);
|
||||
for request in rx.try_iter() {
|
||||
gallery.push_latest(request);
|
||||
}
|
||||
while let Some(request) = gallery.pop_latest() {
|
||||
let file = request.file.clone();
|
||||
let done = process(request);
|
||||
|
|
@ -248,6 +250,10 @@ fn gallery_loop(gallery: Arc<GalleryStack>, tx: Sender<IoDone>) {
|
|||
return;
|
||||
}
|
||||
SignalToUI::set_ui_signal();
|
||||
for request in rx.try_iter() {
|
||||
gallery.push_latest(request);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -264,24 +270,22 @@ struct GalleryInner {
|
|||
}
|
||||
|
||||
struct GalleryStack {
|
||||
inner: Mutex<GalleryInner>,
|
||||
cv: Condvar,
|
||||
inner: GalleryInner,
|
||||
}
|
||||
|
||||
impl GalleryStack {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
inner: Mutex::new(GalleryInner {
|
||||
inner: GalleryInner {
|
||||
stack: Vec::new(),
|
||||
decoding: HashSet::new(),
|
||||
shutdown: false,
|
||||
}),
|
||||
cv: Condvar::new(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn push_latest(&self, request: IoRequest) {
|
||||
let mut g = self.inner.lock().expect("gallery stack");
|
||||
fn push_latest(&mut self, request: IoRequest) {
|
||||
let g = &mut self.inner;
|
||||
if g.shutdown {
|
||||
return;
|
||||
}
|
||||
|
|
@ -294,12 +298,10 @@ impl GalleryStack {
|
|||
let drop_n = g.stack.len() - GALLERY_STACK_CAP;
|
||||
g.stack.drain(0..drop_n);
|
||||
}
|
||||
self.cv.notify_one();
|
||||
}
|
||||
|
||||
fn pop_latest(&self) -> Option<IoRequest> {
|
||||
let mut g = self.inner.lock().expect("gallery stack");
|
||||
loop {
|
||||
fn pop_latest(&mut self) -> Option<IoRequest> {
|
||||
let g = &mut self.inner;
|
||||
if g.shutdown {
|
||||
return None;
|
||||
}
|
||||
|
|
@ -308,29 +310,18 @@ impl GalleryStack {
|
|||
return Some(request);
|
||||
}
|
||||
}
|
||||
g = self.cv.wait(g).expect("gallery stack");
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn finish(&self, file: &str) {
|
||||
let mut g = self.inner.lock().expect("gallery stack");
|
||||
g.decoding.remove(file);
|
||||
fn finish(&mut self, file: &str) {
|
||||
self.inner.decoding.remove(file);
|
||||
}
|
||||
|
||||
fn shutdown(&self) {
|
||||
let mut g = self.inner.lock().expect("gallery stack");
|
||||
g.shutdown = true;
|
||||
self.cv.notify_all();
|
||||
fn shutdown(&mut self) {
|
||||
self.inner.shutdown = true;
|
||||
}
|
||||
}
|
||||
|
||||
fn gallery_worker_count() -> usize {
|
||||
std::thread::available_parallelism()
|
||||
.map(|n| n.get())
|
||||
.unwrap_or(4)
|
||||
.clamp(2, 8)
|
||||
}
|
||||
|
||||
/// A read whose bytes may come from the store. Everything past the fetch is
|
||||
/// the same decode the local-file path uses — one viewer, two sources.
|
||||
fn process_with_store(
|
||||
|
|
@ -865,7 +856,8 @@ mod tests {
|
|||
std::fs::write(&payload, b"mp4-bytes").unwrap();
|
||||
let copy = dir.join("viewer-open.mp4");
|
||||
|
||||
let io = ArtifactIo::start();
|
||||
let cx = makepad_widgets::Cx::new(Box::new(|_, _| {}));
|
||||
let io = ArtifactIo::start(cx.thread_spawner());
|
||||
io.request(IoRequest {
|
||||
file: "clip.mp4".into(),
|
||||
path: payload.clone(),
|
||||
|
|
@ -1143,7 +1135,7 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn gallery_stack_is_last_requested_first_and_rebumps() {
|
||||
let stack = GalleryStack::new();
|
||||
let mut stack = GalleryStack::new();
|
||||
let mk = |file: &str| IoRequest {
|
||||
file: file.into(),
|
||||
path: PathBuf::from(file),
|
||||
|
|
@ -1168,7 +1160,7 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn gallery_stack_drops_oldest_when_capped() {
|
||||
let stack = GalleryStack::new();
|
||||
let mut stack = GalleryStack::new();
|
||||
for i in 0..(GALLERY_STACK_CAP + 10) {
|
||||
stack.push_latest(IoRequest {
|
||||
file: format!("f{i}"),
|
||||
|
|
|
|||
|
|
@ -56,6 +56,9 @@ use makepad_asset_client::{
|
|||
use makepad_asset_data::{AssetId, AssetRevisionId};
|
||||
pub use makepad_asset_data::AssetKind;
|
||||
use makepad_widgets::log;
|
||||
use makepad_widgets::makepad_platform::thread::{
|
||||
Lane, TaskHandle, TaskPool, ThreadOptions, ThreadSpawner,
|
||||
};
|
||||
use std::collections::VecDeque;
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
|
@ -360,6 +363,8 @@ pub struct SearchResults {
|
|||
/// happens through `start`/`poll`/`submit_search`/`select` only.
|
||||
#[derive(Default)]
|
||||
pub struct AssetStore {
|
||||
pool: Option<TaskPool>,
|
||||
spawner: Option<ThreadSpawner>,
|
||||
/// Continuous ai-content-library → catalog publisher. Declared BEFORE
|
||||
/// `embedded` so it is joined while the server it publishes into is
|
||||
/// still alive.
|
||||
|
|
@ -397,7 +402,7 @@ pub struct AssetStore {
|
|||
/// boxes directly (the store advertises nothing any more — generation
|
||||
/// is client-driven, aicore §9).
|
||||
pub profiles: Remote<Vec<JobProfileDto>>,
|
||||
profiles_rx: Option<std::sync::mpsc::Receiver<Vec<JobProfileDto>>>,
|
||||
profiles_task: Option<TaskHandle<Vec<JobProfileDto>>>,
|
||||
/// Committed catalog events, newest first, capped.
|
||||
pub events: VecDeque<CatalogEventDto>,
|
||||
/// The event feed delivered its initial cursor and is following commits.
|
||||
|
|
@ -480,7 +485,7 @@ pub struct AssetStore {
|
|||
succession_note: Option<String>,
|
||||
/// The previous session is being torn down off-thread. Flips when its
|
||||
/// cache roots are free for the next session to open.
|
||||
releasing: Option<Arc<AtomicBool>>,
|
||||
releasing: Option<TaskHandle<()>>,
|
||||
/// What to connect to once `releasing` flips.
|
||||
pending_session: Option<PendingSession>,
|
||||
}
|
||||
|
|
@ -498,11 +503,13 @@ impl AssetStore {
|
|||
/// the client finds it through the same discovery/health path any LAN
|
||||
/// peer would. Set the env var to skip embed and talk to a standalone
|
||||
/// server instead.
|
||||
pub fn start(&mut self, library_dir: PathBuf) {
|
||||
pub fn start(&mut self, library_dir: PathBuf, pool: TaskPool, spawner: ThreadSpawner) {
|
||||
if self.connector.is_some() || self.server.is_some() {
|
||||
return;
|
||||
}
|
||||
self.library_dir = library_dir;
|
||||
self.pool = Some(pool);
|
||||
self.spawner = Some(spawner);
|
||||
self.server_root = default_asset_server_root();
|
||||
self.beacon = beacon_from_env();
|
||||
self.embed = embed_policy_from_env();
|
||||
|
|
@ -642,7 +649,7 @@ impl AssetStore {
|
|||
label: handles.server_label.clone(),
|
||||
server_id: handles.server_id,
|
||||
});
|
||||
self.endpoints = Some(handles.endpoints);
|
||||
self.endpoints = handles.endpoints;
|
||||
self.token = handles.token.clone();
|
||||
self.handles = Some(*handles);
|
||||
self.connector = None;
|
||||
|
|
@ -655,17 +662,15 @@ impl AssetStore {
|
|||
}
|
||||
}
|
||||
// Fleet-built generation profiles landing from their worker thread.
|
||||
if let Some(rx) = &self.profiles_rx {
|
||||
match rx.try_recv() {
|
||||
if let Some(result) = self.profiles_task.as_mut().and_then(TaskHandle::try_take) {
|
||||
self.profiles_task = None;
|
||||
match result {
|
||||
Ok(profiles) => {
|
||||
self.profiles_rx = None;
|
||||
self.profiles = Remote::Ready(profiles);
|
||||
changed = true;
|
||||
}
|
||||
Err(std::sync::mpsc::TryRecvError::Empty) => {}
|
||||
Err(std::sync::mpsc::TryRecvError::Disconnected) => {
|
||||
self.profiles_rx = None;
|
||||
self.profiles = Remote::Failed("fleet probe thread died".to_string());
|
||||
Err(error) => {
|
||||
self.profiles = Remote::Failed(format!("fleet profile job failed: {error}"));
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
|
|
@ -806,13 +811,23 @@ impl AssetStore {
|
|||
/// Start the embedded server's background loops and become the host.
|
||||
fn begin_hosting(&mut self, server: makepad_asset_store::AssetServer, token: &str) {
|
||||
if self.host_loops == HostLoops::Run {
|
||||
self.publish =
|
||||
start_publish_loop(&server, token, self.library_dir.clone());
|
||||
let Some(spawner) = self.spawner.as_ref() else {
|
||||
log!("asset store: host workers unavailable (thread runtime not configured)");
|
||||
self.role = ServerRole::Host;
|
||||
self.embedded = Some(server);
|
||||
return;
|
||||
};
|
||||
self.publish = start_publish_loop(
|
||||
spawner,
|
||||
&server,
|
||||
token,
|
||||
self.library_dir.clone(),
|
||||
);
|
||||
// LIVECODING: observed origin directories, catalogued in place.
|
||||
// Only the HOST runs this — reference admission is loopback
|
||||
// privilege, so an attached client never observes for somebody
|
||||
// else's store.
|
||||
self.observe = start_observe_loop(&server, token);
|
||||
self.observe = start_observe_loop(spawner, &server, token);
|
||||
}
|
||||
self.role = ServerRole::Host;
|
||||
self.embedded = Some(server);
|
||||
|
|
@ -856,40 +871,40 @@ impl AssetStore {
|
|||
self.search_continuation = false;
|
||||
self.next_cursor = None;
|
||||
self.detail_req = None;
|
||||
self.profiles_rx = None;
|
||||
self.profiles_task = None;
|
||||
self.probe_req = None;
|
||||
self.gc_req = None;
|
||||
self.gc_cancel_req = None;
|
||||
self.retire_reqs.clear();
|
||||
let released = Arc::new(AtomicBool::new(false));
|
||||
self.releasing = Some(released.clone());
|
||||
let Some(handles) = self.handles.take() else {
|
||||
released.store(true, Ordering::Release);
|
||||
let Some(pool) = &self.pool else {
|
||||
log!("asset store: session release refused (runtime pool unavailable)");
|
||||
return;
|
||||
};
|
||||
let done = released.clone();
|
||||
let spawned = std::thread::Builder::new()
|
||||
.name("asset-ui-session-release".to_string())
|
||||
.spawn(move || {
|
||||
handles.shutdown();
|
||||
done.store(true, Ordering::Release);
|
||||
});
|
||||
if let Err(error) = spawned {
|
||||
// The closure (and the session inside it) was dropped, which
|
||||
// already joined the runtimes right here. Nothing is left to
|
||||
// wait for, so never leave the swap parked on a thread that
|
||||
// does not exist.
|
||||
log!("asset store: session release thread refused ({error}); released inline");
|
||||
released.store(true, Ordering::Release);
|
||||
let slot = match pool.reserve(Lane::Heavy) {
|
||||
Ok(slot) => slot,
|
||||
Err(error) => {
|
||||
log!("asset store: session release delayed ({error})");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let Some(handles) = self.handles.take() else {
|
||||
self.releasing = None;
|
||||
return;
|
||||
};
|
||||
self.releasing = Some(slot.submit(move || handles.shutdown()));
|
||||
}
|
||||
|
||||
/// Open the decided session once the previous one has let go.
|
||||
fn finish_swap(&mut self) -> bool {
|
||||
if let Some(released) = &self.releasing {
|
||||
if !released.load(Ordering::Acquire) {
|
||||
if self.releasing.is_none() && self.handles.is_some() {
|
||||
self.begin_release();
|
||||
return false;
|
||||
}
|
||||
if let Some(releasing) = &mut self.releasing {
|
||||
let Some(result) = releasing.try_take() else { return false };
|
||||
if let Err(error) = result {
|
||||
log!("asset store: session release failed: {error}");
|
||||
}
|
||||
}
|
||||
self.releasing = None;
|
||||
let Some(pending) = self.pending_session.take() else {
|
||||
|
|
@ -1150,18 +1165,20 @@ impl AssetStore {
|
|||
/// they can execute right now. Runs on its own thread — LAN probes must
|
||||
/// never stall a frame — and lands through `profiles_rx` in [`Self::poll`].
|
||||
fn submit_profiles(&mut self) {
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
self.profiles_rx = Some(rx);
|
||||
self.profiles = Remote::Loading;
|
||||
let _ = std::thread::Builder::new()
|
||||
.name("asset-ui-profiles".to_string())
|
||||
.spawn(move || {
|
||||
let Some(pool) = &self.pool else {
|
||||
self.profiles = Remote::Failed("runtime task pool unavailable".into());
|
||||
return;
|
||||
};
|
||||
match pool.submit(Lane::Light, move || {
|
||||
let snapshots = makepad_asset_creator::runner::fleet_snapshots();
|
||||
let profiles = makepad_asset_importer::gen_profiles::build_profiles(
|
||||
&snapshots, "gen",
|
||||
);
|
||||
let _ = tx.send(profiles);
|
||||
});
|
||||
makepad_asset_importer::gen_profiles::build_profiles(&snapshots, "gen")
|
||||
}) {
|
||||
Ok(handle) => self.profiles_task = Some(handle),
|
||||
Err(error) => {
|
||||
self.profiles = Remote::Failed(format!("fleet profile job refused: {error}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn on_catalog_event(&mut self, event: ClientEvent) -> bool {
|
||||
|
|
@ -1673,21 +1690,17 @@ fn start_embedded_asset_server_at(
|
|||
Ok((server, token))
|
||||
}
|
||||
|
||||
/// Stop flag for the single in-process publish loop. A `static` (not an
|
||||
/// `Arc`) because `watch::run` borrows it for the thread's whole life and
|
||||
/// there is at most one loop per process.
|
||||
static PUBLISH_STOP: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
/// Owns the publisher thread; dropping the store stops and joins it.
|
||||
struct PublishLoop {
|
||||
join: Option<std::thread::JoinHandle<()>>,
|
||||
stop: Arc<AtomicBool>,
|
||||
task: Option<TaskHandle<()>>,
|
||||
}
|
||||
|
||||
impl Drop for PublishLoop {
|
||||
fn drop(&mut self) {
|
||||
PUBLISH_STOP.store(true, Ordering::Release);
|
||||
if let Some(join) = self.join.take() {
|
||||
let _ = join.join();
|
||||
self.stop.store(true, Ordering::Release);
|
||||
if let Some(task) = self.task.take() {
|
||||
task.detach();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1697,6 +1710,7 @@ impl Drop for PublishLoop {
|
|||
/// slow or refused connection can never stall the UI, and every failure is
|
||||
/// a log line — never a panic.
|
||||
fn start_publish_loop(
|
||||
spawner: &ThreadSpawner,
|
||||
server: &makepad_asset_store::AssetServer,
|
||||
token: &str,
|
||||
library_dir: PathBuf,
|
||||
|
|
@ -1705,10 +1719,14 @@ fn start_publish_loop(
|
|||
let server_id = server.server_id();
|
||||
let token = token.to_string();
|
||||
let cache = asset_ui_home().join("publish-cache");
|
||||
PUBLISH_STOP.store(false, Ordering::Release);
|
||||
let join = std::thread::Builder::new()
|
||||
.name("asset-ui-publish".to_string())
|
||||
.spawn(move || {
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let worker_stop = stop.clone();
|
||||
let task = spawner.spawn_worker(
|
||||
ThreadOptions {
|
||||
name: Some("asset-ui-publish".into()),
|
||||
..Default::default()
|
||||
},
|
||||
move || {
|
||||
let mut config = makepad_asset_client::ClientConfig::new(cache);
|
||||
config.token = Some(token);
|
||||
let mut client = match makepad_asset_client::AssetClient::connect(
|
||||
|
|
@ -1736,12 +1754,13 @@ fn start_publish_loop(
|
|||
// Log publications, failures and retries; out-of-scope rows
|
||||
// (the pack-import bulk) stay silent by design.
|
||||
true,
|
||||
&PUBLISH_STOP,
|
||||
&worker_stop,
|
||||
);
|
||||
log!("publish loop: stopped");
|
||||
});
|
||||
match join {
|
||||
Ok(join) => Some(PublishLoop { join: Some(join) }),
|
||||
},
|
||||
);
|
||||
match task {
|
||||
Ok(task) => Some(PublishLoop { stop, task: Some(task) }),
|
||||
Err(error) => {
|
||||
log!("publish loop: could not spawn: {error}");
|
||||
None
|
||||
|
|
@ -1749,19 +1768,17 @@ fn start_publish_loop(
|
|||
}
|
||||
}
|
||||
|
||||
/// Stop flag for the single in-process observe loop.
|
||||
static OBSERVE_STOP: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
/// Owns the observer thread; dropping the store stops and joins it.
|
||||
struct ObserveLoop {
|
||||
join: Option<std::thread::JoinHandle<()>>,
|
||||
stop: Arc<AtomicBool>,
|
||||
task: Option<TaskHandle<()>>,
|
||||
}
|
||||
|
||||
impl Drop for ObserveLoop {
|
||||
fn drop(&mut self) {
|
||||
OBSERVE_STOP.store(true, Ordering::Release);
|
||||
if let Some(join) = self.join.take() {
|
||||
let _ = join.join();
|
||||
self.stop.store(true, Ordering::Release);
|
||||
if let Some(task) = self.task.take() {
|
||||
task.detach();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1789,6 +1806,7 @@ fn observe_origins() -> Vec<PathBuf> {
|
|||
/// Observe effect-document origins into the server this process hosts.
|
||||
/// Connect + watch happen on the thread; every failure is a log line.
|
||||
fn start_observe_loop(
|
||||
spawner: &ThreadSpawner,
|
||||
server: &makepad_asset_store::AssetServer,
|
||||
token: &str,
|
||||
) -> Option<ObserveLoop> {
|
||||
|
|
@ -1797,10 +1815,14 @@ fn start_observe_loop(
|
|||
let token = token.to_string();
|
||||
let cache = asset_ui_home().join("observe-cache");
|
||||
let origins = observe_origins();
|
||||
OBSERVE_STOP.store(false, Ordering::Release);
|
||||
let join = std::thread::Builder::new()
|
||||
.name("asset-ui-observe".to_string())
|
||||
.spawn(move || {
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let worker_stop = stop.clone();
|
||||
let task = spawner.spawn_worker(
|
||||
ThreadOptions {
|
||||
name: Some("asset-ui-observe".into()),
|
||||
..Default::default()
|
||||
},
|
||||
move || {
|
||||
let mut config = makepad_asset_client::ClientConfig::new(cache);
|
||||
config.token = Some(token);
|
||||
let mut client = match makepad_asset_client::AssetClient::connect(
|
||||
|
|
@ -1817,12 +1839,13 @@ fn start_observe_loop(
|
|||
makepad_asset_store::observe::run(
|
||||
&mut client,
|
||||
&makepad_asset_store::observe::ObserveConfig::vjfx(origins),
|
||||
&OBSERVE_STOP,
|
||||
&worker_stop,
|
||||
);
|
||||
log!("observe loop: stopped");
|
||||
});
|
||||
match join {
|
||||
Ok(join) => Some(ObserveLoop { join: Some(join) }),
|
||||
},
|
||||
);
|
||||
match task {
|
||||
Ok(task) => Some(ObserveLoop { stop, task: Some(task) }),
|
||||
Err(error) => {
|
||||
log!("observe loop: could not spawn: {error}");
|
||||
None
|
||||
|
|
@ -2078,6 +2101,13 @@ mod tests {
|
|||
namespace: "game".into(),
|
||||
kind: None,
|
||||
title: title.into(),
|
||||
creator: String::new(),
|
||||
artist: String::new(),
|
||||
artist_url: String::new(),
|
||||
album: String::new(),
|
||||
source_url: String::new(),
|
||||
license: String::new(),
|
||||
license_url: String::new(),
|
||||
snippet: String::new(),
|
||||
score: 0,
|
||||
live: true,
|
||||
|
|
@ -2368,6 +2398,14 @@ mod tests {
|
|||
(server, token)
|
||||
}
|
||||
|
||||
fn ensure_test_runtime(store: &mut AssetStore) {
|
||||
if store.pool.is_none() {
|
||||
let cx = makepad_widgets::Cx::new(Box::new(|_, _| {}));
|
||||
store.pool = Some(cx.task_pool());
|
||||
store.spawner = Some(cx.thread_spawner());
|
||||
}
|
||||
}
|
||||
|
||||
/// Publish one minimal real asset directly (the synchronous
|
||||
/// `AssetClient`, not through `AssetStore` — publishing is the import
|
||||
/// pipeline's job, already covered elsewhere; this test starts from
|
||||
|
|
@ -2412,6 +2450,7 @@ mod tests {
|
|||
/// `AssetStore::poll()` loop (no discovery — explicit endpoints), and
|
||||
/// wait for the initial auto-search `poll()` fires on connect to land.
|
||||
fn connect_store_to(store: &mut AssetStore, server: &makepad_asset_store::AssetServer, token: &str) {
|
||||
ensure_test_runtime(store);
|
||||
let config = SessionConfig {
|
||||
endpoints: Some(ApiEndpoints { control: server.control_addr(), data: server.data_addr() }),
|
||||
server_id: Some(server.server_id()),
|
||||
|
|
@ -2448,6 +2487,7 @@ mod tests {
|
|||
secs: u64,
|
||||
ready: F,
|
||||
) {
|
||||
ensure_test_runtime(store);
|
||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(secs);
|
||||
loop {
|
||||
store.poll();
|
||||
|
|
|
|||
|
|
@ -1,15 +1,16 @@
|
|||
//! Artifact playback + waveform strip.
|
||||
//!
|
||||
//! Same shape as the sandbox's `VideoAudio` mixer (apps/sandbox/src/
|
||||
//! video_player.rs): a process-global resampling stereo queue mixed
|
||||
//! additively from the `cx.audio_output` callback, so playback needs no
|
||||
//! plumbing through the widget tree. The service emits PCM16 WAV (kokoro:
|
||||
//! mono 24kHz, sa3-sfx: stereo 44.1kHz), decoded here with a minimal RIFF
|
||||
//! parser (libs/asset/ai wav.rs is encode-only).
|
||||
//! The UI owns a transport handle and the device callback owns the playback
|
||||
//! engine. Commands and atomic snapshots cross between them; neither side
|
||||
//! waits on a mutex. The service emits PCM16 WAV (kokoro: mono 24kHz,
|
||||
//! sa3-sfx: stereo 44.1kHz), decoded here with a minimal RIFF parser
|
||||
//! (libs/asset/ai wav.rs is encode-only).
|
||||
|
||||
use makepad_widgets::makepad_platform::audio::AudioBuffer;
|
||||
use makepad_widgets::makepad_platform::thread::{Lane, TaskHandle, TaskPool};
|
||||
use std::cell::RefCell;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, LazyLock, Mutex};
|
||||
use std::sync::{mpsc, Arc};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct WavPcm {
|
||||
|
|
@ -103,24 +104,22 @@ pub fn parse_wav(bytes: &[u8]) -> Result<WavPcm, String> {
|
|||
/// advances it; UI code only loads, pauses and seeks.
|
||||
const FP_ONE: u64 = 1 << 32;
|
||||
|
||||
struct WavMixer {
|
||||
clip: Mutex<Option<Arc<WavPcm>>>,
|
||||
struct AudioSnapshot {
|
||||
cursor_fp: AtomicU64,
|
||||
playing: AtomicBool,
|
||||
ack: AtomicU64,
|
||||
}
|
||||
|
||||
impl Default for WavMixer {
|
||||
impl Default for AudioSnapshot {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
clip: Mutex::new(None),
|
||||
cursor_fp: AtomicU64::new(0),
|
||||
playing: AtomicBool::new(false),
|
||||
ack: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static WAV_MIXER: LazyLock<WavMixer> = LazyLock::new(WavMixer::default);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Separated layers ("split audio layers")
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -150,33 +149,301 @@ pub const STEM_LANES: usize = 4;
|
|||
/// files are published and fetched in, so no index ever has to be remapped.
|
||||
pub const STEM_LANE_NAMES: [&str; STEM_LANES] = ["drums", "bass", "vocals", "other"];
|
||||
|
||||
struct StemMixer {
|
||||
lanes: Mutex<Option<Arc<[StemPcm; STEM_LANES]>>>,
|
||||
/// Per-lane mute. Read by the audio callback, written by the UI.
|
||||
mute: [AtomicBool; STEM_LANES],
|
||||
/// Whether the layers are what the transport plays. Kept separate from
|
||||
/// the lock so the callback can tell "no stems" from "stems, but the UI
|
||||
/// holds the lock this quantum" — the second must be silence, not a
|
||||
/// blip of the mixed track.
|
||||
active: AtomicBool,
|
||||
/// The clip generation these layers belong to. The mixed audio and the
|
||||
/// four stems arrive on two different workers in either order; this is
|
||||
/// what lets the second one to land know it is the same track.
|
||||
generation: AtomicU64,
|
||||
enum AudioCommand {
|
||||
InstallClip {
|
||||
serial: u64,
|
||||
clip: Arc<WavPcm>,
|
||||
},
|
||||
ClearClip {
|
||||
serial: u64,
|
||||
},
|
||||
InstallStems {
|
||||
serial: u64,
|
||||
lanes: Arc<[StemPcm; STEM_LANES]>,
|
||||
},
|
||||
ClearStems {
|
||||
serial: u64,
|
||||
},
|
||||
Play {
|
||||
serial: u64,
|
||||
},
|
||||
Pause {
|
||||
serial: u64,
|
||||
},
|
||||
Stop {
|
||||
serial: u64,
|
||||
},
|
||||
Seek {
|
||||
serial: u64,
|
||||
cursor_fp: u64,
|
||||
},
|
||||
MuteLane {
|
||||
serial: u64,
|
||||
lane: usize,
|
||||
muted: bool,
|
||||
},
|
||||
}
|
||||
|
||||
impl Default for StemMixer {
|
||||
fn default() -> Self {
|
||||
enum RetiredAudio {
|
||||
Clip(Arc<WavPcm>),
|
||||
Stems(Arc<[StemPcm; STEM_LANES]>),
|
||||
}
|
||||
|
||||
struct PendingDecode {
|
||||
generation: u64,
|
||||
task: TaskHandle<Result<WavPcm, String>>,
|
||||
}
|
||||
|
||||
/// UI-thread transport handle. It owns the requested state and communicates
|
||||
/// with the realtime callback exclusively through commands and atomics.
|
||||
struct AudioMixer {
|
||||
commands: mpsc::Sender<AudioCommand>,
|
||||
retired: mpsc::Receiver<RetiredAudio>,
|
||||
snapshot: Arc<AudioSnapshot>,
|
||||
engine: Option<AudioEngine>,
|
||||
clip: Option<Arc<WavPcm>>,
|
||||
stems: Option<Arc<[StemPcm; STEM_LANES]>>,
|
||||
stem_generation: u64,
|
||||
muted: [bool; STEM_LANES],
|
||||
cursor_fp: u64,
|
||||
playing: bool,
|
||||
serial: u64,
|
||||
load_generation: u64,
|
||||
pending_decodes: Vec<PendingDecode>,
|
||||
}
|
||||
|
||||
/// Realtime-owned state. Once installed in `cx.audio_output`, only the audio
|
||||
/// callback touches these payloads and cursors.
|
||||
pub struct AudioEngine {
|
||||
commands: mpsc::Receiver<AudioCommand>,
|
||||
retired: mpsc::Sender<RetiredAudio>,
|
||||
snapshot: Arc<AudioSnapshot>,
|
||||
clip: Option<Arc<WavPcm>>,
|
||||
stems: Option<Arc<[StemPcm; STEM_LANES]>>,
|
||||
muted: [bool; STEM_LANES],
|
||||
cursor_fp: u64,
|
||||
playing: bool,
|
||||
ack: u64,
|
||||
}
|
||||
|
||||
impl AudioMixer {
|
||||
fn new() -> Self {
|
||||
let (command_tx, command_rx) = mpsc::channel();
|
||||
let (retired_tx, retired_rx) = mpsc::channel();
|
||||
let snapshot = Arc::new(AudioSnapshot::default());
|
||||
let engine = AudioEngine {
|
||||
commands: command_rx,
|
||||
retired: retired_tx,
|
||||
snapshot: snapshot.clone(),
|
||||
clip: None,
|
||||
stems: None,
|
||||
muted: [false; STEM_LANES],
|
||||
cursor_fp: 0,
|
||||
playing: false,
|
||||
ack: 0,
|
||||
};
|
||||
Self {
|
||||
lanes: Mutex::new(None),
|
||||
mute: Default::default(),
|
||||
active: AtomicBool::new(false),
|
||||
generation: AtomicU64::new(u64::MAX),
|
||||
commands: command_tx,
|
||||
retired: retired_rx,
|
||||
snapshot,
|
||||
engine: Some(engine),
|
||||
clip: None,
|
||||
stems: None,
|
||||
stem_generation: u64::MAX,
|
||||
muted: [false; STEM_LANES],
|
||||
cursor_fp: 0,
|
||||
playing: false,
|
||||
serial: 0,
|
||||
load_generation: 0,
|
||||
pending_decodes: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn take_engine(&mut self) -> Option<AudioEngine> {
|
||||
self.engine.take()
|
||||
}
|
||||
|
||||
fn next_serial(&mut self) -> u64 {
|
||||
self.serial = self.serial.wrapping_add(1).max(1);
|
||||
self.serial
|
||||
}
|
||||
|
||||
fn send(&self, command: AudioCommand) {
|
||||
let _ = self.commands.send(command);
|
||||
}
|
||||
|
||||
fn refresh_snapshot(&mut self) {
|
||||
let ack = self.snapshot.ack.load(Ordering::Acquire);
|
||||
if ack >= self.serial {
|
||||
self.cursor_fp = self.snapshot.cursor_fp.load(Ordering::Acquire);
|
||||
self.playing = self.snapshot.playing.load(Ordering::Acquire);
|
||||
}
|
||||
}
|
||||
|
||||
fn pump(&mut self) {
|
||||
self.refresh_snapshot();
|
||||
for retired in self.retired.try_iter() {
|
||||
match retired {
|
||||
RetiredAudio::Clip(clip) => drop(clip),
|
||||
RetiredAudio::Stems(stems) => drop(stems),
|
||||
}
|
||||
}
|
||||
let pending_decodes = std::mem::take(&mut self.pending_decodes);
|
||||
let mut waiting = Vec::with_capacity(pending_decodes.len());
|
||||
for mut pending in pending_decodes {
|
||||
let Some(result) = pending.task.try_take() else {
|
||||
waiting.push(pending);
|
||||
continue;
|
||||
};
|
||||
match result {
|
||||
Ok(Ok(pcm)) if pending.generation == self.load_generation => {
|
||||
self.install(pcm, pending.generation);
|
||||
}
|
||||
Ok(Err(error)) if pending.generation == self.load_generation => {
|
||||
makepad_widgets::log!("audio decode failed: {error}");
|
||||
}
|
||||
Err(error) if pending.generation == self.load_generation => {
|
||||
makepad_widgets::log!("audio decode task failed: {error}");
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
self.pending_decodes = waiting;
|
||||
}
|
||||
|
||||
fn clear_stems(&mut self) {
|
||||
self.stems = None;
|
||||
self.stem_generation = u64::MAX;
|
||||
self.muted = [false; STEM_LANES];
|
||||
let serial = self.next_serial();
|
||||
self.send(AudioCommand::ClearStems { serial });
|
||||
}
|
||||
|
||||
fn clear(&mut self) {
|
||||
self.clear_stems();
|
||||
self.clip = None;
|
||||
self.cursor_fp = 0;
|
||||
self.playing = false;
|
||||
let serial = self.next_serial();
|
||||
self.send(AudioCommand::ClearClip { serial });
|
||||
}
|
||||
|
||||
fn install(&mut self, pcm: WavPcm, generation: u64) -> bool {
|
||||
if self.load_generation != generation {
|
||||
return false;
|
||||
}
|
||||
if pcm.frames.is_empty() || pcm.sample_rate == 0 {
|
||||
self.clear();
|
||||
return false;
|
||||
}
|
||||
if self.stem_generation != generation {
|
||||
self.clear_stems();
|
||||
}
|
||||
let clip = Arc::new(pcm);
|
||||
self.clip = Some(clip.clone());
|
||||
self.cursor_fp = 0;
|
||||
self.playing = false;
|
||||
let serial = self.next_serial();
|
||||
self.send(AudioCommand::InstallClip { serial, clip });
|
||||
true
|
||||
}
|
||||
|
||||
fn load(&mut self, pcm: WavPcm) -> bool {
|
||||
self.load_generation = self.load_generation.wrapping_add(1);
|
||||
self.install(pcm, self.load_generation)
|
||||
}
|
||||
|
||||
fn set_stems(&mut self, lanes: [StemPcm; STEM_LANES], generation: u64) -> bool {
|
||||
if self.load_generation != generation {
|
||||
return false;
|
||||
}
|
||||
if lanes
|
||||
.iter()
|
||||
.any(|lane| lane.frames.is_empty() || lane.sample_rate == 0)
|
||||
{
|
||||
self.clear_stems();
|
||||
return false;
|
||||
}
|
||||
let lanes = Arc::new(lanes);
|
||||
self.stems = Some(lanes.clone());
|
||||
self.stem_generation = generation;
|
||||
self.muted = [false; STEM_LANES];
|
||||
let serial = self.next_serial();
|
||||
self.send(AudioCommand::InstallStems { serial, lanes });
|
||||
true
|
||||
}
|
||||
|
||||
fn play(&mut self) {
|
||||
let Some(clip) = self.clip.as_ref() else {
|
||||
return;
|
||||
};
|
||||
let end = (clip.frames.len() as u64) << 32;
|
||||
if self.cursor_fp >= end {
|
||||
self.cursor_fp = 0;
|
||||
}
|
||||
self.playing = true;
|
||||
let serial = self.next_serial();
|
||||
self.send(AudioCommand::Play { serial });
|
||||
}
|
||||
|
||||
fn pause(&mut self) {
|
||||
self.playing = false;
|
||||
let serial = self.next_serial();
|
||||
self.send(AudioCommand::Pause { serial });
|
||||
}
|
||||
|
||||
fn stop(&mut self) {
|
||||
self.playing = false;
|
||||
self.cursor_fp = 0;
|
||||
let serial = self.next_serial();
|
||||
self.send(AudioCommand::Stop { serial });
|
||||
}
|
||||
|
||||
fn seek_fraction(&mut self, fraction: f64) {
|
||||
let Some(clip) = self.clip.as_ref() else {
|
||||
return;
|
||||
};
|
||||
let frame = (fraction.clamp(0.0, 1.0) * clip.frames.len() as f64) as u64;
|
||||
self.cursor_fp = frame.min(clip.frames.len() as u64) << 32;
|
||||
let serial = self.next_serial();
|
||||
self.send(AudioCommand::Seek {
|
||||
serial,
|
||||
cursor_fp: self.cursor_fp,
|
||||
});
|
||||
}
|
||||
|
||||
fn set_lane_muted(&mut self, lane: usize, muted: bool) {
|
||||
let Some(slot) = self.muted.get_mut(lane) else {
|
||||
return;
|
||||
};
|
||||
*slot = muted;
|
||||
let serial = self.next_serial();
|
||||
self.send(AudioCommand::MuteLane {
|
||||
serial,
|
||||
lane,
|
||||
muted,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
static STEM_MIXER: LazyLock<StemMixer> = LazyLock::new(StemMixer::default);
|
||||
thread_local! {
|
||||
static AUDIO_MIXER: RefCell<AudioMixer> = RefCell::new(AudioMixer::new());
|
||||
}
|
||||
|
||||
fn with_mixer<T>(f: impl FnOnce(&mut AudioMixer) -> T) -> T {
|
||||
AUDIO_MIXER.with(|mixer| f(&mut mixer.borrow_mut()))
|
||||
}
|
||||
|
||||
/// Move the engine into the app's one audio callback. It is intentionally
|
||||
/// one-shot: there must never be a second owner of realtime state.
|
||||
pub fn take_engine() -> AudioEngine {
|
||||
with_mixer(|mixer| mixer.take_engine()).expect("audio engine is installed once")
|
||||
}
|
||||
|
||||
/// Poll background decodes and reclaim callback-retired payloads on the UI.
|
||||
pub fn pump() {
|
||||
with_mixer(AudioMixer::pump);
|
||||
}
|
||||
|
||||
/// Install four separated layers over the loaded clip. From here the
|
||||
/// transport plays their SUM instead of the mixed track — which is also the
|
||||
|
|
@ -190,63 +457,32 @@ static STEM_MIXER: LazyLock<StemMixer> = LazyLock::new(StemMixer::default);
|
|||
/// Refused (and the layers cleared) when a lane is empty: half a stem set is
|
||||
/// a lie about what the asset carries.
|
||||
pub fn set_stems(lanes: [StemPcm; STEM_LANES], generation: u64) -> bool {
|
||||
if LOAD_GENERATION.load(Ordering::Acquire) != generation {
|
||||
return false;
|
||||
}
|
||||
if lanes
|
||||
.iter()
|
||||
.any(|lane| lane.frames.is_empty() || lane.sample_rate == 0)
|
||||
{
|
||||
clear_stems();
|
||||
return false;
|
||||
}
|
||||
for mute in &STEM_MIXER.mute {
|
||||
mute.store(false, Ordering::Release);
|
||||
}
|
||||
*STEM_MIXER.lanes.lock().unwrap() = Some(Arc::new(lanes));
|
||||
STEM_MIXER.generation.store(generation, Ordering::Release);
|
||||
STEM_MIXER.active.store(true, Ordering::Release);
|
||||
true
|
||||
with_mixer(|mixer| mixer.set_stems(lanes, generation))
|
||||
}
|
||||
|
||||
/// Back to the mixed track. Called whenever the clip changes, so a new
|
||||
/// selection can never play the previous track's layers.
|
||||
pub fn clear_stems() {
|
||||
STEM_MIXER.active.store(false, Ordering::Release);
|
||||
STEM_MIXER.generation.store(u64::MAX, Ordering::Release);
|
||||
*STEM_MIXER.lanes.lock().unwrap() = None;
|
||||
for mute in &STEM_MIXER.mute {
|
||||
mute.store(false, Ordering::Release);
|
||||
}
|
||||
with_mixer(AudioMixer::clear_stems);
|
||||
}
|
||||
|
||||
/// True when the transport is playing separated layers.
|
||||
pub fn stems_ready() -> bool {
|
||||
STEM_MIXER.active.load(Ordering::Acquire)
|
||||
with_mixer(|mixer| mixer.stems.is_some())
|
||||
}
|
||||
|
||||
pub fn lane_muted(lane: usize) -> bool {
|
||||
STEM_MIXER
|
||||
.mute
|
||||
.get(lane)
|
||||
.is_some_and(|mute| mute.load(Ordering::Acquire))
|
||||
with_mixer(|mixer| mixer.muted.get(lane).copied().unwrap_or(false))
|
||||
}
|
||||
|
||||
pub fn set_lane_muted(lane: usize, muted: bool) {
|
||||
if let Some(mute) = STEM_MIXER.mute.get(lane) {
|
||||
mute.store(muted, Ordering::Release);
|
||||
}
|
||||
with_mixer(|mixer| mixer.set_lane_muted(lane, muted));
|
||||
}
|
||||
|
||||
/// Length of the installed layers, for the honest "these are this track's
|
||||
/// stems" check a host wants before it draws the toggles.
|
||||
pub fn stems_seconds() -> f64 {
|
||||
STEM_MIXER
|
||||
.lanes
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_ref()
|
||||
.map_or(0.0, |lanes| lanes[0].seconds())
|
||||
with_mixer(|mixer| mixer.stems.as_ref().map_or(0.0, |lanes| lanes[0].seconds()))
|
||||
}
|
||||
|
||||
/// One lane at a fixed-point cursor, linearly interpolated — the same
|
||||
|
|
@ -276,88 +512,44 @@ fn sample_lane(lane: &StemPcm, cursor: u64) -> (f32, f32) {
|
|||
/// a new clip generation, which invalidates any separated layers installed
|
||||
/// for the previous one.
|
||||
pub fn load(pcm: WavPcm) -> bool {
|
||||
let generation = LOAD_GENERATION.fetch_add(1, Ordering::AcqRel) + 1;
|
||||
install(pcm, generation)
|
||||
}
|
||||
|
||||
/// Install a decoded clip under the generation it was decoded FOR. A newer
|
||||
/// pick having happened in the meantime drops it.
|
||||
///
|
||||
/// The layers are cleared unless they were installed for THIS clip: the
|
||||
/// fetch of a track's stems and the decode of its mixed audio run on two
|
||||
/// workers, and whichever finishes second must not wipe the first.
|
||||
fn install(pcm: WavPcm, generation: u64) -> bool {
|
||||
if LOAD_GENERATION.load(Ordering::Acquire) != generation {
|
||||
return false;
|
||||
}
|
||||
if pcm.frames.is_empty() || pcm.sample_rate == 0 {
|
||||
clear();
|
||||
return false;
|
||||
}
|
||||
if STEM_MIXER.generation.load(Ordering::Acquire) != generation {
|
||||
clear_stems();
|
||||
}
|
||||
WAV_MIXER.playing.store(false, Ordering::Release);
|
||||
*WAV_MIXER.clip.lock().unwrap() = Some(Arc::new(pcm));
|
||||
WAV_MIXER.cursor_fp.store(0, Ordering::Release);
|
||||
true
|
||||
with_mixer(|mixer| mixer.load(pcm))
|
||||
}
|
||||
|
||||
/// Discard the loaded clip and make the transport unavailable.
|
||||
pub fn clear() {
|
||||
clear_stems();
|
||||
WAV_MIXER.playing.store(false, Ordering::Release);
|
||||
*WAV_MIXER.clip.lock().unwrap() = None;
|
||||
WAV_MIXER.cursor_fp.store(0, Ordering::Release);
|
||||
with_mixer(AudioMixer::clear);
|
||||
}
|
||||
|
||||
/// Which load request is current. A user clicking down a list starts several;
|
||||
/// only the newest may install itself, or a slow decode of the track before
|
||||
/// last lands on top of the one they are looking at.
|
||||
static LOAD_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
/// Take a track's bytes — WAV, MP3 or Ogg — and make the transport play them.
|
||||
///
|
||||
/// Decoding happens on a worker: the music library is MP3s, and turning six
|
||||
/// minutes of one into PCM on the frame thread is a visible stall. The mixer
|
||||
/// is process-global, so the worker installs the result itself; there is
|
||||
/// nothing to plumb back through the widget tree.
|
||||
/// Decoding happens in the runtime pool: the music library is MP3s, and
|
||||
/// turning six minutes of one into PCM on the frame thread is a visible
|
||||
/// stall. The UI polls the task and sends the accepted result to the engine.
|
||||
///
|
||||
/// The transport goes unavailable immediately, because the previous track is
|
||||
/// no longer what the well is showing — a stale clip left loaded is a play
|
||||
/// button that plays the wrong song.
|
||||
/// Returns the clip generation this request claimed, which is what a
|
||||
/// side-channel fetch for the same track carries back into [`set_stems`].
|
||||
pub fn load_clip_async(bytes: Vec<u8>) -> u64 {
|
||||
let generation = LOAD_GENERATION.fetch_add(1, Ordering::AcqRel) + 1;
|
||||
clear();
|
||||
let bytes = Arc::new(bytes);
|
||||
let spawned = std::thread::Builder::new()
|
||||
.name("asset-ui-audio-decode".into())
|
||||
.spawn({
|
||||
let bytes = Arc::clone(&bytes);
|
||||
move || {
|
||||
let Ok(pcm) = decode_clip(&bytes) else {
|
||||
return;
|
||||
};
|
||||
// A newer pick happened while this was decoding: drop it.
|
||||
install(pcm, generation);
|
||||
}
|
||||
});
|
||||
if spawned.is_err() {
|
||||
// No worker to be had: decode here rather than leave a dead
|
||||
// transport under a drawn waveform.
|
||||
if let Ok(pcm) = decode_clip(&bytes) {
|
||||
install(pcm, generation);
|
||||
}
|
||||
pub fn load_clip_async(pool: &TaskPool, bytes: Vec<u8>) -> u64 {
|
||||
with_mixer(|mixer| {
|
||||
mixer.load_generation = mixer.load_generation.wrapping_add(1);
|
||||
let generation = mixer.load_generation;
|
||||
mixer.clear();
|
||||
match pool.submit(Lane::Heavy, move || decode_clip(&bytes)) {
|
||||
Ok(task) => mixer
|
||||
.pending_decodes
|
||||
.push(PendingDecode { generation, task }),
|
||||
Err(error) => makepad_widgets::log!("audio decode job refused: {error}"),
|
||||
}
|
||||
generation
|
||||
})
|
||||
}
|
||||
|
||||
/// The clip generation currently claimed. A side-channel fetch records it
|
||||
/// with the request and hands it back to [`set_stems`].
|
||||
pub fn clip_generation() -> u64 {
|
||||
LOAD_GENERATION.load(Ordering::Acquire)
|
||||
with_mixer(|mixer| mixer.load_generation)
|
||||
}
|
||||
|
||||
/// Any container the catalog carries, in the mixer's shape. RIFF is parsed
|
||||
|
|
@ -388,48 +580,45 @@ pub fn decode_clip(bytes: &[u8]) -> Result<WavPcm, String> {
|
|||
|
||||
/// Start or resume. Starting from the end restarts at zero.
|
||||
pub fn play() {
|
||||
let clip = WAV_MIXER.clip.lock().unwrap();
|
||||
let Some(clip) = clip.as_ref() else { return };
|
||||
let end = (clip.frames.len() as u64) << 32;
|
||||
if WAV_MIXER.cursor_fp.load(Ordering::Acquire) >= end {
|
||||
WAV_MIXER.cursor_fp.store(0, Ordering::Release);
|
||||
}
|
||||
WAV_MIXER.playing.store(true, Ordering::Release);
|
||||
with_mixer(AudioMixer::play);
|
||||
}
|
||||
|
||||
pub fn pause() {
|
||||
WAV_MIXER.playing.store(false, Ordering::Release);
|
||||
with_mixer(AudioMixer::pause);
|
||||
}
|
||||
|
||||
/// Stop returns to the start but retains the decoded clip for replay.
|
||||
pub fn stop() {
|
||||
pause();
|
||||
WAV_MIXER.cursor_fp.store(0, Ordering::Release);
|
||||
with_mixer(AudioMixer::stop);
|
||||
}
|
||||
|
||||
pub fn is_ready() -> bool {
|
||||
WAV_MIXER.clip.lock().unwrap().is_some()
|
||||
with_mixer(|mixer| mixer.clip.is_some())
|
||||
}
|
||||
|
||||
pub fn is_playing() -> bool {
|
||||
WAV_MIXER.playing.load(Ordering::Acquire) && !at_end()
|
||||
with_mixer(|mixer| {
|
||||
mixer.refresh_snapshot();
|
||||
let end = mixer
|
||||
.clip
|
||||
.as_ref()
|
||||
.map_or(0, |clip| (clip.frames.len() as u64) << 32);
|
||||
mixer.playing && mixer.cursor_fp < end
|
||||
})
|
||||
}
|
||||
|
||||
pub fn duration_secs() -> f64 {
|
||||
WAV_MIXER
|
||||
.clip
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_ref()
|
||||
.map_or(0.0, |clip| clip.seconds())
|
||||
with_mixer(|mixer| mixer.clip.as_ref().map_or(0.0, |clip| clip.seconds()))
|
||||
}
|
||||
|
||||
/// Truthful device-clocked playhead, never derived from a UI timer.
|
||||
pub fn playhead_secs() -> f64 {
|
||||
let clip = WAV_MIXER.clip.lock().unwrap();
|
||||
let Some(clip) = clip.as_ref() else { return 0.0 };
|
||||
(WAV_MIXER.cursor_fp.load(Ordering::Acquire) as f64 / FP_ONE as f64)
|
||||
/ clip.sample_rate as f64
|
||||
with_mixer(|mixer| {
|
||||
mixer.refresh_snapshot();
|
||||
mixer.clip.as_ref().map_or(0.0, |clip| {
|
||||
(mixer.cursor_fp as f64 / FP_ONE as f64) / clip.sample_rate as f64
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// Normalized playhead across the loaded clip for the waveform overlay:
|
||||
|
|
@ -443,19 +632,18 @@ pub fn playhead_fraction() -> f64 {
|
|||
}
|
||||
|
||||
pub fn at_end() -> bool {
|
||||
let clip = WAV_MIXER.clip.lock().unwrap();
|
||||
let Some(clip) = clip.as_ref() else { return false };
|
||||
WAV_MIXER.cursor_fp.load(Ordering::Acquire) >= (clip.frames.len() as u64) << 32
|
||||
with_mixer(|mixer| {
|
||||
mixer.refresh_snapshot();
|
||||
mixer
|
||||
.clip
|
||||
.as_ref()
|
||||
.is_some_and(|clip| mixer.cursor_fp >= (clip.frames.len() as u64) << 32)
|
||||
})
|
||||
}
|
||||
|
||||
/// Sample-accurate fractional seek, clamped to the decoded clip.
|
||||
pub fn seek_fraction(frac: f64) {
|
||||
let clip = WAV_MIXER.clip.lock().unwrap();
|
||||
let Some(clip) = clip.as_ref() else { return };
|
||||
let frame = (frac.clamp(0.0, 1.0) * clip.frames.len() as f64) as u64;
|
||||
WAV_MIXER
|
||||
.cursor_fp
|
||||
.store((frame.min(clip.frames.len() as u64)) << 32, Ordering::Release);
|
||||
with_mixer(|mixer| mixer.seek_fraction(frac));
|
||||
}
|
||||
|
||||
/// Long-form threshold for the audition policy below: at/under this a voice
|
||||
|
|
@ -484,58 +672,130 @@ pub fn format_time(secs: f64) -> String {
|
|||
format!("{minutes}:{:04.1}", secs - minutes as f64 * 60.0)
|
||||
}
|
||||
|
||||
/// One additive source in the app's single `cx.audio_output` callback.
|
||||
/// The callback never blocks on a UI load/seek: a contended quantum is silent.
|
||||
pub fn mix_into(output: &mut AudioBuffer, device_rate: f64) {
|
||||
if !WAV_MIXER.playing.load(Ordering::Acquire) || device_rate <= 0.0 {
|
||||
impl AudioEngine {
|
||||
fn retire(&self, retired: RetiredAudio) {
|
||||
let _ = self.retired.send(retired);
|
||||
}
|
||||
|
||||
fn drain_commands(&mut self) {
|
||||
while let Ok(command) = self.commands.try_recv() {
|
||||
let serial = match command {
|
||||
AudioCommand::InstallClip { serial, clip } => {
|
||||
if let Some(old) = self.clip.replace(clip) {
|
||||
self.retire(RetiredAudio::Clip(old));
|
||||
}
|
||||
self.cursor_fp = 0;
|
||||
self.playing = false;
|
||||
serial
|
||||
}
|
||||
AudioCommand::ClearClip { serial } => {
|
||||
if let Some(old) = self.clip.take() {
|
||||
self.retire(RetiredAudio::Clip(old));
|
||||
}
|
||||
self.cursor_fp = 0;
|
||||
self.playing = false;
|
||||
serial
|
||||
}
|
||||
AudioCommand::InstallStems { serial, lanes } => {
|
||||
if let Some(old) = self.stems.replace(lanes) {
|
||||
self.retire(RetiredAudio::Stems(old));
|
||||
}
|
||||
self.muted = [false; STEM_LANES];
|
||||
serial
|
||||
}
|
||||
AudioCommand::ClearStems { serial } => {
|
||||
if let Some(old) = self.stems.take() {
|
||||
self.retire(RetiredAudio::Stems(old));
|
||||
}
|
||||
self.muted = [false; STEM_LANES];
|
||||
serial
|
||||
}
|
||||
AudioCommand::Play { serial } => {
|
||||
if let Some(clip) = self.clip.as_ref() {
|
||||
let end = (clip.frames.len() as u64) << 32;
|
||||
if self.cursor_fp >= end {
|
||||
self.cursor_fp = 0;
|
||||
}
|
||||
self.playing = true;
|
||||
}
|
||||
serial
|
||||
}
|
||||
AudioCommand::Pause { serial } => {
|
||||
self.playing = false;
|
||||
serial
|
||||
}
|
||||
AudioCommand::Stop { serial } => {
|
||||
self.playing = false;
|
||||
self.cursor_fp = 0;
|
||||
serial
|
||||
}
|
||||
AudioCommand::Seek { serial, cursor_fp } => {
|
||||
self.cursor_fp = self.clip.as_ref().map_or(0, |clip| {
|
||||
cursor_fp.min((clip.frames.len() as u64) << 32)
|
||||
});
|
||||
serial
|
||||
}
|
||||
AudioCommand::MuteLane {
|
||||
serial,
|
||||
lane,
|
||||
muted,
|
||||
} => {
|
||||
if let Some(slot) = self.muted.get_mut(lane) {
|
||||
*slot = muted;
|
||||
}
|
||||
serial
|
||||
}
|
||||
};
|
||||
self.ack = serial;
|
||||
}
|
||||
}
|
||||
|
||||
fn publish(&self) {
|
||||
self.snapshot.cursor_fp.store(self.cursor_fp, Ordering::Relaxed);
|
||||
self.snapshot.playing.store(self.playing, Ordering::Relaxed);
|
||||
self.snapshot.ack.store(self.ack, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Add this transport to the app's output. Commands are drained first;
|
||||
/// no application lock or wait occurs on the realtime callback.
|
||||
pub fn mix_into(&mut self, output: &mut AudioBuffer, device_rate: f64) {
|
||||
self.drain_commands();
|
||||
if !self.playing || device_rate <= 0.0 {
|
||||
self.publish();
|
||||
return;
|
||||
}
|
||||
let Ok(clip) = WAV_MIXER.clip.try_lock() else { return };
|
||||
let Some(clip) = clip.as_ref() else {
|
||||
WAV_MIXER.playing.store(false, Ordering::Release);
|
||||
let Some(clip) = self.clip.as_ref() else {
|
||||
self.playing = false;
|
||||
self.publish();
|
||||
return;
|
||||
};
|
||||
let end = (clip.frames.len() as u64) << 32;
|
||||
let mut cursor = WAV_MIXER.cursor_fp.load(Ordering::Acquire);
|
||||
if cursor >= end {
|
||||
WAV_MIXER.playing.store(false, Ordering::Release);
|
||||
if self.cursor_fp >= end {
|
||||
self.playing = false;
|
||||
self.publish();
|
||||
return;
|
||||
}
|
||||
let step = ((clip.sample_rate as f64 / device_rate) * FP_ONE as f64) as u64;
|
||||
if step == 0 {
|
||||
self.publish();
|
||||
return;
|
||||
}
|
||||
const GAIN: f32 = 0.9;
|
||||
|
||||
// Separated layers REPLACE the mixed track while they are installed.
|
||||
// A contended lane lock is silence for this quantum, never a blip of
|
||||
// the original — the same rule the clip lock above follows.
|
||||
if STEM_MIXER.active.load(Ordering::Acquire) {
|
||||
let Ok(guard) = STEM_MIXER.lanes.try_lock() else { return };
|
||||
let Some(lanes) = guard.as_ref() else { return };
|
||||
// ONE cursor for all four lanes — they cannot drift from each other
|
||||
// — re-derived from the track cursor at every quantum, so they
|
||||
// cannot drift from the timeline the waveform and the transport
|
||||
// draw either. The layers are at the model's rate; the clip may not
|
||||
// be, hence the second step.
|
||||
if let Some(lanes) = self.stems.as_ref() {
|
||||
let stem_rate = lanes[0].sample_rate.max(1) as f64;
|
||||
let stem_step = ((stem_rate / device_rate) * FP_ONE as f64) as u64;
|
||||
let secs = (cursor as f64 / FP_ONE as f64) / clip.sample_rate as f64;
|
||||
let secs = (self.cursor_fp as f64 / FP_ONE as f64) / clip.sample_rate as f64;
|
||||
let mut stem_cursor = (secs * stem_rate * FP_ONE as f64) as u64;
|
||||
// The mute flags are read ONCE per quantum, not per sample: a
|
||||
// toggle lands on the next buffer, which is inaudible, and the
|
||||
// callback stays free of per-sample atomics.
|
||||
let audible: [bool; STEM_LANES] =
|
||||
std::array::from_fn(|lane| !STEM_MIXER.mute[lane].load(Ordering::Relaxed));
|
||||
for frame in 0..output.frame_count() {
|
||||
if cursor >= end {
|
||||
WAV_MIXER.playing.store(false, Ordering::Release);
|
||||
cursor = end;
|
||||
if self.cursor_fp >= end {
|
||||
self.playing = false;
|
||||
self.cursor_fp = end;
|
||||
break;
|
||||
}
|
||||
let (mut l, mut r) = (0.0f32, 0.0f32);
|
||||
for (lane, on) in lanes.iter().zip(audible.iter()) {
|
||||
if !on {
|
||||
for (index, lane) in lanes.iter().enumerate() {
|
||||
if self.muted[index] {
|
||||
continue;
|
||||
}
|
||||
let (ll, rr) = sample_lane(lane, stem_cursor);
|
||||
|
|
@ -547,21 +807,22 @@ pub fn mix_into(output: &mut AudioBuffer, device_rate: f64) {
|
|||
for channel in 0..output.channel_count() {
|
||||
output.channel_mut(channel)[frame] += if channel == 0 { l } else { r };
|
||||
}
|
||||
cursor = cursor.saturating_add(step);
|
||||
self.cursor_fp = self.cursor_fp.saturating_add(step);
|
||||
stem_cursor = stem_cursor.saturating_add(stem_step);
|
||||
}
|
||||
WAV_MIXER.cursor_fp.store(cursor.min(end), Ordering::Release);
|
||||
self.cursor_fp = self.cursor_fp.min(end);
|
||||
self.publish();
|
||||
return;
|
||||
}
|
||||
|
||||
for frame in 0..output.frame_count() {
|
||||
if cursor >= end {
|
||||
WAV_MIXER.playing.store(false, Ordering::Release);
|
||||
cursor = end;
|
||||
if self.cursor_fp >= end {
|
||||
self.playing = false;
|
||||
self.cursor_fp = end;
|
||||
break;
|
||||
}
|
||||
let index = (cursor >> 32) as usize;
|
||||
let fraction = (cursor & (FP_ONE - 1)) as f32 / FP_ONE as f32;
|
||||
let index = (self.cursor_fp >> 32) as usize;
|
||||
let fraction = (self.cursor_fp & (FP_ONE - 1)) as f32 / FP_ONE as f32;
|
||||
let next = (index + 1).min(clip.frames.len() - 1);
|
||||
let (al, ar) = clip.frames[index];
|
||||
let (bl, br) = clip.frames[next];
|
||||
|
|
@ -570,9 +831,11 @@ pub fn mix_into(output: &mut AudioBuffer, device_rate: f64) {
|
|||
for channel in 0..output.channel_count() {
|
||||
output.channel_mut(channel)[frame] += if channel == 0 { l } else { r };
|
||||
}
|
||||
cursor = cursor.saturating_add(step);
|
||||
self.cursor_fp = self.cursor_fp.saturating_add(step);
|
||||
}
|
||||
self.cursor_fp = self.cursor_fp.min(end);
|
||||
self.publish();
|
||||
}
|
||||
WAV_MIXER.cursor_fp.store(cursor.min(end), Ordering::Release);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -660,7 +923,10 @@ pub fn waveform_bgra(pcm: &WavPcm, width: usize, height: usize) -> Vec<u32> {
|
|||
mod tests {
|
||||
use super::*;
|
||||
|
||||
static TRANSPORT_TEST_LOCK: Mutex<()> = Mutex::new(());
|
||||
fn reset_transport() -> AudioEngine {
|
||||
AUDIO_MIXER.with(|slot| *slot.borrow_mut() = AudioMixer::new());
|
||||
take_engine()
|
||||
}
|
||||
|
||||
fn transport_pcm() -> WavPcm {
|
||||
WavPcm {
|
||||
|
|
@ -715,7 +981,7 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn transport_is_device_clocked_pauseable_seekable_and_restarts_at_end() {
|
||||
let _serial = TRANSPORT_TEST_LOCK.lock().unwrap();
|
||||
let mut engine = reset_transport();
|
||||
clear();
|
||||
assert!(load(transport_pcm()));
|
||||
assert!(is_ready());
|
||||
|
|
@ -728,7 +994,7 @@ mod tests {
|
|||
assert_eq!(playhead_secs(), 0.0);
|
||||
assert_eq!(playhead_fraction(), 0.0);
|
||||
let mut output = AudioBuffer::new_with_size(2, 2);
|
||||
mix_into(&mut output, 10.0);
|
||||
engine.mix_into(&mut output, 10.0);
|
||||
assert!((playhead_secs() - 0.2).abs() < 1e-9);
|
||||
// The drawn playhead tracks the same device-clocked cursor.
|
||||
assert!((playhead_fraction() - 0.5).abs() < 1e-9);
|
||||
|
|
@ -737,7 +1003,7 @@ mod tests {
|
|||
pause();
|
||||
let paused_at = playhead_secs();
|
||||
let mut silent = AudioBuffer::new_with_size(2, 2);
|
||||
mix_into(&mut silent, 10.0);
|
||||
engine.mix_into(&mut silent, 10.0);
|
||||
assert_eq!(playhead_secs(), paused_at);
|
||||
assert!(silent.channel(0).iter().all(|sample| *sample == 0.0));
|
||||
|
||||
|
|
@ -752,7 +1018,7 @@ mod tests {
|
|||
assert_eq!(playhead_secs(), 0.0);
|
||||
|
||||
let mut to_end = AudioBuffer::new_with_size(8, 2);
|
||||
mix_into(&mut to_end, 10.0);
|
||||
engine.mix_into(&mut to_end, 10.0);
|
||||
assert!(at_end());
|
||||
assert!(!is_playing());
|
||||
clear();
|
||||
|
|
@ -768,7 +1034,7 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn layers_replace_the_mixed_track_and_mute_one_at_a_time() {
|
||||
let _serial = TRANSPORT_TEST_LOCK.lock().unwrap();
|
||||
let mut engine = reset_transport();
|
||||
clear();
|
||||
assert!(!stems_ready(), "no clip, no layers");
|
||||
assert!(load(transport_pcm()));
|
||||
|
|
@ -788,7 +1054,7 @@ mod tests {
|
|||
|
||||
play();
|
||||
let mut all = AudioBuffer::new_with_size(1, 2);
|
||||
mix_into(&mut all, 10.0);
|
||||
engine.mix_into(&mut all, 10.0);
|
||||
assert!(
|
||||
(all.channel(0)[0] - expect(1_000 + 2_000 + 4_000 + 8_000)).abs() < 1e-4,
|
||||
"all four layers sum: {}",
|
||||
|
|
@ -803,7 +1069,7 @@ mod tests {
|
|||
set_lane_muted(2, true);
|
||||
assert!(lane_muted(2) && !lane_muted(0));
|
||||
let mut without_vocals = AudioBuffer::new_with_size(1, 2);
|
||||
mix_into(&mut without_vocals, 10.0);
|
||||
engine.mix_into(&mut without_vocals, 10.0);
|
||||
assert!(
|
||||
(without_vocals.channel(0)[0] - expect(1_000 + 2_000 + 8_000)).abs() < 1e-4,
|
||||
"vocals muted: {}",
|
||||
|
|
@ -816,7 +1082,7 @@ mod tests {
|
|||
set_lane_muted(index, true);
|
||||
}
|
||||
let mut silent = AudioBuffer::new_with_size(1, 2);
|
||||
mix_into(&mut silent, 10.0);
|
||||
engine.mix_into(&mut silent, 10.0);
|
||||
assert!(silent.channel(0)[0].abs() < 1e-6, "{}", silent.channel(0)[0]);
|
||||
|
||||
// Clearing the layers hands playback back to the mixed track.
|
||||
|
|
@ -824,7 +1090,7 @@ mod tests {
|
|||
assert!(!stems_ready());
|
||||
seek_fraction(0.5);
|
||||
let mut mixed = AudioBuffer::new_with_size(1, 2);
|
||||
mix_into(&mut mixed, 10.0);
|
||||
engine.mix_into(&mut mixed, 10.0);
|
||||
assert!(
|
||||
(mixed.channel(0)[0] - 0.8 * GAIN).abs() < 1e-4,
|
||||
"the clip's own third frame: {}",
|
||||
|
|
@ -854,7 +1120,7 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn an_empty_layer_is_refused_rather_than_played_as_a_hole() {
|
||||
let _serial = TRANSPORT_TEST_LOCK.lock().unwrap();
|
||||
let _engine = reset_transport();
|
||||
clear();
|
||||
assert!(load(transport_pcm()));
|
||||
assert!(!set_stems(
|
||||
|
|
@ -909,7 +1175,7 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn empty_clip_is_unavailable() {
|
||||
let _serial = TRANSPORT_TEST_LOCK.lock().unwrap();
|
||||
let _engine = reset_transport();
|
||||
clear();
|
||||
assert!(!load(WavPcm {
|
||||
frames: Vec::new(),
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ use makepad_chat_ui::{ChatFeed, ClientTools, FeedConfig};
|
|||
use makepad_asset_client::dto::ChatToolOutcomeDto;
|
||||
use makepad_asset_client::json::{self, Value};
|
||||
use makepad_asset_client::{ApiEndpoints, ChatAttachment};
|
||||
use makepad_widgets::Cx;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::mpsc::{self, Receiver, Sender};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
|
@ -493,7 +494,13 @@ impl ChatBridge {
|
|||
|
||||
/// The store session is up: open the chat on its broker. The session
|
||||
/// itself is created lazily, on the first turn.
|
||||
pub fn connect(&mut self, endpoints: ApiEndpoints, token: Option<String>, cache: PathBuf) {
|
||||
pub fn connect(
|
||||
&mut self,
|
||||
cx: &Cx,
|
||||
endpoints: ApiEndpoints,
|
||||
token: Option<String>,
|
||||
cache: PathBuf,
|
||||
) {
|
||||
let tools = AppTools {
|
||||
defaults: self.defaults.clone(),
|
||||
fleet: self.fleet.clone(),
|
||||
|
|
@ -503,6 +510,7 @@ impl ChatBridge {
|
|||
self.feed = Some(ChatFeed::start(
|
||||
FeedConfig::new(endpoints, token, cache, "gen", "gen"),
|
||||
Box::new(tools),
|
||||
cx.thread_spawner(),
|
||||
));
|
||||
}
|
||||
|
||||
|
|
@ -939,12 +947,13 @@ mod tests {
|
|||
fn snap(url: &str, domain: &str, id: &str, state: &str) -> BoxSnapshot {
|
||||
BoxSnapshot {
|
||||
base_url: url.into(),
|
||||
health: Some(HealthJson { realtime: None,
|
||||
health: Some(HealthJson { realtime: None, activity: None,
|
||||
service: "makepad-asset-ai".into(),
|
||||
version: "t".into(),
|
||||
gpu: Some("RTX".into()),
|
||||
vram_free_mb: Some(20000),
|
||||
vram_total_mb: Some(24576),
|
||||
vram_usable_mb: None,
|
||||
models_loaded: vec![id.into()],
|
||||
jobs_pending: Some(0),
|
||||
node_id: Some(1),
|
||||
|
|
@ -953,6 +962,7 @@ mod tests {
|
|||
capabilities: Some(vec![domain.into()]),
|
||||
vram_reserve_mb: Some(1024),
|
||||
queue_limit: Some(4),
|
||||
max_job_body_bytes: None,
|
||||
fleet: None,
|
||||
lanes: None,
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -18,13 +18,13 @@ use makepad_asset_client::{
|
|||
AnnotationUpload, ApiEndpoints, AssetClient, ClientConfig,
|
||||
};
|
||||
use makepad_asset_data::{sha256, AssetKind, BlobId};
|
||||
use makepad_widgets::makepad_platform::thread::{Lane, TaskPool};
|
||||
use makepad_widgets::{log, vec3f};
|
||||
use std::collections::BTreeSet;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::mpsc::{self, Receiver, TryRecvError};
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
|
||||
/// Import-card preview strip size.
|
||||
/// How many imported items the LOAD grid remembers a picture for. A grid,
|
||||
|
|
@ -835,6 +835,7 @@ pub struct ImportPage {
|
|||
/// status prefix — without it the bar restarted per pack, which over a
|
||||
/// 38-pack run read as noise.
|
||||
all_run: Option<(usize, usize)>,
|
||||
pool: Option<TaskPool>,
|
||||
}
|
||||
|
||||
impl Default for ImportPage {
|
||||
|
|
@ -854,11 +855,16 @@ impl Default for ImportPage {
|
|||
rx: None,
|
||||
icon_resume: IconResumeGate::default(),
|
||||
all_run: None,
|
||||
pool: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ImportPage {
|
||||
pub fn set_task_pool(&mut self, pool: TaskPool) {
|
||||
self.pool = Some(pool);
|
||||
}
|
||||
|
||||
/// Selected full Kenney kit (name, official page).
|
||||
pub fn selected_pack_id(&self) -> (String, String) {
|
||||
let packs = on_disk_kenney_packs();
|
||||
|
|
@ -1532,9 +1538,10 @@ impl ImportPage {
|
|||
self.icon_resume = icon_resume;
|
||||
self.kenney_phase = ImportPhase::compiling(pack_name.clone());
|
||||
let cancel = self.cancel.clone();
|
||||
thread::Builder::new()
|
||||
.name("asset-ui-kenney-import".into())
|
||||
.spawn(move || {
|
||||
self.pool
|
||||
.as_ref()
|
||||
.ok_or("runtime task pool is not configured")?
|
||||
.submit(Lane::Heavy, move || {
|
||||
let phase = run_kenney_import(
|
||||
&dir,
|
||||
&out,
|
||||
|
|
@ -1548,7 +1555,8 @@ impl ImportPage {
|
|||
);
|
||||
let _ = tx.send(phase);
|
||||
})
|
||||
.map_err(|e| format!("failed to start compile thread: {e}"))?;
|
||||
.map(|handle| handle.detach())
|
||||
.map_err(|e| format!("failed to submit compile job: {e}"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -1580,13 +1588,15 @@ impl ImportPage {
|
|||
self.icon_resume = icon_resume;
|
||||
self.kenney_phase = ImportPhase::compiling("kaykit");
|
||||
let cancel = self.cancel.clone();
|
||||
thread::Builder::new()
|
||||
.name("asset-ui-kaykit-import".into())
|
||||
.spawn(move || {
|
||||
self.pool
|
||||
.as_ref()
|
||||
.ok_or("runtime task pool is not configured")?
|
||||
.submit(Lane::Heavy, move || {
|
||||
let phase = run_kaykit_import(&dir, &out, spec, server, &tx, &cancel, &icon_resume_rx);
|
||||
let _ = tx.send(phase);
|
||||
})
|
||||
.map_err(|e| format!("failed to start KayKit import thread: {e}"))?;
|
||||
.map(|handle| handle.detach())
|
||||
.map_err(|e| format!("failed to submit KayKit import job: {e}"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -1619,9 +1629,10 @@ impl ImportPage {
|
|||
self.kenney_phase = ImportPhase::compiling("all");
|
||||
self.all_run = Some((0, present.len()));
|
||||
let cancel = self.cancel.clone();
|
||||
thread::Builder::new()
|
||||
.name("asset-ui-kenney-import-all".into())
|
||||
.spawn(move || {
|
||||
self.pool
|
||||
.as_ref()
|
||||
.ok_or("runtime task pool is not configured")?
|
||||
.submit(Lane::Heavy, move || {
|
||||
let total = present.len();
|
||||
let mut ok = Vec::new();
|
||||
let mut failed = Vec::new();
|
||||
|
|
@ -1749,7 +1760,8 @@ impl ImportPage {
|
|||
skipped,
|
||||
});
|
||||
})
|
||||
.map_err(|e| format!("failed to start import-all thread: {e}"))?;
|
||||
.map(|handle| handle.detach())
|
||||
.map_err(|e| format!("failed to submit import-all job: {e}"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -4003,6 +4015,12 @@ fn publish_compiled_pack(
|
|||
categories: vec!["kenney".into(), pack_name.to_string()],
|
||||
tags,
|
||||
creator: KENNEY_CREDITS.to_string(),
|
||||
artist: String::new(),
|
||||
artist_url: String::new(),
|
||||
album: String::new(),
|
||||
source_url: String::new(),
|
||||
license: String::new(),
|
||||
license_url: String::new(),
|
||||
generator: "pack_import".into(),
|
||||
backend: "asset-ui".into(),
|
||||
model: pack_name.to_string(),
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ use std::path::{Path, PathBuf};
|
|||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::mpsc::{self, Receiver, TryRecvError};
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
|
||||
use crate::import::{KENNEY_MODULE, KAYKIT_MODULE, NASA_SKY_MODULE, PackModule};
|
||||
use makepad_asset_importer::tdm_zipsync::{
|
||||
|
|
@ -306,6 +305,7 @@ pub struct ClassicImportCard {
|
|||
/// a classic pack publishes real thumbnails and its staging can be
|
||||
/// reclaimed the moment publish succeeds. See [`IconResumeGate`].
|
||||
icon_resume: IconResumeGate,
|
||||
pool: Option<TaskPool>,
|
||||
}
|
||||
|
||||
struct IsoSync {
|
||||
|
|
@ -358,6 +358,7 @@ impl ClassicImportCard {
|
|||
tdm: None,
|
||||
iso: None,
|
||||
icon_resume: IconResumeGate::default(),
|
||||
pool: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -658,6 +659,7 @@ impl ClassicImportCard {
|
|||
path_override: String,
|
||||
server: Option<ServerSession>,
|
||||
) -> Result<(), String> {
|
||||
self.pool = Some(cx.task_pool());
|
||||
if self.compiling() {
|
||||
return Err(format!(
|
||||
"a {} import is already running",
|
||||
|
|
@ -1540,9 +1542,10 @@ impl ClassicImportCard {
|
|||
// until the UI has taken every landing for icon rendering.
|
||||
let (gate, icon_resume_rx) = IconResumeGate::armed();
|
||||
self.icon_resume = gate;
|
||||
thread::Builder::new()
|
||||
.name(format!("asset-ui-{}-import", source.id()))
|
||||
.spawn(move || {
|
||||
self.pool
|
||||
.as_ref()
|
||||
.ok_or("runtime task pool is not configured")?
|
||||
.submit(Lane::Heavy, move || {
|
||||
let phase = run_classic_import(
|
||||
&dir,
|
||||
&out,
|
||||
|
|
@ -1555,7 +1558,8 @@ impl ClassicImportCard {
|
|||
);
|
||||
let _ = tx.send(phase);
|
||||
})
|
||||
.map_err(|e| format!("failed to start classic import thread: {e}"))?;
|
||||
.map(|handle| handle.detach())
|
||||
.map_err(|e| format!("failed to submit classic import job: {e}"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -2267,6 +2271,12 @@ fn publish_classic_pack(
|
|||
categories: vec![source.id().into(), pack_name.to_string()],
|
||||
tags,
|
||||
creator: source.credits().to_string(),
|
||||
artist: String::new(),
|
||||
artist_url: String::new(),
|
||||
album: String::new(),
|
||||
source_url: String::new(),
|
||||
license: String::new(),
|
||||
license_url: String::new(),
|
||||
generator: "classic_import".into(),
|
||||
backend: "asset-ui".into(),
|
||||
model: pack_name.to_string(),
|
||||
|
|
|
|||
|
|
@ -55,7 +55,9 @@ mod import_classic;
|
|||
mod store_content;
|
||||
mod library;
|
||||
mod mask_paint;
|
||||
mod mesh_view;
|
||||
mod mesh_view {
|
||||
pub use makepad_media_view::mesh_view::*;
|
||||
}
|
||||
mod music_page;
|
||||
use crate::mask_paint::{MaskPaint, MaskPaintAction};
|
||||
mod pipeline;
|
||||
|
|
@ -63,7 +65,9 @@ mod runs_chip;
|
|||
mod scheduler;
|
||||
mod store_views;
|
||||
mod thumbnail_renderer;
|
||||
mod video_player;
|
||||
mod video_player {
|
||||
pub use makepad_media_view::{FileVideoPlayer as VideoPlayer, VideoDecoder};
|
||||
}
|
||||
mod webcam;
|
||||
|
||||
use crate::artifact_io::{
|
||||
|
|
@ -244,7 +248,7 @@ use crate::store_views::{
|
|||
StoreListPanel, StoreRow,
|
||||
TileDelete,
|
||||
};
|
||||
use crate::video_player::VideoPlayer;
|
||||
use crate::video_player::{VideoDecoder, VideoPlayer};
|
||||
|
||||
use makepad_micro_serde::SerJson;
|
||||
use makepad_widgets::*;
|
||||
|
|
@ -4010,7 +4014,7 @@ pub struct App {
|
|||
fleet_timer: Timer,
|
||||
/// LAN beacon listener; polled on the fleet timer.
|
||||
#[rust]
|
||||
discovered: Option<makepad_ai_hub::discovery::Discovered>,
|
||||
discovered: Option<makepad_ai_hub::discovery::Discovery>,
|
||||
#[rust]
|
||||
job_timer: Timer,
|
||||
#[rust]
|
||||
|
|
@ -4036,6 +4040,8 @@ pub struct App {
|
|||
library: Option<Library>,
|
||||
#[rust]
|
||||
video: Option<VideoPlayer>,
|
||||
#[rust]
|
||||
video_decoder: Option<VideoDecoder>,
|
||||
/// The file the viewer's current video came from — Restart and the loop
|
||||
/// toggle re-open it.
|
||||
video_path: Option<PathBuf>,
|
||||
|
|
@ -4323,8 +4329,16 @@ impl App {
|
|||
// -- setup ---------------------------------------------------------------
|
||||
|
||||
fn setup(&mut self, cx: &mut Cx) {
|
||||
let pool = cx.task_pool();
|
||||
let spawner = cx.thread_spawner();
|
||||
let (video_decoder, mut video_audio) =
|
||||
VideoDecoder::start(spawner.clone()).expect("asset-ui video decoder worker");
|
||||
self.video_decoder = Some(video_decoder);
|
||||
let _ = std::fs::create_dir_all(artifacts_dir());
|
||||
self.artifact_io = Some(ArtifactIo::start());
|
||||
self.artifact_io = Some(ArtifactIo::start(spawner.clone()));
|
||||
self.analysis = Some(AnalysisQueue::start(spawner.clone()));
|
||||
self.import_page.set_task_pool(pool.clone());
|
||||
self.music_import_page.set_task_pool(pool.clone());
|
||||
self.load_fleet_prefs();
|
||||
self.library = Some(Library::open(repo_path("local/ai_content_library")));
|
||||
self.saved_presets = fast_presets::load(&fast_presets::store_path());
|
||||
|
|
@ -4349,7 +4363,11 @@ impl App {
|
|||
// The store hosts the embedded Asset Server; hand it the library it
|
||||
// must publish. Library::open ran above, so the product backfill is
|
||||
// already on disk when the watcher's first poll reads index.json.
|
||||
self.store.start(PathBuf::from(repo_path("local/ai_content_library")));
|
||||
self.store.start(
|
||||
PathBuf::from(repo_path("local/ai_content_library")),
|
||||
pool,
|
||||
spawner,
|
||||
);
|
||||
self.asset_store_timer = cx.start_interval(0.2);
|
||||
// Opening stays metadata-only; every missing preview is queued here
|
||||
// and regenerated a bounded slice at a time once frames are flowing.
|
||||
|
|
@ -4548,11 +4566,12 @@ impl App {
|
|||
self.refresh_voice_ui(cx);
|
||||
self.sync_preset_name_box(cx);
|
||||
|
||||
// Speakers: wav artifacts + video soundtrack.
|
||||
// Speakers: both engines move into the callback and own their state.
|
||||
let mut audio_engine = crate::audio::take_engine();
|
||||
cx.audio_output(0, move |info, output| {
|
||||
output.zero();
|
||||
crate::audio::mix_into(output, info.sample_rate);
|
||||
crate::video_player::mix_into(output, info.sample_rate);
|
||||
audio_engine.mix_into(output, info.sample_rate);
|
||||
video_audio.mix_into(output, info.sample_rate);
|
||||
});
|
||||
|
||||
// Headless drive.
|
||||
|
|
@ -7541,7 +7560,7 @@ impl App {
|
|||
// WAV must not call play() or a 200ms DS_* / Quake
|
||||
// shot becomes a loop (play-at-end restarts).
|
||||
if audition && audio::autoplay_one_shot(domain, pcm.seconds()) {
|
||||
crate::video_player::stop_audio();
|
||||
self.stop_video_audio();
|
||||
audio::play();
|
||||
self.arm_audio_pump(cx);
|
||||
}
|
||||
|
|
@ -7589,7 +7608,8 @@ impl App {
|
|||
// behind a new open or an error state.
|
||||
self.stop_video_playback();
|
||||
self.clear_video_frame(cx);
|
||||
match VideoPlayer::new(&path.to_string_lossy()) {
|
||||
let decoder = self.video_decoder.as_ref().expect("video decoder started");
|
||||
match VideoPlayer::new(&path.to_string_lossy(), decoder) {
|
||||
Ok(player) => {
|
||||
self.ui.label(cx, ids!(video_info)).set_text(
|
||||
cx,
|
||||
|
|
@ -9101,7 +9121,7 @@ impl App {
|
|||
&& audio::is_ready()
|
||||
&& !audio::is_playing()
|
||||
{
|
||||
crate::video_player::stop_audio();
|
||||
self.stop_video_audio();
|
||||
audio::play();
|
||||
self.arm_audio_pump(cx);
|
||||
self.sync_audio_ui(cx);
|
||||
|
|
@ -9963,7 +9983,7 @@ impl App {
|
|||
if !self.chat.is_linked() {
|
||||
if let Some(endpoints) = self.store.endpoints {
|
||||
let cache = session_config_from_env().cache_parent.join("cache-chat");
|
||||
self.chat.connect(endpoints, self.store.token.clone(), cache);
|
||||
self.chat.connect(cx, endpoints, self.store.token.clone(), cache);
|
||||
// The pane says "waiting for the asset server" until
|
||||
// something redraws it, and the feed only marks itself
|
||||
// dirty once a turn runs — so the line would sit there
|
||||
|
|
@ -10714,7 +10734,8 @@ impl App {
|
|||
if let Some(path) = item.as_ref().and_then(|item| item.payload.clone()) {
|
||||
self.stop_video_playback();
|
||||
self.clear_video_frame(cx);
|
||||
match VideoPlayer::new(&path.to_string_lossy()) {
|
||||
let decoder = self.video_decoder.as_ref().expect("video decoder started");
|
||||
match VideoPlayer::new(&path.to_string_lossy(), decoder) {
|
||||
Ok(player) => {
|
||||
self.library_video_file = Some(file.clone());
|
||||
self.video = Some(player);
|
||||
|
|
@ -10789,7 +10810,8 @@ impl App {
|
|||
self.library_audio_file = Some(file.clone());
|
||||
// The transport: decoded off the frame thread and
|
||||
// installed when it lands.
|
||||
let clip_gen = crate::audio::load_clip_async(bytes.clone());
|
||||
let pool = cx.task_pool();
|
||||
let clip_gen = crate::audio::load_clip_async(&pool, bytes.clone());
|
||||
// And, exactly once per track, what the store
|
||||
// already holds BESIDE the mixed audio: the four
|
||||
// separated layers and the transcript. The clip
|
||||
|
|
@ -10930,11 +10952,9 @@ impl App {
|
|||
|
||||
// -- "Split audio layers": the bake queue and its consumers -----------
|
||||
|
||||
/// The bake + fetch lanes, started on first use. Two threads parked on
|
||||
/// a channel is the whole cost of having them.
|
||||
/// The bake + fetch lanes, started once with the app and fed by channels.
|
||||
fn analysis(&mut self) -> &mut analysis::AnalysisQueue {
|
||||
self.analysis
|
||||
.get_or_insert_with(analysis::AnalysisQueue::start)
|
||||
self.analysis.as_mut().expect("analysis workers started")
|
||||
}
|
||||
|
||||
/// The selected catalog hit when it is an AUDIO asset: id and title.
|
||||
|
|
@ -12400,7 +12420,13 @@ impl App {
|
|||
/// [`Self::clear_video_frame`] is also called.
|
||||
fn stop_video_playback(&mut self) {
|
||||
self.video = None;
|
||||
crate::video_player::stop_audio();
|
||||
self.stop_video_audio();
|
||||
}
|
||||
|
||||
fn stop_video_audio(&self) {
|
||||
if let Some(decoder) = &self.video_decoder {
|
||||
decoder.stop_audio();
|
||||
}
|
||||
}
|
||||
|
||||
/// Blank the actual video WIDGET texture (not only the app-side handle),
|
||||
|
|
@ -12792,7 +12818,8 @@ impl App {
|
|||
fn restart_viewer_video(&mut self, cx: &mut Cx) -> bool {
|
||||
let Some(path) = self.video_path.clone() else { return false };
|
||||
self.stop_video_playback();
|
||||
match VideoPlayer::new(&path.to_string_lossy()) {
|
||||
let decoder = self.video_decoder.as_ref().expect("video decoder started");
|
||||
match VideoPlayer::new(&path.to_string_lossy(), decoder) {
|
||||
Ok(player) => {
|
||||
self.video = Some(player);
|
||||
self.sync_video_transport(cx);
|
||||
|
|
@ -14133,7 +14160,7 @@ impl MatchEvent for App {
|
|||
} else {
|
||||
// A user-resumed WAV preview wins over a stale video
|
||||
// soundtrack in the shared device callback.
|
||||
crate::video_player::stop_audio();
|
||||
self.stop_video_audio();
|
||||
audio::play();
|
||||
self.arm_audio_pump(cx);
|
||||
}
|
||||
|
|
@ -14434,13 +14461,11 @@ impl AppMain for App {
|
|||
fn script_mod(vm: &mut ScriptVm) -> ScriptValue {
|
||||
crate::makepad_widgets::script_mod(vm);
|
||||
// Draw shaders must register before the widgets that declare them.
|
||||
makepad_render::script_mod(vm);
|
||||
makepad_xr::script_mod(vm);
|
||||
makepad_media_view::script_mod(vm);
|
||||
// Shared preview widgets (ContentPreview / AudioView): the pool this
|
||||
// app draws catalog content with, and the same one the VJ and DJ
|
||||
// surfaces adopt.
|
||||
makepad_asset_widgets::script_mod(vm);
|
||||
crate::mesh_view::script_mod(vm);
|
||||
crate::mask_paint::script_mod(vm);
|
||||
crate::billboard_view::script_mod(vm);
|
||||
crate::thumbnail_renderer::script_mod(vm);
|
||||
|
|
@ -14691,7 +14716,9 @@ impl AppMain for App {
|
|||
}
|
||||
}
|
||||
self.scrub_audio(cx, event);
|
||||
if self.audio_timer.is_event(event).is_some() && audio::is_ready() {
|
||||
if self.audio_timer.is_event(event).is_some() {
|
||||
audio::pump();
|
||||
if audio::is_ready() {
|
||||
self.sync_audio_ui(cx);
|
||||
// The Library rail has its own transport over the same mixer.
|
||||
if self.surface == Surface::Library && self.library_audio_file.is_some() {
|
||||
|
|
@ -14701,6 +14728,7 @@ impl AppMain for App {
|
|||
self.arm_lyrics_pump(cx);
|
||||
}
|
||||
}
|
||||
}
|
||||
if self.audio_timer.is_event(event).is_some() && self.webcam.capturing {
|
||||
self.pump_webcam(cx);
|
||||
}
|
||||
|
|
@ -15918,6 +15946,13 @@ mod world_style_tests {
|
|||
namespace: namespace.to_string(),
|
||||
kind: Some(makepad_asset_data::AssetKind::World),
|
||||
title: title.to_string(),
|
||||
creator: String::new(),
|
||||
artist: String::new(),
|
||||
artist_url: String::new(),
|
||||
album: String::new(),
|
||||
source_url: String::new(),
|
||||
license: String::new(),
|
||||
license_url: String::new(),
|
||||
snippet: String::new(),
|
||||
score: 0,
|
||||
live: true,
|
||||
|
|
|
|||
|
|
@ -12,11 +12,11 @@
|
|||
use crate::import::{ImportPhase, ServerSession};
|
||||
use makepad_asset_client::{AssetClient, ClientConfig};
|
||||
use makepad_asset_importer::music_import::{self, MusicReport};
|
||||
use makepad_widgets::makepad_platform::thread::{Lane, TaskPool};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::mpsc::{self, Receiver, TryRecvError};
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
|
||||
/// The card's whole state: which folder, what the worker is doing, and the
|
||||
/// one-line result of the last run.
|
||||
|
|
@ -44,6 +44,7 @@ pub struct MusicImportPage {
|
|||
/// Aliases the finished run landed, waiting to be handed to the
|
||||
/// analysis queue. Non-empty only when `split_layers` was on.
|
||||
pending_analysis: Vec<String>,
|
||||
pool: Option<TaskPool>,
|
||||
}
|
||||
|
||||
/// What the worker sends back: live progress, then exactly one verdict.
|
||||
|
|
@ -65,6 +66,7 @@ impl Default for MusicImportPage {
|
|||
split_layers: false,
|
||||
bake_lyrics: false,
|
||||
pending_analysis: Vec::new(),
|
||||
pool: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -74,6 +76,10 @@ impl Default for MusicImportPage {
|
|||
pub const MUSIC_NAMESPACE: &str = "music";
|
||||
|
||||
impl MusicImportPage {
|
||||
pub fn set_task_pool(&mut self, pool: TaskPool) {
|
||||
self.pool = Some(pool);
|
||||
}
|
||||
|
||||
/// The job this card would enqueue right now, or `None` while no folder
|
||||
/// has been picked.
|
||||
pub fn job(&self) -> Option<crate::import::ImportJob> {
|
||||
|
|
@ -209,13 +215,18 @@ impl MusicImportPage {
|
|||
total: 0,
|
||||
current: String::new(),
|
||||
};
|
||||
thread::Builder::new()
|
||||
.name("asset-ui-music-import".into())
|
||||
.spawn(move || {
|
||||
let Some(pool) = self.pool.clone() else {
|
||||
return Err(self.refuse("runtime task pool is not configured".into()));
|
||||
};
|
||||
match pool.submit(Lane::Heavy, move || {
|
||||
let msg = run_music_import(&dir, server, &tx, &cancel);
|
||||
let _ = tx.send(msg);
|
||||
})
|
||||
.map_err(|e| self.refuse(format!("failed to start music import thread: {e}")))?;
|
||||
}) {
|
||||
Ok(handle) => handle.detach(),
|
||||
Err(error) => {
|
||||
return Err(self.refuse(format!("failed to submit music import job: {error}")));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -218,16 +218,16 @@ impl Preset {
|
|||
/// Python rigger, or another arbitrary model merely because it is resident.
|
||||
// This is the warm, local Makepad-Llama model advertised by the Mac fleet
|
||||
// node. Keeping it as fallback avoids an accidental large-model cold pull.
|
||||
const CHARACTER_LLM_MODEL: &str = "qwen3.5-9b";
|
||||
use makepad_asset_creator::character::CHARACTER_LLM_MODEL;
|
||||
/// Fleet-wide default for text expansion once a node has the audited weights
|
||||
/// ready. This is a preference, not a hard pin: first-run provisioning stays
|
||||
/// explicit and the warm 9B lane remains immediately usable.
|
||||
const PREFERRED_EXPAND_MODEL: &str = "qwen3.8-27b";
|
||||
const CHARACTER_IMAGE_MODEL: &str = "flux1-dev";
|
||||
const CHARACTER_MATTE_MODEL: &str = "birefnet-hr";
|
||||
const CHARACTER_MESH_MODEL: &str = "trellis-2";
|
||||
const CHARACTER_RIG_MODEL: &str = "skintokens";
|
||||
const CHARACTER_MOTION_MODEL: &str = "hy-motion";
|
||||
use makepad_asset_creator::character::CHARACTER_IMAGE_MODEL;
|
||||
use makepad_asset_creator::character::CHARACTER_MATTE_MODEL;
|
||||
use makepad_asset_creator::character::CHARACTER_MESH_MODEL;
|
||||
use makepad_asset_creator::character::CHARACTER_RIG_MODEL;
|
||||
use makepad_asset_creator::character::CHARACTER_MOTION_MODEL;
|
||||
/// Instruction image editing (reference image + "change …" prompt).
|
||||
const EDIT_MODEL: &str = "flux2-klein-4b";
|
||||
/// Sprite enhancement runs on the 32B dev DiT, NOT the 4-step distilled
|
||||
|
|
@ -257,7 +257,7 @@ const SPLAT_MODEL: &str = "triposplat";
|
|||
/// A character expansion substantially shorter than the 40-90 words asked
|
||||
/// for by `expand_rig.txt` is not a usable rig-safe brief. Refuse to quietly
|
||||
/// continue with it; the user can see and retry the failed LLM stage.
|
||||
const CHARACTER_EXPANSION_MIN_WORDS: usize = 24;
|
||||
// The shared character contract owns brief validation.
|
||||
|
||||
/// A character reconstruction gets two deterministic second chances when the
|
||||
/// rig or animated-skin quality gate rejects it. The matte/image are
|
||||
|
|
@ -544,7 +544,7 @@ pub const PRESETS: &[Preset] = &[
|
|||
// (idle/walk/jump locomotion, see mesh_view play mode).
|
||||
Preset::linear(
|
||||
"character (playable)",
|
||||
&["text", "image", "matte", "mesh", "rig", "motion"],
|
||||
makepad_asset_creator::character::CHARACTER_DOMAINS,
|
||||
// Character geometry is downstream of this one image: Schnell's
|
||||
// four-step distillation is useful for previews, but it is the wrong
|
||||
// silent affinity fallback for the rig master. Pin the validated
|
||||
|
|
@ -1329,21 +1329,7 @@ impl Pipeline {
|
|||
return unusable("was empty");
|
||||
}
|
||||
if self.is_character_pipeline() {
|
||||
let words = text.split_whitespace().count();
|
||||
if words < CHARACTER_EXPANSION_MIN_WORDS {
|
||||
return Err(format!(
|
||||
"LLM character brief is too short ({words} words, need at least {CHARACTER_EXPANSION_MIN_WORDS}); refusing to start image generation"
|
||||
));
|
||||
}
|
||||
if !text
|
||||
.to_lowercase()
|
||||
.contains(&self.prompt.trim().to_lowercase())
|
||||
{
|
||||
return Err(format!(
|
||||
"LLM character brief dropped identity anchor {:?}; refusing to start image generation",
|
||||
self.prompt.trim()
|
||||
));
|
||||
}
|
||||
makepad_asset_creator::character::validate_brief(&self.prompt, text)?;
|
||||
}
|
||||
return Ok(text.to_string());
|
||||
}
|
||||
|
|
@ -1501,16 +1487,7 @@ impl Pipeline {
|
|||
let is_music_target = target == "music";
|
||||
request.target_domain = Some(target);
|
||||
if self.is_character_pipeline() {
|
||||
request.identity_anchor = Some(self.prompt.trim().to_string());
|
||||
// Named-character identity and rig-safe presentation are
|
||||
// constraints, not a variant hunt. Keep this expansion
|
||||
// low-temperature and deterministic enough to avoid
|
||||
// inventing conflicting signature traits.
|
||||
request.temperature = Some(0.0);
|
||||
request.style = Some(
|
||||
"When the intent names an established character, preserve the exact named identity and canonical official design unchanged. Do not redesign, genericize, or guess traits. If a visual trait is uncertain, omit it instead of inventing it; it is better to say 'canonical official design unchanged' and spend the remaining prompt on full-body framing, a relaxed wide A-pose with straight diagonal arms and hands clear above the hips, visible gaps between every limb and the torso, even studio light, a uniform plain background, and a clean separated silhouette. Rigging constraints may change pose and spacing but never delete canonical anatomy or worn pieces."
|
||||
.to_string(),
|
||||
);
|
||||
makepad_asset_creator::character::configure_expansion(&mut request, &self.prompt);
|
||||
}
|
||||
// Music expansion carries a compact structured production
|
||||
// brief AND original section-tagged lyrics. Scale its budget
|
||||
|
|
@ -3381,7 +3358,7 @@ impl Pipeline {
|
|||
} => {
|
||||
let bytes = response
|
||||
.filter(|response| !failed && response.status_code == 200)
|
||||
.and_then(|response| response.body.clone());
|
||||
.and_then(|response| response.body.as_deref().map(<[u8]>::to_vec));
|
||||
let Some(bytes) = bytes else {
|
||||
return self.candidate_failed(
|
||||
cx,
|
||||
|
|
@ -3589,7 +3566,7 @@ impl Pipeline {
|
|||
Req::Artifact(stage, artifact) => {
|
||||
let bytes = response
|
||||
.filter(|r| !failed && r.status_code == 200)
|
||||
.and_then(|r| r.body.clone());
|
||||
.and_then(|r| r.body.as_deref().map(<[u8]>::to_vec));
|
||||
let Some(bytes) = bytes else {
|
||||
return self.fail_stage_or_skip_expander(
|
||||
cx,
|
||||
|
|
@ -3950,12 +3927,13 @@ mod tests {
|
|||
use makepad_ai_hub::protocol::{HealthJson, ModelInfoJson, MODEL_STATE_LOADED};
|
||||
BoxSnapshot {
|
||||
base_url: url.to_string(),
|
||||
health: Some(HealthJson { realtime: None,
|
||||
health: Some(HealthJson { realtime: None, activity: None,
|
||||
service: "test".to_string(),
|
||||
version: "1".to_string(),
|
||||
gpu: Some("GPU".to_string()),
|
||||
vram_free_mb: Some(24_000),
|
||||
vram_total_mb: Some(24_000),
|
||||
vram_usable_mb: None,
|
||||
models_loaded: vec!["flux1-schnell".to_string()],
|
||||
jobs_pending: Some(0),
|
||||
node_id: Some(1),
|
||||
|
|
@ -3964,6 +3942,7 @@ mod tests {
|
|||
capabilities: Some(vec!["image".to_string()]),
|
||||
vram_reserve_mb: Some(0),
|
||||
queue_limit: Some(8),
|
||||
max_job_body_bytes: None,
|
||||
fleet: None,
|
||||
lanes: None,
|
||||
}),
|
||||
|
|
@ -3995,12 +3974,13 @@ mod tests {
|
|||
use makepad_ai_hub::protocol::{HealthJson, ModelInfoJson, MODEL_STATE_LOADED};
|
||||
BoxSnapshot {
|
||||
base_url: url.to_string(),
|
||||
health: Some(HealthJson { realtime: None,
|
||||
health: Some(HealthJson { realtime: None, activity: None,
|
||||
service: "test".to_string(),
|
||||
version: "1".to_string(),
|
||||
gpu: Some("24 GB GPU".to_string()),
|
||||
vram_free_mb: Some(24 * 1024),
|
||||
vram_total_mb: Some(24 * 1024),
|
||||
vram_usable_mb: None,
|
||||
models_loaded: models
|
||||
.iter()
|
||||
.filter(|(_, state)| *state == MODEL_STATE_LOADED)
|
||||
|
|
@ -4013,6 +3993,7 @@ mod tests {
|
|||
capabilities: Some(vec!["text".to_string()]),
|
||||
vram_reserve_mb: Some(0),
|
||||
queue_limit: Some(8),
|
||||
max_job_body_bytes: None,
|
||||
fleet: None,
|
||||
lanes: None,
|
||||
}),
|
||||
|
|
@ -4957,7 +4938,10 @@ Arrangement: Pulsing bass, gated drums and widening analog pads."
|
|||
let mut want: Vec<(String, String)> = [
|
||||
("audio", "moss-sfx"),
|
||||
("audio", "sa3-sfx"),
|
||||
("audio", "salamander-drumkit"),
|
||||
("audio", "woosh-sfx"),
|
||||
("beats", "beat-this"),
|
||||
("body", "sam3dbody"),
|
||||
("control", "flux1-canny-dev"),
|
||||
("control", "flux1-depth-dev"),
|
||||
("depth", "da3-metric-large"),
|
||||
|
|
@ -4975,6 +4959,8 @@ Arrangement: Pulsing bass, gated drums and widening analog pads."
|
|||
("music", "minimax-music3"),
|
||||
("music", "minimax-music3-q4"),
|
||||
("music", "ace-step-1.5-xl"),
|
||||
("notes", "basic-pitch"),
|
||||
("ocr", "chandra-ocr-2"),
|
||||
("paint", "hunyuan3d-paint-2.1"),
|
||||
("rig", "skintokens"),
|
||||
("rig", "skintokens-oracle"),
|
||||
|
|
@ -4982,6 +4968,8 @@ Arrangement: Pulsing bass, gated drums and widening analog pads."
|
|||
("splat", "triposplat"),
|
||||
("speech", "indextts-2.5"),
|
||||
("speech", "kokoro"),
|
||||
("stems", "bs-roformer-4stem"),
|
||||
("stt", "whisper-large-v3-turbo"),
|
||||
("text", "qwen3.8-27b"),
|
||||
("upscale", "realesrgan-x4plus"),
|
||||
("vision", "qwen3.8-27b-vision"),
|
||||
|
|
@ -5042,7 +5030,7 @@ Arrangement: Pulsing bass, gated drums and widening analog pads."
|
|||
/// fails visibly as a service gap instead of rerouting.
|
||||
const DOCUMENTED_OVERRIDE_PINS: &[(&str, &str)] = &[("text", CHARACTER_LLM_MODEL)];
|
||||
|
||||
/// Every pin must reference a model the registry actually has in the
|
||||
/// Every pin must reference a model the registry can serve for the
|
||||
/// pinned domain — or be a documented cache-registry override above.
|
||||
/// Catches typos and silent registry drift.
|
||||
#[test]
|
||||
|
|
@ -5057,7 +5045,13 @@ Arrangement: Pulsing bass, gated drums and widening analog pads."
|
|||
registry
|
||||
.models
|
||||
.iter()
|
||||
.any(|entry| entry.id == *model && entry.domain.as_str() == *domain),
|
||||
.any(|entry| {
|
||||
entry.id == *model
|
||||
&& (entry.domain.as_str() == *domain
|
||||
|| (*domain == "edit"
|
||||
&& entry.domain.as_str() == "image"
|
||||
&& model.starts_with("flux2-dev")))
|
||||
}),
|
||||
"preset {:?} pins unknown model {domain}/{model}",
|
||||
preset.name
|
||||
);
|
||||
|
|
|
|||
|
|
@ -2542,6 +2542,13 @@ mod tests {
|
|||
namespace: "gen".into(),
|
||||
kind: Some(AssetKind::Prop),
|
||||
title: title.into(),
|
||||
creator: String::new(),
|
||||
artist: String::new(),
|
||||
artist_url: String::new(),
|
||||
album: String::new(),
|
||||
source_url: String::new(),
|
||||
license: String::new(),
|
||||
license_url: String::new(),
|
||||
snippet: "a thing".into(),
|
||||
score: 10,
|
||||
live,
|
||||
|
|
|
|||
|
|
@ -823,6 +823,7 @@ impl ThumbnailRenderer {
|
|||
color_adjust: vec4(0.0, 1.0, 1.0, 0.0),
|
||||
dynamic: true,
|
||||
depth_order: 0.0,
|
||||
custom_material: None,
|
||||
part_poses: Vec::new(),
|
||||
}),
|
||||
frame,
|
||||
|
|
|
|||
|
|
@ -1,637 +0,0 @@
|
|||
//! Video artifact playback — the sandbox's proven decode pattern
|
||||
//! (apps/sandbox/src/video_player.rs): a decode thread pulls frames + audio
|
||||
//! from the platform video-file seam (`makepad_platform::video_file`,
|
||||
//! hardware codecs), a small ring buffer hands BGRA frames to the render
|
||||
//! thread paced by pts against a wall clock, and the audio track mixes into
|
||||
//! this app's `cx.audio_output` closure.
|
||||
//!
|
||||
//! Playback is PLAY-ONCE: at end-of-stream the remaining audio drains and
|
||||
//! the clip stops (the sandbox pattern's loop-forever reopen was what users
|
||||
//! heard as "the soundtrack never ends"). Loading a new artifact drops the
|
||||
//! previous player (its `Drop` silences the queue); `stop_audio()` silences
|
||||
//! immediately and stays muted until the next clip starts.
|
||||
//!
|
||||
//! Copied rather than imported: the sandbox is an app crate under active
|
||||
//! concurrent development, not a library — and this pattern is ~250 lines.
|
||||
|
||||
use makepad_widgets::log;
|
||||
use makepad_widgets::makepad_platform::audio::AudioBuffer;
|
||||
use makepad_widgets::makepad_platform::video_file::{nv12, VideoFileDecoder};
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const RING_FRAMES: usize = 3;
|
||||
const AUDIO_AHEAD_SECS: f64 = 1.0;
|
||||
|
||||
struct Frame {
|
||||
pts_100ns: i64,
|
||||
bgra: Vec<u32>,
|
||||
}
|
||||
|
||||
struct Shared {
|
||||
frames: Mutex<VecDeque<Frame>>,
|
||||
stop: AtomicBool,
|
||||
/// The decode thread exited for good — a honored stop or a fatal
|
||||
/// decode error. End-of-stream no longer ends the thread: it PARKS
|
||||
/// (see `eos`) so seeks — loop restarts, scrubs — stay instant.
|
||||
done: AtomicBool,
|
||||
/// The stream is fully decoded and the thread is parked waiting for a
|
||||
/// seek or a stop. With an empty ring this is the player's EOS state.
|
||||
eos: AtomicBool,
|
||||
/// Requested playback position in 100ns units; -1 = none. The decode
|
||||
/// thread consumes it (an in-place decoder seek — no reopen).
|
||||
seek_100ns: AtomicI64,
|
||||
}
|
||||
|
||||
/// Soundtrack-queue ownership ticket. Every player claims a fresh epoch; a
|
||||
/// DETACHED decode thread of a dropped player still holds its old epoch and
|
||||
/// its pushes bounce off the queue (see [`VideoPlayer::drop`]).
|
||||
static NEXT_CLIP_EPOCH: AtomicU64 = AtomicU64::new(1);
|
||||
|
||||
/// Half a frame at 60 fps: a seek target inside a frame's span lands on
|
||||
/// that frame instead of the next.
|
||||
const FRAME_EPS_100NS: i64 = 83_000;
|
||||
|
||||
pub struct VideoPlayer {
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
/// Container-reported duration; 0 when the container does not say.
|
||||
pub duration_100ns: i64,
|
||||
shared: Arc<Shared>,
|
||||
started: Option<Instant>,
|
||||
last_pts: i64,
|
||||
/// While paused: when the pause began. The clock rebases by the paused
|
||||
/// span on resume, so playback continues where it stopped instead of
|
||||
/// skipping the frames "missed" on the wall clock.
|
||||
paused_at: Option<Instant>,
|
||||
epoch: u64,
|
||||
}
|
||||
|
||||
impl VideoPlayer {
|
||||
pub fn new(path: &str) -> Result<Self, String> {
|
||||
let info = VideoFileDecoder::open(path)
|
||||
.map_err(|e| e.to_string())?
|
||||
.info()
|
||||
.clone();
|
||||
if info.width == 0 || info.height == 0 {
|
||||
return Err(format!(
|
||||
"video reports zero size: {}x{}",
|
||||
info.width, info.height
|
||||
));
|
||||
}
|
||||
log!(
|
||||
"video: {} {}x{} {}/{} fps codec {:?} audio {} ({} Hz, {} ch)",
|
||||
path,
|
||||
info.width,
|
||||
info.height,
|
||||
info.fps_num,
|
||||
info.fps_den,
|
||||
info.video_codec,
|
||||
info.has_audio,
|
||||
info.audio_sample_rate,
|
||||
info.audio_channels
|
||||
);
|
||||
// Fresh clip: take ownership of the soundtrack queue. Claiming a new
|
||||
// epoch drops any previous tail, lifts the sticky stop_audio() mute,
|
||||
// and locks every straggler push from an older detached decode
|
||||
// thread out of the queue.
|
||||
let epoch = NEXT_CLIP_EPOCH.fetch_add(1, Ordering::Relaxed);
|
||||
{
|
||||
let mut audio = video_audio().lock().unwrap();
|
||||
audio.clear();
|
||||
audio.muted = false;
|
||||
audio.owner = epoch;
|
||||
}
|
||||
let shared = Arc::new(Shared {
|
||||
frames: Mutex::new(VecDeque::new()),
|
||||
stop: AtomicBool::new(false),
|
||||
done: AtomicBool::new(false),
|
||||
eos: AtomicBool::new(false),
|
||||
seek_100ns: AtomicI64::new(-1),
|
||||
});
|
||||
let thread_shared = shared.clone();
|
||||
let thread_path = path.to_string();
|
||||
// The JoinHandle is deliberately dropped: teardown must never join a
|
||||
// possibly wedged hardware decoder on the UI thread. The thread is
|
||||
// detached; `stop` + the audio epoch make that safe.
|
||||
std::thread::Builder::new()
|
||||
.name("asset-ui-video-decode".into())
|
||||
.spawn(move || {
|
||||
match VideoFileDecoder::open(&thread_path) {
|
||||
Ok(decoder) => decode_loop(thread_path, decoder, &thread_shared, epoch),
|
||||
Err(e) => log!("video: decode thread open failed: {}", e),
|
||||
}
|
||||
thread_shared.done.store(true, Ordering::Release);
|
||||
})
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(Self {
|
||||
width: info.width,
|
||||
height: info.height,
|
||||
duration_100ns: info.duration_100ns,
|
||||
shared,
|
||||
started: None,
|
||||
last_pts: 0,
|
||||
paused_at: None,
|
||||
epoch,
|
||||
})
|
||||
}
|
||||
|
||||
/// The newest frame whose pts has been reached; `None` keeps whatever is
|
||||
/// on the texture. Call once per render frame.
|
||||
pub fn is_paused(&self) -> bool {
|
||||
self.paused_at.is_some()
|
||||
}
|
||||
|
||||
/// Freeze the picture; the soundtrack mutes with it. Idempotent.
|
||||
pub fn pause(&mut self) {
|
||||
if self.paused_at.is_none() {
|
||||
self.paused_at = Some(Instant::now());
|
||||
video_audio().lock().unwrap().muted = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// Continue from the paused position by pushing the clock base forward
|
||||
/// by the paused span. Idempotent.
|
||||
pub fn resume(&mut self) {
|
||||
if let Some(paused_at) = self.paused_at.take() {
|
||||
if let Some(started) = &mut self.started {
|
||||
*started += paused_at.elapsed();
|
||||
}
|
||||
video_audio().lock().unwrap().muted = false;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn take_due_frame(&mut self) -> Option<Vec<u32>> {
|
||||
if self.paused_at.is_some() {
|
||||
// Paused normally: hold the picture. Freshly seeked while
|
||||
// paused (clock unset): show the seek target frame once.
|
||||
if self.started.is_some() {
|
||||
return None;
|
||||
}
|
||||
let mut frames = self.shared.frames.lock().unwrap();
|
||||
let frame = frames.pop_front()?;
|
||||
self.last_pts = frame.pts_100ns;
|
||||
// Keep the clock unset: resume rebases at the next frame.
|
||||
return Some(frame.bgra);
|
||||
}
|
||||
let mut frames = self.shared.frames.lock().unwrap();
|
||||
let first_pts = frames.front()?.pts_100ns;
|
||||
let started = *self.started.get_or_insert_with(|| {
|
||||
Instant::now() - Duration::from_nanos(first_pts.max(0) as u64 * 100)
|
||||
});
|
||||
let media_100ns = (started.elapsed().as_nanos() / 100) as i64;
|
||||
let mut due = None;
|
||||
while frames.front().is_some_and(|f| f.pts_100ns <= media_100ns) {
|
||||
due = frames.pop_front();
|
||||
}
|
||||
if let Some(frame) = &due {
|
||||
self.last_pts = frame.pts_100ns;
|
||||
}
|
||||
due.map(|f| f.bgra)
|
||||
}
|
||||
|
||||
pub fn position_secs(&self) -> f64 {
|
||||
self.last_pts as f64 / 10_000_000.0
|
||||
}
|
||||
|
||||
pub fn duration_secs(&self) -> f64 {
|
||||
self.duration_100ns as f64 / 10_000_000.0
|
||||
}
|
||||
|
||||
/// Jump playback to `secs`. The decode thread reopens the file and
|
||||
/// discards up to the target; the picture clock rebases on the first
|
||||
/// frame that arrives, so play continues from there (paused stays
|
||||
/// paused, showing the seeked frame).
|
||||
pub fn seek(&mut self, secs: f64) {
|
||||
let target = (secs.max(0.0) * 10_000_000.0) as i64;
|
||||
self.shared.seek_100ns.store(target, Ordering::Release);
|
||||
self.shared.frames.lock().unwrap().clear();
|
||||
self.started = None;
|
||||
self.last_pts = target;
|
||||
let mut audio = video_audio().lock().unwrap();
|
||||
if audio.owner == self.epoch {
|
||||
audio.clear();
|
||||
}
|
||||
}
|
||||
|
||||
/// True end-of-playback for the frame pump: the decode thread has exited
|
||||
/// (end of stream, decode error, or stop) and every buffered frame has
|
||||
/// been taken. The soundtrack tail may still be draining in the audio
|
||||
/// callback — that needs no frame pump.
|
||||
pub fn at_end(&self) -> bool {
|
||||
(self.shared.eos.load(Ordering::Acquire) || self.shared.done.load(Ordering::Acquire))
|
||||
&& self.shared.frames.lock().unwrap().is_empty()
|
||||
}
|
||||
|
||||
/// True while a seek request is still unconsumed by the decode thread —
|
||||
/// the host coalesces scrub drags on this instead of flooding.
|
||||
pub fn seek_pending(&self) -> bool {
|
||||
self.shared.seek_100ns.load(Ordering::Acquire) >= 0
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for VideoPlayer {
|
||||
fn drop(&mut self) {
|
||||
self.shared.stop.store(true, Ordering::Relaxed);
|
||||
// NEVER join the decode thread here: a wedged hardware decoder call
|
||||
// would hang the UI thread. The detached thread observes `stop`
|
||||
// between packets and exits on its own; until then the epoch guard
|
||||
// keeps its audio pushes out of the queue, and its frame ring dies
|
||||
// with the last Arc.
|
||||
let mut audio = video_audio().lock().unwrap();
|
||||
if audio.owner == self.epoch {
|
||||
audio.clear();
|
||||
audio.owner = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_loop(_path: String, mut decoder: VideoFileDecoder, shared: &Shared, epoch: u64) {
|
||||
let info = decoder.info().clone();
|
||||
let mut audio_eos = false;
|
||||
let mut rgb_scratch = Vec::new();
|
||||
loop {
|
||||
if shared.stop.load(Ordering::Relaxed) {
|
||||
return;
|
||||
}
|
||||
let seek = shared.seek_100ns.swap(-1, Ordering::AcqRel);
|
||||
if seek >= 0 {
|
||||
shared.eos.store(false, Ordering::Release);
|
||||
// In-place decoder seek (SetCurrentPosition / reader rebuild in
|
||||
// the platform layer, ~10 ms) — never a full reopen, which is
|
||||
// what makes SCRUBBING realtime and a loop restart seamless.
|
||||
match decoder.seek(seek) {
|
||||
Ok(()) => {
|
||||
audio_eos = !info.has_audio;
|
||||
shared.frames.lock().unwrap().clear();
|
||||
loop {
|
||||
if shared.stop.load(Ordering::Relaxed) {
|
||||
return;
|
||||
}
|
||||
if shared.seek_100ns.load(Ordering::Acquire) >= 0 {
|
||||
break; // newer scrub target supersedes this one
|
||||
}
|
||||
match decoder.next_frame() {
|
||||
Ok(Some(frame)) if frame.pts_100ns + FRAME_EPS_100NS < seek => {}
|
||||
Ok(Some(frame)) => {
|
||||
nv12::nv12_to_rgb8(
|
||||
&frame.nv12,
|
||||
frame.width,
|
||||
frame.height,
|
||||
&mut rgb_scratch,
|
||||
);
|
||||
let mut bgra =
|
||||
Vec::with_capacity((frame.width * frame.height) as usize);
|
||||
for px in rgb_scratch.chunks_exact(3) {
|
||||
bgra.push(
|
||||
0xff00_0000
|
||||
| ((px[0] as u32) << 16)
|
||||
| ((px[1] as u32) << 8)
|
||||
| px[2] as u32,
|
||||
);
|
||||
}
|
||||
shared
|
||||
.frames
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push_back(Frame { pts_100ns: frame.pts_100ns, bgra });
|
||||
break;
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(e) => {
|
||||
log!("video: seek decode error: {}", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Audio follows the picture: drop queued samples and
|
||||
// skip the soundtrack forward to the target.
|
||||
if info.has_audio {
|
||||
video_audio().lock().unwrap().clear_for(epoch);
|
||||
loop {
|
||||
if shared.stop.load(Ordering::Relaxed) {
|
||||
return;
|
||||
}
|
||||
match decoder.next_audio() {
|
||||
Ok(Some(chunk)) if chunk.pts_100ns < seek => {}
|
||||
Ok(Some(chunk)) => {
|
||||
video_audio().lock().unwrap().push_i16(
|
||||
epoch,
|
||||
&chunk.samples,
|
||||
chunk.channels,
|
||||
chunk.sample_rate,
|
||||
);
|
||||
break;
|
||||
}
|
||||
Ok(None) => {
|
||||
audio_eos = true;
|
||||
break;
|
||||
}
|
||||
Err(_) => {
|
||||
audio_eos = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => log!("video: decoder seek failed: {}", e),
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if info.has_audio && !audio_eos {
|
||||
while video_audio().lock().unwrap().buffered_secs() < AUDIO_AHEAD_SECS {
|
||||
match decoder.next_audio() {
|
||||
Ok(Some(chunk)) => video_audio().lock().unwrap().push_i16(
|
||||
epoch,
|
||||
&chunk.samples,
|
||||
chunk.channels,
|
||||
chunk.sample_rate,
|
||||
),
|
||||
Ok(None) => {
|
||||
audio_eos = true;
|
||||
break;
|
||||
}
|
||||
Err(e) => {
|
||||
log!("video: audio decode error: {}", e);
|
||||
audio_eos = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if shared.frames.lock().unwrap().len() >= RING_FRAMES {
|
||||
std::thread::sleep(Duration::from_millis(4));
|
||||
continue;
|
||||
}
|
||||
match decoder.next_frame() {
|
||||
Ok(Some(frame)) => {
|
||||
nv12::nv12_to_rgb8(&frame.nv12, frame.width, frame.height, &mut rgb_scratch);
|
||||
let mut bgra = Vec::with_capacity((frame.width * frame.height) as usize);
|
||||
for px in rgb_scratch.chunks_exact(3) {
|
||||
bgra.push(
|
||||
0xff00_0000
|
||||
| ((px[0] as u32) << 16)
|
||||
| ((px[1] as u32) << 8)
|
||||
| px[2] as u32,
|
||||
);
|
||||
}
|
||||
shared.frames.lock().unwrap().push_back(Frame {
|
||||
pts_100ns: frame.pts_100ns,
|
||||
bgra,
|
||||
});
|
||||
}
|
||||
Ok(None) => {
|
||||
// End of stream: drain the soundtrack tail, then PARK. The
|
||||
// thread stays alive serving seeks — a loop restart or a
|
||||
// scrub back into the clip is a ~10 ms decoder seek, not a
|
||||
// teardown and reopen.
|
||||
if info.has_audio && !audio_eos {
|
||||
loop {
|
||||
if shared.stop.load(Ordering::Relaxed) {
|
||||
return;
|
||||
}
|
||||
match decoder.next_audio() {
|
||||
Ok(Some(chunk)) => video_audio().lock().unwrap().push_i16(
|
||||
epoch,
|
||||
&chunk.samples,
|
||||
chunk.channels,
|
||||
chunk.sample_rate,
|
||||
),
|
||||
Ok(None) => break,
|
||||
Err(e) => {
|
||||
log!("video: audio tail decode error: {}", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
audio_eos = true;
|
||||
}
|
||||
shared.eos.store(true, Ordering::Release);
|
||||
while shared.eos.load(Ordering::Acquire) {
|
||||
if shared.stop.load(Ordering::Relaxed) {
|
||||
return;
|
||||
}
|
||||
if shared.seek_100ns.load(Ordering::Acquire) >= 0 {
|
||||
break; // the top of the loop consumes it
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(e) => {
|
||||
log!("video: decode error: {}", e);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Soundtrack queue (identical mixer shape to the wav player in audio.rs)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
impl VideoAudio {
|
||||
/// Epoch-guarded queue drop for a seek: only the owning player's decode
|
||||
/// thread may flush what it queued.
|
||||
fn clear_for(&mut self, epoch: u64) {
|
||||
if self.owner == epoch {
|
||||
self.frames.clear();
|
||||
self.cursor = 0.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct VideoAudio {
|
||||
frames: VecDeque<(f32, f32)>,
|
||||
cursor: f64,
|
||||
source_rate: f64,
|
||||
/// Sticky mute raised by [`stop_audio`]: the decode thread may still be
|
||||
/// refilling the queue, so a plain clear would go audible again ~a
|
||||
/// second later. Cleared when the next clip starts.
|
||||
muted: bool,
|
||||
/// Epoch of the ONE player allowed to push (0 = none). Detached decode
|
||||
/// threads of dropped players carry stale epochs and are locked out.
|
||||
owner: u64,
|
||||
}
|
||||
|
||||
impl VideoAudio {
|
||||
const fn new() -> Self {
|
||||
Self {
|
||||
frames: VecDeque::new(),
|
||||
cursor: 0.0,
|
||||
source_rate: 0.0,
|
||||
muted: false,
|
||||
owner: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn clear(&mut self) {
|
||||
self.frames.clear();
|
||||
self.cursor = 0.0;
|
||||
}
|
||||
|
||||
fn buffered_secs(&self) -> f64 {
|
||||
if self.source_rate <= 0.0 {
|
||||
return 0.0;
|
||||
}
|
||||
(self.frames.len() as f64 - self.cursor).max(0.0) / self.source_rate
|
||||
}
|
||||
|
||||
fn push_i16(&mut self, epoch: u64, samples: &[i16], channels: u16, rate: u32) {
|
||||
if self.muted || self.owner != epoch {
|
||||
return;
|
||||
}
|
||||
self.source_rate = rate as f64;
|
||||
let ch = channels.max(1) as usize;
|
||||
const GAIN: f32 = 0.9;
|
||||
for frame in samples.chunks_exact(ch) {
|
||||
let l = frame[0] as f32 / 32768.0 * GAIN;
|
||||
let r = frame[ch - 1] as f32 / 32768.0 * GAIN;
|
||||
self.frames.push_back((l, r));
|
||||
}
|
||||
}
|
||||
|
||||
fn mix(&mut self, output: &mut AudioBuffer, device_rate: f64) {
|
||||
if self.frames.is_empty() || self.source_rate <= 0.0 || device_rate <= 0.0 {
|
||||
return;
|
||||
}
|
||||
let step = self.source_rate / device_rate;
|
||||
let channels = output.channel_count();
|
||||
for frame in 0..output.frame_count() {
|
||||
let index = self.cursor as usize;
|
||||
if index + 1 >= self.frames.len() {
|
||||
break;
|
||||
}
|
||||
let fraction = (self.cursor - index as f64) as f32;
|
||||
let (al, ar) = self.frames[index];
|
||||
let (bl, br) = self.frames[index + 1];
|
||||
let l = al + (bl - al) * fraction;
|
||||
let r = ar + (br - ar) * fraction;
|
||||
for channel in 0..channels {
|
||||
let s = if channel == 0 { l } else { r };
|
||||
output.channel_mut(channel)[frame] += s;
|
||||
}
|
||||
self.cursor += step;
|
||||
}
|
||||
let consumed = self.cursor as usize;
|
||||
if consumed > 0 {
|
||||
self.frames.drain(..consumed.min(self.frames.len()));
|
||||
self.cursor -= consumed as f64;
|
||||
}
|
||||
// A lone trailing frame can never be interpolated: once the queue is
|
||||
// down to it the clip is over (or hard-underrun) — release it so the
|
||||
// soundtrack ends instead of pinning the final sample forever.
|
||||
if self.frames.len() <= 1 {
|
||||
self.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static VIDEO_AUDIO: Mutex<VideoAudio> = Mutex::new(VideoAudio::new());
|
||||
|
||||
fn video_audio() -> &'static Mutex<VideoAudio> {
|
||||
&VIDEO_AUDIO
|
||||
}
|
||||
|
||||
/// Mix queued video audio into the device buffer (one line in the app's
|
||||
/// `cx.audio_output` closure).
|
||||
pub fn mix_into(output: &mut AudioBuffer, device_rate: f64) {
|
||||
if let Ok(mut audio) = video_audio().lock() {
|
||||
audio.mix(output, device_rate);
|
||||
}
|
||||
}
|
||||
|
||||
/// Silences the video soundtrack immediately and keeps it silent (the
|
||||
/// decode thread may still be refilling) until the next clip starts —
|
||||
/// stop-button / tab-switch hook. Revoking ownership locks every live
|
||||
/// decode thread out of the queue, not just muting it.
|
||||
pub fn stop_audio() {
|
||||
if let Ok(mut audio) = video_audio().lock() {
|
||||
audio.clear();
|
||||
audio.muted = true;
|
||||
audio.owner = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Both tests touch the process-global soundtrack queue.
|
||||
static VIDEO_TEST_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
#[test]
|
||||
fn at_end_needs_decode_exit_and_a_drained_ring() {
|
||||
let _serial = VIDEO_TEST_LOCK.lock().unwrap();
|
||||
let shared = Arc::new(Shared {
|
||||
frames: Mutex::new(VecDeque::new()),
|
||||
stop: AtomicBool::new(false),
|
||||
done: AtomicBool::new(false),
|
||||
eos: AtomicBool::new(false),
|
||||
seek_100ns: AtomicI64::new(-1),
|
||||
});
|
||||
shared.frames.lock().unwrap().push_back(Frame {
|
||||
pts_100ns: 0,
|
||||
bgra: vec![0xff00_0000; 4],
|
||||
});
|
||||
let mut player = VideoPlayer {
|
||||
width: 2,
|
||||
height: 2,
|
||||
duration_100ns: 0,
|
||||
shared: shared.clone(),
|
||||
started: None,
|
||||
last_pts: 0,
|
||||
paused_at: None,
|
||||
epoch: u64::MAX,
|
||||
};
|
||||
// Still decoding: never EOS, with or without buffered frames.
|
||||
assert!(!player.at_end());
|
||||
// Decode exited, but a due frame is still buffered: pump keeps going.
|
||||
shared.done.store(true, Ordering::Release);
|
||||
assert!(!player.at_end());
|
||||
// The pts-0 frame is immediately due; taking it drains the ring.
|
||||
assert!(player.take_due_frame().is_some());
|
||||
assert!(player.at_end(), "decode done + drained ring is EOS");
|
||||
// A stopped-but-undrained ring is also not EOS until taken.
|
||||
assert!(player.take_due_frame().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn soundtrack_queue_ignores_stale_epochs_and_revoked_ownership() {
|
||||
let _serial = VIDEO_TEST_LOCK.lock().unwrap();
|
||||
{
|
||||
let mut audio = video_audio().lock().unwrap();
|
||||
audio.clear();
|
||||
audio.muted = false;
|
||||
audio.owner = 7;
|
||||
audio.source_rate = 0.0;
|
||||
}
|
||||
// A dropped player's detached thread (stale epoch) cannot push.
|
||||
video_audio()
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push_i16(6, &[1000, 1000], 2, 48_000);
|
||||
assert_eq!(video_audio().lock().unwrap().frames.len(), 0);
|
||||
// The owning epoch pushes fine.
|
||||
video_audio()
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push_i16(7, &[1000, 1000, 2000, 2000], 2, 48_000);
|
||||
assert_eq!(video_audio().lock().unwrap().frames.len(), 2);
|
||||
// stop_audio revokes ownership: even the former owner is locked out.
|
||||
stop_audio();
|
||||
video_audio()
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push_i16(7, &[1000, 1000], 2, 48_000);
|
||||
let audio = video_audio().lock().unwrap();
|
||||
assert_eq!(audio.frames.len(), 0);
|
||||
assert_eq!(audio.owner, 0);
|
||||
assert!(audio.muted);
|
||||
}
|
||||
}
|
||||
29
apps/browser/Cargo.toml
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
# browser — a Chrome-like browser as a plain full-window Makepad app.
|
||||
#
|
||||
# Chromium (CEF, libs/cef) renders ONLY the page, GPU-accelerated straight
|
||||
# into an IOSurface-backed Makepad texture. All browser chrome — the tab
|
||||
# strip, the toolbar with back/forward/reload, the omnibox and the menu —
|
||||
# is Makepad splash UI, styled from the wm theme when hosted by
|
||||
# makepad-wm (`MAKEPAD_WM_THEME_SPLASH`) and from a Chrome-dark palette otherwise.
|
||||
#
|
||||
# Runs standalone, and unmodified inside makepad-wm / Studio tiles via the
|
||||
# shared --stdin-loop client runtime every Makepad app has.
|
||||
|
||||
[package]
|
||||
name = "makepad-browser"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
default-run = "browser"
|
||||
|
||||
[[bin]]
|
||||
name = "browser"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
makepad-widgets = { path = "../../widgets", features = ["cef"] }
|
||||
makepad-cef = { path = "../../libs/cef" }
|
||||
makepad-ai-services = { path = "../../libs/ai/services" }
|
||||
makepad-strict-json = { path = "../../libs/strict_json" }
|
||||
makepad-wm-theme = { path = "../../libs/wm_theme" }
|
||||
# The window manager's vocabulary: the bar's title, the polite close.
|
||||
makepad-wm-api = { path = "../../libs/wm_api" }
|
||||
|
Before Width: | Height: | Size: 231 B After Width: | Height: | Size: 231 B |
|
Before Width: | Height: | Size: 310 B After Width: | Height: | Size: 310 B |
|
Before Width: | Height: | Size: 228 B After Width: | Height: | Size: 228 B |
|
Before Width: | Height: | Size: 950 B After Width: | Height: | Size: 950 B |
|
Before Width: | Height: | Size: 346 B After Width: | Height: | Size: 346 B |
|
Before Width: | Height: | Size: 314 B After Width: | Height: | Size: 314 B |
|
Before Width: | Height: | Size: 867 B After Width: | Height: | Size: 867 B |
|
Before Width: | Height: | Size: 892 B After Width: | Height: | Size: 892 B |
|
Before Width: | Height: | Size: 284 B After Width: | Height: | Size: 284 B |
220
apps/browser/src/ai.rs
Normal file
|
|
@ -0,0 +1,220 @@
|
|||
//! The browser on the desktop's AI bus.
|
||||
//!
|
||||
//! The current CEF wrapper mirrors navigation metadata but does not bind its
|
||||
//! frame text/source callbacks, so `page` reports the active title and URL.
|
||||
|
||||
use makepad_ai_services::wire::{Risk, ServiceCall, ServiceManifest, ToolDef, ToolResult};
|
||||
use makepad_strict_json::{self as json, Value};
|
||||
|
||||
pub struct PageState {
|
||||
pub title: String,
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
pub struct TabState {
|
||||
pub title: String,
|
||||
pub url: String,
|
||||
pub active: bool,
|
||||
}
|
||||
|
||||
/// The bus-facing subset of the webview. Keeping it as a trait makes the
|
||||
/// closed dispatcher testable without starting CEF or a window.
|
||||
pub trait BrowserTarget {
|
||||
fn page(&self) -> Option<PageState>;
|
||||
fn tabs(&self) -> Vec<TabState>;
|
||||
fn navigate(&mut self, url: &str) -> bool;
|
||||
fn new_tab(&mut self, url: &str);
|
||||
}
|
||||
|
||||
pub fn manifest() -> ServiceManifest {
|
||||
ServiceManifest::new(
|
||||
"browser",
|
||||
"Browser",
|
||||
"The live web browser. Its read tools report the active page and all tabs; its action tools steer the active tab or open a new one.",
|
||||
)
|
||||
.with_tool(ToolDef::new(
|
||||
"page",
|
||||
"Read the active tab's displayed title and URL. The current CEF binding does not expose visible page text or source, so this tool cannot return page text.",
|
||||
r#"{"type":"object","properties":{},"additionalProperties":false}"#,
|
||||
Risk::Read,
|
||||
))
|
||||
.with_tool(ToolDef::new(
|
||||
"tabs",
|
||||
"Read every open tab's displayed title and URL, with the active tab marked.",
|
||||
r#"{"type":"object","properties":{},"additionalProperties":false}"#,
|
||||
Risk::Read,
|
||||
))
|
||||
.with_tool(ToolDef::new(
|
||||
"navigate",
|
||||
"Navigate the active tab to an http:// or https:// URL, or to about:blank.",
|
||||
r#"{"type":"object","properties":{"url":{"type":"string"}},"required":["url"],"additionalProperties":false}"#,
|
||||
Risk::Act,
|
||||
))
|
||||
.with_tool(ToolDef::new(
|
||||
"new_tab",
|
||||
"Open and activate a new tab at an http:// or https:// URL, or at about:blank.",
|
||||
r#"{"type":"object","properties":{"url":{"type":"string"}},"required":["url"],"additionalProperties":false}"#,
|
||||
Risk::Act,
|
||||
))
|
||||
}
|
||||
|
||||
/// Answer one browser call through a closed match over the four advertised
|
||||
/// names. URL actions use the webview's existing navigation methods.
|
||||
pub fn answer(call: &ServiceCall, target: &mut impl BrowserTarget) -> ToolResult {
|
||||
match call.tool.as_str() {
|
||||
"page" => {
|
||||
if let Err(error) = empty_args(&call.args) {
|
||||
return ToolResult::refused(&call.call_id, error);
|
||||
}
|
||||
match target.page() {
|
||||
Some(page) => ToolResult::ok(
|
||||
&call.call_id,
|
||||
format!("title: {}\nurl: {}", page.title, page.url),
|
||||
format!("{} — {}", page.title, page.url),
|
||||
),
|
||||
None => ToolResult::unavailable(&call.call_id, "there is no active browser tab"),
|
||||
}
|
||||
}
|
||||
"tabs" => {
|
||||
if let Err(error) = empty_args(&call.args) {
|
||||
return ToolResult::refused(&call.call_id, error);
|
||||
}
|
||||
let tabs = target.tabs();
|
||||
let mut text = String::new();
|
||||
for (index, tab) in tabs.iter().enumerate() {
|
||||
if index > 0 {
|
||||
text.push('\n');
|
||||
}
|
||||
text.push_str(if tab.active { "[active] " } else { " " });
|
||||
text.push_str(&tab.title);
|
||||
text.push_str(" — ");
|
||||
text.push_str(&tab.url);
|
||||
}
|
||||
if text.is_empty() {
|
||||
text.push_str("no tabs open");
|
||||
}
|
||||
ToolResult::ok(&call.call_id, text, format!("{} tabs", tabs.len()))
|
||||
}
|
||||
"navigate" => {
|
||||
let url = match url_arg(&call.args, "navigate") {
|
||||
Ok(url) => url,
|
||||
Err(error) => return ToolResult::refused(&call.call_id, error),
|
||||
};
|
||||
if target.navigate(&url) {
|
||||
ToolResult::ok(&call.call_id, format!("navigating to {url}"), "navigating")
|
||||
} else {
|
||||
ToolResult::unavailable(&call.call_id, "there is no active browser tab")
|
||||
}
|
||||
}
|
||||
"new_tab" => {
|
||||
let url = match url_arg(&call.args, "new_tab") {
|
||||
Ok(url) => url,
|
||||
Err(error) => return ToolResult::refused(&call.call_id, error),
|
||||
};
|
||||
target.new_tab(&url);
|
||||
ToolResult::ok(&call.call_id, format!("opened a new tab at {url}"), "navigating")
|
||||
}
|
||||
other => ToolResult::refused(
|
||||
&call.call_id,
|
||||
format!("browser has no tool `{other}`; it has page, tabs, navigate, new_tab"),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn empty_args(args: &str) -> Result<(), String> {
|
||||
let fields = object_args(args)?;
|
||||
if let Some((key, _)) = fields.first() {
|
||||
return Err(format!("unknown argument `{key}`"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn url_arg(args: &str, tool: &str) -> Result<String, String> {
|
||||
let fields = object_args(args)?;
|
||||
if let Some((key, _)) = fields.iter().find(|(key, _)| key != "url") {
|
||||
return Err(format!("unknown argument `{key}`"));
|
||||
}
|
||||
let Some(url) = fields
|
||||
.iter()
|
||||
.find(|(key, _)| key == "url")
|
||||
.and_then(|(_, value)| value.as_str())
|
||||
else {
|
||||
return Err(format!("{tool}.url must be a string"));
|
||||
};
|
||||
if !allowed_url(url) {
|
||||
return Err(format!(
|
||||
"{tool}.url must start with http:// or https://, or be exactly about:blank"
|
||||
));
|
||||
}
|
||||
Ok(url.to_string())
|
||||
}
|
||||
|
||||
fn object_args(args: &str) -> Result<Vec<(String, Value)>, String> {
|
||||
match json::parse(args.as_bytes()) {
|
||||
Ok(Value::Obj(fields)) => Ok(fields),
|
||||
Ok(_) => Err("tool arguments must be a JSON object".to_string()),
|
||||
Err(error) => Err(format!("invalid tool arguments: {error}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn allowed_url(url: &str) -> bool {
|
||||
if url == "about:blank" {
|
||||
return true;
|
||||
}
|
||||
if url.chars().any(char::is_whitespace) || url.chars().any(char::is_control) {
|
||||
return false;
|
||||
}
|
||||
url.strip_prefix("http://")
|
||||
.or_else(|| url.strip_prefix("https://"))
|
||||
.is_some_and(|rest| !rest.is_empty())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use makepad_ai_services::wire::ToolOutcome;
|
||||
|
||||
#[derive(Default)]
|
||||
struct FakeTarget {
|
||||
navigated: Vec<String>,
|
||||
}
|
||||
|
||||
impl BrowserTarget for FakeTarget {
|
||||
fn page(&self) -> Option<PageState> {
|
||||
None
|
||||
}
|
||||
|
||||
fn tabs(&self) -> Vec<TabState> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
fn navigate(&mut self, url: &str) -> bool {
|
||||
self.navigated.push(url.to_string());
|
||||
true
|
||||
}
|
||||
|
||||
fn new_tab(&mut self, url: &str) {
|
||||
self.navigated.push(url.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn browser_manifest_validates() {
|
||||
manifest().validate().expect("a valid browser manifest");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn navigate_refuses_non_http_urls() {
|
||||
let mut target = FakeTarget::default();
|
||||
for url in ["file:///etc/passwd", "javascript:alert(1)", "data:text/plain,no"] {
|
||||
let call = ServiceCall {
|
||||
call_id: "c1".into(),
|
||||
tool: "navigate".into(),
|
||||
args: format!(r#"{{"url":"{url}"}}"#),
|
||||
};
|
||||
let result = answer(&call, &mut target);
|
||||
assert_eq!(result.outcome, ToolOutcome::Refused);
|
||||
}
|
||||
assert!(target.navigated.is_empty());
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
//! The browser chrome: a custom-drawn Chrome-style tab strip (favicon +
|
||||
//! title tabs with a close x, a + for a new tab) and the toolbar
|
||||
//! (back / forward / reload, the omnibox with search icon and bookmark star,
|
||||
//! the menu button). Hard-square Omarchy look, colours from `mod.mpb_theme`.
|
||||
//! the menu button). Hard-square Omarchy look, colours from `mod.browser_theme`.
|
||||
|
||||
use crate::tabs::{TabId, TabSummary};
|
||||
use makepad_widgets::image::DrawImage;
|
||||
|
|
@ -18,62 +18,62 @@ script_mod! {
|
|||
height: 36
|
||||
|
||||
draw_bg +: {
|
||||
color: uniform(mod.mpb_theme.darker_background)
|
||||
color: uniform(mod.browser_theme.darker_background)
|
||||
pixel: fn() {
|
||||
return self.color
|
||||
}
|
||||
}
|
||||
draw_tab +: {
|
||||
color: uniform(mod.mpb_theme.darker_background)
|
||||
color: uniform(mod.browser_theme.darker_background)
|
||||
pixel: fn() {
|
||||
return self.color
|
||||
}
|
||||
}
|
||||
draw_tab_hover +: {
|
||||
color: uniform(mod.mpb_theme.dark_background)
|
||||
color: uniform(mod.browser_theme.dark_background)
|
||||
pixel: fn() {
|
||||
return self.color
|
||||
}
|
||||
}
|
||||
draw_tab_active +: {
|
||||
color: uniform(mod.mpb_theme.background)
|
||||
color: uniform(mod.browser_theme.background)
|
||||
pixel: fn() {
|
||||
return self.color
|
||||
}
|
||||
}
|
||||
draw_sep +: {
|
||||
color: uniform(mod.mpb_theme.muted)
|
||||
color: uniform(mod.browser_theme.muted)
|
||||
pixel: fn() {
|
||||
return self.color
|
||||
}
|
||||
}
|
||||
draw_text +: {
|
||||
color: mod.mpb_theme.foreground
|
||||
color: mod.browser_theme.foreground
|
||||
text_style: theme.font_regular{
|
||||
font_size: 9.5
|
||||
}
|
||||
}
|
||||
draw_text_dim +: {
|
||||
color: mod.mpb_theme.dark_foreground
|
||||
color: mod.browser_theme.dark_foreground
|
||||
text_style: theme.font_regular{
|
||||
font_size: 9.5
|
||||
}
|
||||
}
|
||||
draw_close +: {
|
||||
svg: crate_resource("self:resources/icons/close.svg")
|
||||
color: mod.mpb_theme.dark_foreground
|
||||
color: mod.browser_theme.dark_foreground
|
||||
}
|
||||
draw_close_active +: {
|
||||
svg: crate_resource("self:resources/icons/close.svg")
|
||||
color: mod.mpb_theme.foreground
|
||||
color: mod.browser_theme.foreground
|
||||
}
|
||||
draw_plus +: {
|
||||
svg: crate_resource("self:resources/icons/plus.svg")
|
||||
color: mod.mpb_theme.foreground
|
||||
color: mod.browser_theme.foreground
|
||||
}
|
||||
draw_globe +: {
|
||||
svg: crate_resource("self:resources/icons/globe.svg")
|
||||
color: mod.mpb_theme.dark_foreground
|
||||
color: mod.browser_theme.dark_foreground
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -87,14 +87,14 @@ script_mod! {
|
|||
align: Align{x: 0.5 y: 0.5}
|
||||
icon_walk: Walk{width: 16 height: 16}
|
||||
draw_icon +: {
|
||||
color: mod.mpb_theme.foreground
|
||||
color: mod.browser_theme.foreground
|
||||
}
|
||||
draw_bg +: {
|
||||
border_radius: 0.0
|
||||
border_size: 0.0
|
||||
color: #00000000
|
||||
color_hover: mod.mpb_theme.lighter_background
|
||||
color_down: mod.mpb_theme.muted
|
||||
color_hover: mod.browser_theme.lighter_background
|
||||
color_down: mod.browser_theme.muted
|
||||
color_focus: #00000000
|
||||
}
|
||||
}
|
||||
|
|
@ -109,8 +109,8 @@ script_mod! {
|
|||
padding: Inset{left: 14 right: 14 top: 0 bottom: 0}
|
||||
margin: Inset{left: 0 right: 0 top: 0 bottom: 0}
|
||||
draw_text +: {
|
||||
color: mod.mpb_theme.foreground
|
||||
color_hover: mod.mpb_theme.bright_foreground
|
||||
color: mod.browser_theme.foreground
|
||||
color_hover: mod.browser_theme.bright_foreground
|
||||
text_style: theme.font_regular{
|
||||
font_size: 10
|
||||
}
|
||||
|
|
@ -119,8 +119,8 @@ script_mod! {
|
|||
border_radius: 0.0
|
||||
border_size: 0.0
|
||||
color: #00000000
|
||||
color_hover: mod.mpb_theme.lighter_background
|
||||
color_down: mod.mpb_theme.muted
|
||||
color_hover: mod.browser_theme.lighter_background
|
||||
color_down: mod.browser_theme.muted
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -137,7 +137,7 @@ script_mod! {
|
|||
padding: Inset{left: 6 right: 6 top: 4 bottom: 4}
|
||||
spacing: 2
|
||||
draw_bg +: {
|
||||
color: mod.mpb_theme.background
|
||||
color: mod.browser_theme.background
|
||||
}
|
||||
|
||||
back_btn := MpToolButton{
|
||||
|
|
@ -162,14 +162,14 @@ script_mod! {
|
|||
padding: Inset{left: 10 right: 2 top: 0 bottom: 0}
|
||||
spacing: 6
|
||||
draw_bg +: {
|
||||
color: mod.mpb_theme.darker_background
|
||||
color: mod.browser_theme.darker_background
|
||||
}
|
||||
Icon{
|
||||
margin: Inset{top: 9 bottom: 0 left: 0 right: 0}
|
||||
icon_walk: Walk{width: 14 height: 14}
|
||||
draw_icon +: {
|
||||
svg: crate_resource("self:resources/icons/search.svg")
|
||||
color: mod.mpb_theme.dark_foreground
|
||||
color: mod.browser_theme.dark_foreground
|
||||
}
|
||||
}
|
||||
omnibox := TextInputFlat{
|
||||
|
|
@ -197,14 +197,14 @@ script_mod! {
|
|||
border_color_empty: #00000000
|
||||
}
|
||||
draw_cursor +: {
|
||||
color: mod.mpb_theme.bright_foreground
|
||||
color: mod.browser_theme.bright_foreground
|
||||
}
|
||||
draw_text +: {
|
||||
color: mod.mpb_theme.foreground
|
||||
color_hover: mod.mpb_theme.foreground
|
||||
color_focus: mod.mpb_theme.bright_foreground
|
||||
color_empty: mod.mpb_theme.dark_foreground
|
||||
color_empty_hover: mod.mpb_theme.dark_foreground
|
||||
color: mod.browser_theme.foreground
|
||||
color_hover: mod.browser_theme.foreground
|
||||
color_focus: mod.browser_theme.bright_foreground
|
||||
color_empty: mod.browser_theme.dark_foreground
|
||||
color_empty_hover: mod.browser_theme.dark_foreground
|
||||
text_style: theme.font_regular{
|
||||
font_size: 10.5
|
||||
}
|
||||
|
|
@ -217,7 +217,7 @@ script_mod! {
|
|||
icon_walk: Walk{width: 15 height: 15}
|
||||
draw_icon +: {
|
||||
svg: crate_resource("self:resources/icons/star.svg")
|
||||
color: mod.mpb_theme.dark_foreground
|
||||
color: mod.browser_theme.dark_foreground
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,12 +1,14 @@
|
|||
//! mpbrowser: a Chrome-like browser as a plain full-window Makepad app.
|
||||
//! browser: a Chrome-like browser as a plain full-window Makepad app.
|
||||
//! CEF renders the page (GPU-accelerated into a shared IOSurface texture);
|
||||
//! every bit of chrome is Makepad. Runs standalone or inside makepad-wm /
|
||||
//! Studio tiles via the shared --stdin-loop client runtime.
|
||||
|
||||
pub use makepad_widgets;
|
||||
use makepad_ai_services::port::{AiServicePort, PortEvent};
|
||||
use makepad_cef::BootstrapResult;
|
||||
use makepad_widgets::*;
|
||||
|
||||
mod ai;
|
||||
mod chrome;
|
||||
mod tabs;
|
||||
mod theme;
|
||||
|
|
@ -27,7 +29,7 @@ script_mod! {
|
|||
ui: Root{
|
||||
main_window := Window{
|
||||
window.inner_size: vec2(1280, 860)
|
||||
window.title: "mpbrowser"
|
||||
window.title: "browser"
|
||||
body +: {
|
||||
flow: Overlay
|
||||
View{
|
||||
|
|
@ -52,14 +54,14 @@ script_mod! {
|
|||
flow: Down
|
||||
padding: Inset{top: 4 bottom: 4 left: 0 right: 0}
|
||||
draw_bg +: {
|
||||
color: mod.mpb_theme.background
|
||||
color: mod.browser_theme.background
|
||||
}
|
||||
menu_new_tab := MpMenuItem{text: "New tab"}
|
||||
menu_close_tab := MpMenuItem{text: "Close tab"}
|
||||
menu_reload := MpMenuItem{text: "Reload"}
|
||||
Hr{}
|
||||
menu_gpu := MpMenuItem{text: "chrome://gpu"}
|
||||
menu_about := MpMenuItem{text: "About mpbrowser"}
|
||||
menu_about := MpMenuItem{text: "About browser"}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -155,8 +157,46 @@ pub struct App {
|
|||
focus_frame: NextFrame,
|
||||
#[rust]
|
||||
focus_retries: u32,
|
||||
#[rust]
|
||||
ai_port: Option<AiServicePort>,
|
||||
#[rust]
|
||||
ai_context: String,
|
||||
}
|
||||
|
||||
struct AiBrowserTarget<'a> {
|
||||
cx: &'a mut Cx,
|
||||
webview: &'a mut WebView,
|
||||
}
|
||||
|
||||
impl ai::BrowserTarget for AiBrowserTarget<'_> {
|
||||
fn page(&self) -> Option<ai::PageState> {
|
||||
let info = self.webview.active_info();
|
||||
info.id.map(|_| ai::PageState {
|
||||
title: info.title,
|
||||
url: info.url,
|
||||
})
|
||||
}
|
||||
|
||||
fn tabs(&self) -> Vec<ai::TabState> {
|
||||
self.webview
|
||||
.ai_tabs()
|
||||
.into_iter()
|
||||
.map(|(title, url, active)| ai::TabState { title, url, active })
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn navigate(&mut self, url: &str) -> bool {
|
||||
if self.webview.active_id().is_none() {
|
||||
return false;
|
||||
}
|
||||
self.webview.navigate(self.cx, url);
|
||||
true
|
||||
}
|
||||
|
||||
fn new_tab(&mut self, url: &str) {
|
||||
self.webview.new_tab(self.cx, url, true);
|
||||
}
|
||||
}
|
||||
|
||||
impl App {
|
||||
fn with_webview<R>(&self, cx: &mut Cx, f: impl FnOnce(&mut Cx, &mut WebView) -> R) -> Option<R> {
|
||||
|
|
@ -221,6 +261,8 @@ impl App {
|
|||
.with_webview(cx, |_cx, wv| (wv.summaries(), wv.active_info()))
|
||||
.unwrap_or_default();
|
||||
|
||||
self.refresh_ai_context(&info);
|
||||
|
||||
if let Some(mut strip) = self.ui.widget(cx, ids!(tab_strip)).borrow_mut::<TabStrip>() {
|
||||
strip.set_tabs(cx, summaries);
|
||||
}
|
||||
|
|
@ -260,16 +302,18 @@ impl App {
|
|||
});
|
||||
|
||||
let title = if info.title.is_empty() {
|
||||
"mpbrowser".to_string()
|
||||
"browser".to_string()
|
||||
} else {
|
||||
format!("{} — mpbrowser", info.title)
|
||||
format!("{} — browser", info.title)
|
||||
};
|
||||
self.ui.window(cx, ids!(main_window)).set_title(cx, &title);
|
||||
// The bar shows the page title too when the window manager hosts us.
|
||||
makepad_wm_api::set_title(cx, &title);
|
||||
|
||||
if info.render_mode != self.reported_mode && info.render_mode != "None" {
|
||||
self.reported_mode = info.render_mode.clone();
|
||||
log!(
|
||||
"mpbrowser: page rendering is {} (accelerated frames so far: {}, last blit {}us)",
|
||||
"browser: page rendering is {} (accelerated frames so far: {}, last blit {}us)",
|
||||
info.render_mode,
|
||||
info.accelerated_frames,
|
||||
info.last_blit_micros
|
||||
|
|
@ -278,6 +322,63 @@ impl App {
|
|||
self.ui.redraw(cx);
|
||||
}
|
||||
|
||||
fn refresh_ai_context(&mut self, info: &webview::ActiveInfo) {
|
||||
let context = if info.id.is_some() {
|
||||
format!("active tab: {} — {}", info.title, info.url)
|
||||
} else {
|
||||
"no active tab".to_string()
|
||||
};
|
||||
if context == self.ai_context {
|
||||
return;
|
||||
}
|
||||
self.ai_context = context.clone();
|
||||
if let Some(port) = self.ai_port.as_ref() {
|
||||
port.set_context(&context);
|
||||
}
|
||||
}
|
||||
|
||||
fn refresh_ai_context_from_webview(&mut self, cx: &mut Cx) {
|
||||
let info = self
|
||||
.with_webview(cx, |_cx, webview| webview.active_info())
|
||||
.unwrap_or_default();
|
||||
self.refresh_ai_context(&info);
|
||||
}
|
||||
|
||||
fn drain_ai_port(&mut self, cx: &mut Cx, event: &Event) {
|
||||
let events = match self.ai_port.as_mut() {
|
||||
Some(port) => port.handle_event(cx, event),
|
||||
None => return,
|
||||
};
|
||||
for event in events {
|
||||
match event {
|
||||
PortEvent::Registered(endpoint) => {
|
||||
log!("browser: AI service registered as {}", endpoint.as_str());
|
||||
self.ai_context.clear();
|
||||
self.refresh_ai_context_from_webview(cx);
|
||||
}
|
||||
PortEvent::Call(call) => {
|
||||
let result = self
|
||||
.with_webview(cx, |cx, webview| {
|
||||
let mut target = AiBrowserTarget { cx, webview };
|
||||
ai::answer(&call, &mut target)
|
||||
})
|
||||
.unwrap_or_else(|| {
|
||||
makepad_ai_services::wire::ToolResult::unavailable(
|
||||
&call.call_id,
|
||||
"the browser view is not ready",
|
||||
)
|
||||
});
|
||||
if let Some(port) = self.ai_port.as_ref() {
|
||||
port.reply(result);
|
||||
}
|
||||
self.refresh_chrome(cx);
|
||||
}
|
||||
PortEvent::Cancel { .. } | PortEvent::ChatOpen { .. } => {}
|
||||
PortEvent::Subscribe { .. } | PortEvent::Unsubscribe { .. } => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn navigate_from_omnibox(&mut self, cx: &mut Cx, text: &str) {
|
||||
let url = tabs::resolve_omnibox(text);
|
||||
if url.is_empty() {
|
||||
|
|
@ -361,14 +462,20 @@ impl App {
|
|||
|
||||
impl MatchEvent for App {
|
||||
fn handle_startup(&mut self, cx: &mut Cx) {
|
||||
// A warm-pool standby is not a running Browser: its service opens on
|
||||
// `WmEvent::Adopted`, never while dormant — the assistant must not
|
||||
// steer a page nobody can see.
|
||||
if !makepad_wm_api::warm_start() {
|
||||
self.ai_port = AiServicePort::open(cx, ai::manifest());
|
||||
}
|
||||
match makepad_cef::startup_phases() {
|
||||
Some((bundle_ms, exec_gap_ms)) => log!(
|
||||
"mpbrowser: window up at {} ms after main (app bundle prepared in {} ms, exec-to-main gap {} ms)",
|
||||
"browser: window up at {} ms after main (app bundle prepared in {} ms, exec-to-main gap {} ms)",
|
||||
uptime_ms(),
|
||||
bundle_ms,
|
||||
exec_gap_ms
|
||||
),
|
||||
None => log!("mpbrowser: window up at {} ms after main", uptime_ms()),
|
||||
None => log!("browser: window up at {} ms after main", uptime_ms()),
|
||||
}
|
||||
let urls = initial_urls();
|
||||
if urls.is_empty() {
|
||||
|
|
@ -463,8 +570,8 @@ impl MatchEvent for App {
|
|||
.with_webview(cx, |_cx, wv| wv.active_info())
|
||||
.unwrap_or_default();
|
||||
let html = format!(
|
||||
"<!doctype html><title>About mpbrowser</title><body style=\"background:{bg};color:{fg};font:14px -apple-system,Helvetica,sans-serif;padding:32px\">\
|
||||
<h2 style=\"font-weight:500\">mpbrowser</h2>\
|
||||
"<!doctype html><title>About browser</title><body style=\"background:{bg};color:{fg};font:14px -apple-system,Helvetica,sans-serif;padding:32px\">\
|
||||
<h2 style=\"font-weight:500\">browser</h2>\
|
||||
<p>Makepad chrome, Chromium Embedded Framework {cef} page rendering.</p>\
|
||||
<p>Page rendering path: <b>{mode}</b> (accelerated frames: {frames}, last GPU blit: {blit}µs)</p>\
|
||||
<p>ANGLE backend: {angle}</p></body>",
|
||||
|
|
@ -527,8 +634,8 @@ impl AppMain for App {
|
|||
fn script_mod(vm: &mut ScriptVm) -> ScriptValue {
|
||||
crate::makepad_widgets::script_mod(vm);
|
||||
// The family theme bridge retints the stock widgets from the WM
|
||||
// theme; the chrome roles go into mod.mpb_theme.
|
||||
mp_theme::apply(vm);
|
||||
// theme; the chrome roles go into mod.browser_theme.
|
||||
makepad_wm_theme::apply(vm);
|
||||
palette().apply(vm);
|
||||
chrome::script_mod(vm);
|
||||
webview::script_mod(vm);
|
||||
|
|
@ -536,6 +643,22 @@ impl AppMain for App {
|
|||
}
|
||||
|
||||
fn handle_event(&mut self, cx: &mut Cx, event: &Event) {
|
||||
// The window manager asked politely (SUPER+W): go now, ahead of the
|
||||
// kill that follows its grace. A warm-pool browser needs no waking —
|
||||
// CEF paints nothing until a tile presents it.
|
||||
if let Event::Custom(json) = event {
|
||||
match makepad_wm_api::WmEvent::parse(json) {
|
||||
Some(makepad_wm_api::WmEvent::Adopted) if self.ai_port.is_none() => {
|
||||
self.ai_port = AiServicePort::open(cx, ai::manifest());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
if let Some(makepad_wm_api::WmEvent::CloseRequested) = makepad_wm_api::WmEvent::parse(json) {
|
||||
cx.quit();
|
||||
return;
|
||||
}
|
||||
}
|
||||
self.drain_ai_port(cx, event);
|
||||
if let Event::KeyDown(ke) = event {
|
||||
if self.handle_shortcut(cx, ke) {
|
||||
return;
|
||||
|
|
@ -622,5 +745,5 @@ pub fn app_main() {
|
|||
|
||||
#[cfg(any(target_arch = "wasm32", target_os = "android", target_env = "ohos"))]
|
||||
pub fn app_main() {
|
||||
panic!("mpbrowser is desktop-only");
|
||||
panic!("browser is desktop-only");
|
||||
}
|
||||
|
|
@ -1,14 +1,14 @@
|
|||
//! The browser-chrome palette. Theming lives in splash: the chrome reads
|
||||
//! `mod.mpb_theme.*` (tab strip, toolbar, omnibox, icon roles), which this
|
||||
//! `mod.browser_theme.*` (tab strip, toolbar, omnibox, icon roles), which this
|
||||
//! module evaluates into the VM before the UI modules.
|
||||
//!
|
||||
//! Under makepad-wm the roles come from the WM's theme.splash
|
||||
//! (`MPWM_THEME_SPLASH`, line-scanned by `mp_theme`, the family bridge);
|
||||
//! (`MAKEPAD_WM_THEME_SPLASH`, line-scanned by `makepad_wm_theme`, the family bridge);
|
||||
//! standalone runs get Chrome's own dark palette.
|
||||
|
||||
use makepad_widgets::*;
|
||||
|
||||
/// Chrome-dark roles, keyed like the mpwm theme so one mapping serves both.
|
||||
/// Chrome-dark roles, keyed like the wm theme so one mapping serves both.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Palette {
|
||||
/// Tab strip background (the "frame").
|
||||
|
|
@ -43,10 +43,10 @@ impl Palette {
|
|||
}
|
||||
}
|
||||
|
||||
/// The WM palette when mpwm exported one, else Chrome dark.
|
||||
/// The WM palette when wm exported one, else Chrome dark.
|
||||
pub fn current() -> Self {
|
||||
let fallback = Self::chrome_dark();
|
||||
let Some(p) = mp_theme::current() else {
|
||||
let Some(p) = makepad_wm_theme::current() else {
|
||||
return fallback;
|
||||
};
|
||||
Self {
|
||||
|
|
@ -63,11 +63,11 @@ impl Palette {
|
|||
}
|
||||
}
|
||||
|
||||
/// The `mod.mpb_theme = {...}` splash source. Runtime-evaluated, so plain
|
||||
/// The `mod.browser_theme = {...}` splash source. Runtime-evaluated, so plain
|
||||
/// `#hex` (the `#x` escape is a proc-macro-only hazard).
|
||||
pub fn splash_source(&self) -> String {
|
||||
format!(
|
||||
"mod.mpb_theme = {{\n\
|
||||
"mod.browser_theme = {{\n\
|
||||
\x20 darker_background: {}\n\
|
||||
\x20 background: {}\n\
|
||||
\x20 dark_background: {}\n\
|
||||
|
|
@ -93,13 +93,13 @@ impl Palette {
|
|||
)
|
||||
}
|
||||
|
||||
/// Evaluate `mod.mpb_theme` into the VM. Call after
|
||||
/// Evaluate `mod.browser_theme` into the VM. Call after
|
||||
/// `makepad_widgets::script_mod(vm)` and before the chrome modules.
|
||||
pub fn apply(&self, vm: &mut ScriptVm) {
|
||||
let script_mod_id = ScriptMod {
|
||||
cargo_manifest_path: env!("CARGO_MANIFEST_DIR").to_string(),
|
||||
module_path: "mpb_theme".to_string(),
|
||||
file: "mpb_theme.splash".to_string(),
|
||||
module_path: "browser_theme".to_string(),
|
||||
file: "browser_theme.splash".to_string(),
|
||||
line: 0,
|
||||
column: 0,
|
||||
code: self.splash_source(),
|
||||
|
|
@ -107,7 +107,7 @@ impl Palette {
|
|||
};
|
||||
vm.eval(script_mod_id);
|
||||
for e in vm.take_errors() {
|
||||
log!("mpbrowser theme: {}", e);
|
||||
log!("browser theme: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -129,7 +129,7 @@ impl Palette {
|
|||
.c{{display:flex;height:100%;align-items:center;justify-content:center;\
|
||||
flex-direction:column;gap:10px}}.n{{font-size:28px;letter-spacing:1px;color:{fgb}}}\
|
||||
.s{{color:{fgd}}}</style></head><body><div class=c>\
|
||||
<div class=n>mpbrowser</div><div class=s>Type a URL or search in the box above</div>\
|
||||
<div class=n>browser</div><div class=s>Type a URL or search in the box above</div>\
|
||||
</div></body></html>",
|
||||
bg = self.darker_background,
|
||||
fg = self.foreground,
|
||||
|
|
@ -11,9 +11,9 @@
|
|||
use crate::tabs::{TabId, TabModel, TabSummary};
|
||||
// The surface policy is the stock Browser widget's — one source of truth for
|
||||
// how a CEF page survives a resize.
|
||||
use makepad_widgets::browser::{
|
||||
needs_new_surface, surface_alloc, Browser as BrowserKeys, RESIZE_INTERVAL, SETTLE,
|
||||
};
|
||||
#[cfg(target_os = "macos")]
|
||||
use makepad_widgets::browser::{needs_new_surface, surface_alloc, SETTLE};
|
||||
use makepad_widgets::browser::{Browser as BrowserKeys, RESIZE_INTERVAL};
|
||||
use makepad_widgets::image::DrawImage;
|
||||
use makepad_widgets::*;
|
||||
|
||||
|
|
@ -27,13 +27,13 @@ script_mod! {
|
|||
width: Fill
|
||||
height: Fill
|
||||
draw_empty +: {
|
||||
color: uniform(mod.mpb_theme.darker_background)
|
||||
color: uniform(mod.browser_theme.darker_background)
|
||||
pixel: fn() {
|
||||
return self.color
|
||||
}
|
||||
}
|
||||
draw_status +: {
|
||||
color: mod.mpb_theme.dark_foreground
|
||||
color: mod.browser_theme.dark_foreground
|
||||
text_style: theme.font_regular{
|
||||
font_size: 11
|
||||
}
|
||||
|
|
@ -102,11 +102,11 @@ pub struct WebView {
|
|||
pump_started: bool,
|
||||
}
|
||||
|
||||
/// Env-gated resize tracing (`MPB_TRACE=1`): timestamps + sizes on every
|
||||
/// Env-gated resize tracing (`MAKEPAD_BROWSER_TRACE=1`): timestamps + sizes on every
|
||||
/// draw, resize and target swap. Debug rig — not for committing.
|
||||
pub fn trace_on() -> bool {
|
||||
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
|
||||
*ON.get_or_init(|| std::env::var_os("MPB_TRACE").is_some())
|
||||
*ON.get_or_init(|| std::env::var_os("MAKEPAD_BROWSER_TRACE").is_some())
|
||||
}
|
||||
|
||||
macro_rules! trace {
|
||||
|
|
@ -177,6 +177,18 @@ impl WebView {
|
|||
self.tabs.summaries()
|
||||
}
|
||||
|
||||
/// Title, URL and active marker for the browser's AI `tabs` tool.
|
||||
pub fn ai_tabs(&self) -> Vec<(String, String, bool)> {
|
||||
self.tabs
|
||||
.tabs
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, tab)| {
|
||||
(tab.display_title(), tab.url.clone(), index == self.tabs.active)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn active_info(&self) -> ActiveInfo {
|
||||
let Some(tab) = self.tabs.active() else {
|
||||
return ActiveInfo::default();
|
||||
|
|
@ -520,7 +532,7 @@ impl WebView {
|
|||
}
|
||||
if !self.first_frame_logged {
|
||||
self.first_frame_logged = true;
|
||||
log!("mpbrowser: first page frame at {} ms", crate::uptime_ms());
|
||||
log!("browser: first page frame at {} ms", crate::uptime_ms());
|
||||
}
|
||||
}
|
||||
let generation = browser.nav_generation();
|
||||
|
|
@ -705,7 +717,7 @@ impl Widget for WebView {
|
|||
Ok(()) => {
|
||||
self.cef_ready = true;
|
||||
log!(
|
||||
"mpbrowser: CEF {} initialized at {} ms",
|
||||
"browser: CEF {} initialized at {} ms",
|
||||
makepad_cef::CEF_VERSION,
|
||||
crate::uptime_ms()
|
||||
);
|
||||
|
|
@ -18,7 +18,7 @@ gltf = ["dep:makepad-fab-loader-gltf"]
|
|||
makepad-widgets = { path = "../../widgets", version = "2.0.0" }
|
||||
makepad-fab-shell = { path = "../../libs/fab", version = "0.1.0" }
|
||||
makepad-fab-loader-gltf = { path = "loaders/gltf", version = "0.1.0", optional = true }
|
||||
makepad-video = { path = "../../platform/video", version = "1.0.0" }
|
||||
makepad-video = { package = "makepad-platform-video", path = "../../platform/video", version = "1.0.0" }
|
||||
makepad-zune-png = { path = "../../libs/zune/zune-png", version = "0.5.2" }
|
||||
|
||||
[dev-dependencies]
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
//! Encode a directory of `frame_%06d.png` files into an H.264 mp4 via the
|
||||
//! platform hardware encoder (`makepad-video` / VideoToolbox on macOS).
|
||||
//! platform hardware encoder (`makepad-platform-video` / VideoToolbox on macOS).
|
||||
//!
|
||||
//! ```text
|
||||
//! frames_to_mp4 <dir> <out.mp4> [--fps 24] [--start N] [--end M] [--bitrate BPS] [--crf N]
|
||||
|
|
|
|||
18
apps/fabric/Cargo.toml
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
[package]
|
||||
name = "makepad-fabric"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
license = "MIT OR Apache-2.0"
|
||||
description = "Fabric: a photo in, a fitted sewing pattern out — body model, measurements, draft, cut sheet"
|
||||
|
||||
[dependencies]
|
||||
# The window manager's vocabulary: the bar's title and the polite close.
|
||||
makepad-wm-api = { path = "../../libs/wm_api" }
|
||||
makepad-widgets = { path = "../../widgets" }
|
||||
makepad-fabric-measure = { path = "../../libs/fabric/measure" }
|
||||
makepad-fabric-draft = { path = "../../libs/fabric/draft" }
|
||||
# The body model runs in-process (Metal on the Mac, CUDA on a box); the
|
||||
# hub does install, licence acknowledgement and weight location.
|
||||
makepad-ai-body = { path = "../../libs/ai/models/body" }
|
||||
makepad-ai-hub = { path = "../../libs/ai/hub", default-features = false, features = ["local", "body-native"] }
|
||||
makepad-ai-hub-ui = { path = "../../libs/ai/hub_ui" }
|
||||
539
apps/fabric/src/body_view.rs
Normal file
|
|
@ -0,0 +1,539 @@
|
|||
use makepad_fabric_measure::{BodyMesh, Line, Measured, Ring};
|
||||
use makepad_widgets::*;
|
||||
use std::sync::Arc;
|
||||
|
||||
script_mod! {
|
||||
use mod.prelude.widgets_internal.*
|
||||
use mod.widgets.*
|
||||
|
||||
set_type_default() do #(DrawBodyPoint::script_shader(vm)) {
|
||||
..mod.draw.DrawQuad
|
||||
pixel: fn() {
|
||||
let d = length(self.pos - vec2(0.5, 0.5))
|
||||
let a = clamp((0.5 - d) * 7.0, 0.0, 1.0)
|
||||
let near = #x80e7ff
|
||||
let far = #x27435d
|
||||
let c = far.mix(near, 1.0 - self.depth)
|
||||
return vec4(c.xyz * a, a)
|
||||
}
|
||||
}
|
||||
|
||||
set_type_default() do #(DrawFabricLine::script_shader(vm)) {
|
||||
..mod.draw.DrawQuad
|
||||
pixel: fn() {
|
||||
// A line as a distance field inside its bounding quad. The
|
||||
// endpoints are LOCAL to the quad (the turtle may still shift
|
||||
// rect_pos after the instance is written), like the chart's
|
||||
// segment shader.
|
||||
let p = self.pos * self.rect_size
|
||||
let ab = self.p1 - self.p0
|
||||
let t = clamp(dot(p - self.p0, ab) / max(dot(ab, ab), 0.0001), 0.0, 1.0)
|
||||
let d = length(p - (self.p0 + ab * t))
|
||||
let aa = 1.0 - smoothstep(self.half_width - 0.6, self.half_width + 0.6, d)
|
||||
let alpha = aa * self.color.w
|
||||
return vec4(self.color.xyz * alpha, alpha)
|
||||
}
|
||||
}
|
||||
|
||||
mod.widgets.FabricBodyViewBase = #(FabricBodyView::register_widget(vm))
|
||||
mod.widgets.FabricBodyView = set_type_default() do mod.widgets.FabricBodyViewBase {
|
||||
width: Fill
|
||||
height: Fill
|
||||
draw_bg +: {color: #x11161d}
|
||||
draw_point +: {}
|
||||
draw_line +: {}
|
||||
draw_text +: {
|
||||
color: #x9aa8b7
|
||||
text_style: theme.font_regular{font_size: 9.0}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Script, ScriptHook)]
|
||||
#[repr(C)]
|
||||
pub struct DrawBodyPoint {
|
||||
#[deref]
|
||||
draw_super: DrawQuad,
|
||||
#[live]
|
||||
depth: f32,
|
||||
}
|
||||
|
||||
#[derive(Script, ScriptHook)]
|
||||
#[repr(C)]
|
||||
pub struct DrawFabricLine {
|
||||
#[deref]
|
||||
draw_super: DrawQuad,
|
||||
#[live]
|
||||
pub color: Vec4f,
|
||||
#[live]
|
||||
p0: Vec2f,
|
||||
#[live]
|
||||
p1: Vec2f,
|
||||
#[live]
|
||||
half_width: f32,
|
||||
}
|
||||
|
||||
impl DrawFabricLine {
|
||||
pub fn segment(&mut self, cx: &mut Cx2d, from: DVec2, to: DVec2, width: f64) {
|
||||
if (to - from).length() < 0.01 {
|
||||
return;
|
||||
}
|
||||
let half = width * 0.5;
|
||||
let pad = half + 1.0;
|
||||
let min = dvec2(from.x.min(to.x) - pad, from.y.min(to.y) - pad);
|
||||
let max = dvec2(from.x.max(to.x) + pad, from.y.max(to.y) + pad);
|
||||
self.p0 = v2f(from - min);
|
||||
self.p1 = v2f(to - min);
|
||||
self.half_width = half as f32;
|
||||
self.draw_abs(cx, Rect { pos: min, size: max - min });
|
||||
}
|
||||
}
|
||||
|
||||
fn v2f(value: DVec2) -> Vec2f {
|
||||
Vec2f {
|
||||
x: value.x as f32,
|
||||
y: value.y as f32,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct BodyDrag {
|
||||
from: DVec2,
|
||||
yaw: f64,
|
||||
pitch: f64,
|
||||
pan: DVec2,
|
||||
panning: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct BodyPoseMapping {
|
||||
ring_vertices: Vec<Vec<usize>>,
|
||||
line_vertices: Vec<[usize; 2]>,
|
||||
}
|
||||
|
||||
pub(crate) fn map_measurements_to_vertices(
|
||||
mesh: &BodyMesh,
|
||||
measured: &Measured,
|
||||
) -> BodyPoseMapping {
|
||||
let nearest = |point| nearest_vertex_index(&mesh.vertices, measured.scale, point);
|
||||
BodyPoseMapping {
|
||||
ring_vertices: measured
|
||||
.rings
|
||||
.iter()
|
||||
.map(|ring| ring.points.iter().copied().map(nearest).collect())
|
||||
.collect(),
|
||||
line_vertices: measured
|
||||
.lines
|
||||
.iter()
|
||||
.map(|line| [nearest(line.from), nearest(line.to)])
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn nearest_vertex_index(vertices: &[[f32; 3]], scale: f32, point: [f32; 3]) -> usize {
|
||||
vertices
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, vertex)| {
|
||||
let dx = vertex[0] * scale - point[0];
|
||||
let dy = vertex[1] * scale - point[1];
|
||||
let dz = vertex[2] * scale - point[2];
|
||||
(index, dx * dx + dy * dy + dz * dz)
|
||||
})
|
||||
.min_by(|left, right| left.1.total_cmp(&right.1))
|
||||
.map(|(index, _)| index)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
fn mirror_x(value: f64, mirrored: bool) -> f64 {
|
||||
if mirrored { -value } else { value }
|
||||
}
|
||||
|
||||
fn bounds(points: &[[f32; 3]], scale: f32) -> Option<([f32; 3], f32)> {
|
||||
let mut min = [f32::INFINITY; 3];
|
||||
let mut max = [f32::NEG_INFINITY; 3];
|
||||
for point in points {
|
||||
for axis in 0..3 {
|
||||
let value = point[axis] * scale;
|
||||
min[axis] = min[axis].min(value);
|
||||
max[axis] = max[axis].max(value);
|
||||
}
|
||||
}
|
||||
if !min[0].is_finite() {
|
||||
return None;
|
||||
}
|
||||
let centre = [
|
||||
(min[0] + max[0]) * 0.5,
|
||||
(min[1] + max[1]) * 0.5,
|
||||
(min[2] + max[2]) * 0.5,
|
||||
];
|
||||
let dx = max[0] - min[0];
|
||||
let dy = max[1] - min[1];
|
||||
let dz = max[2] - min[2];
|
||||
let radius = (dx * dx + dy * dy + dz * dz).sqrt().max(1.0) * 0.5;
|
||||
Some((centre, radius))
|
||||
}
|
||||
|
||||
#[derive(Script, ScriptHook, Widget)]
|
||||
pub struct FabricBodyView {
|
||||
#[uid]
|
||||
uid: WidgetUid,
|
||||
#[source]
|
||||
source: ScriptObjectRef,
|
||||
#[walk]
|
||||
walk: Walk,
|
||||
#[layout]
|
||||
layout: Layout,
|
||||
#[redraw]
|
||||
#[area]
|
||||
area: Area,
|
||||
#[live]
|
||||
draw_bg: DrawColor,
|
||||
#[live]
|
||||
draw_point: DrawBodyPoint,
|
||||
#[live]
|
||||
draw_line: DrawFabricLine,
|
||||
#[live]
|
||||
draw_text: DrawText,
|
||||
#[rust]
|
||||
mesh: Option<Arc<BodyMesh>>,
|
||||
#[rust]
|
||||
posed: Option<Arc<Vec<[f32; 3]>>>,
|
||||
#[rust]
|
||||
rings: Vec<Ring>,
|
||||
#[rust]
|
||||
lines: Vec<Line>,
|
||||
#[rust]
|
||||
pose_mapping: BodyPoseMapping,
|
||||
#[rust]
|
||||
mesh_scale: f32,
|
||||
#[rust]
|
||||
centre: [f32; 3],
|
||||
#[rust]
|
||||
radius: f32,
|
||||
#[rust(0.35)]
|
||||
yaw: f64,
|
||||
#[rust(-0.06)]
|
||||
pitch: f64,
|
||||
#[rust(1.0)]
|
||||
zoom: f64,
|
||||
#[rust]
|
||||
pan: DVec2,
|
||||
#[rust]
|
||||
drag: Option<BodyDrag>,
|
||||
#[rust(true)]
|
||||
mirrored: bool,
|
||||
}
|
||||
|
||||
impl FabricBodyView {
|
||||
pub fn set_body(
|
||||
&mut self,
|
||||
cx: &mut Cx,
|
||||
mesh: Arc<BodyMesh>,
|
||||
measured: &Measured,
|
||||
pose_mapping: BodyPoseMapping,
|
||||
) {
|
||||
self.mesh_scale = measured.scale;
|
||||
self.rings = measured.rings.clone();
|
||||
self.lines = measured.lines.clone();
|
||||
self.pose_mapping = pose_mapping;
|
||||
if self.posed.is_none() {
|
||||
self.fit_bounds(&mesh.vertices);
|
||||
self.reset_camera();
|
||||
}
|
||||
self.mesh = Some(mesh);
|
||||
self.redraw(cx);
|
||||
}
|
||||
|
||||
pub fn set_pose(&mut self, cx: &mut Cx, posed: Option<Arc<Vec<[f32; 3]>>>) {
|
||||
match posed {
|
||||
Some(posed)
|
||||
if self
|
||||
.mesh
|
||||
.as_ref()
|
||||
.is_some_and(|mesh| mesh.vertices.len() == posed.len()) =>
|
||||
{
|
||||
if self.posed.is_none() {
|
||||
self.fit_bounds(posed.as_slice());
|
||||
self.reset_camera();
|
||||
}
|
||||
self.posed = Some(posed);
|
||||
}
|
||||
_ => {
|
||||
self.posed = None;
|
||||
if let Some(fit) = self
|
||||
.mesh
|
||||
.as_ref()
|
||||
.and_then(|mesh| bounds(&mesh.vertices, self.mesh_scale))
|
||||
{
|
||||
(self.centre, self.radius) = fit;
|
||||
self.reset_camera();
|
||||
}
|
||||
}
|
||||
}
|
||||
self.redraw(cx);
|
||||
}
|
||||
|
||||
pub fn set_mirrored(&mut self, cx: &mut Cx, mirrored: bool) {
|
||||
self.mirrored = mirrored;
|
||||
self.redraw(cx);
|
||||
}
|
||||
|
||||
fn fit_bounds(&mut self, points: &[[f32; 3]]) {
|
||||
if let Some((centre, radius)) = bounds(points, self.mesh_scale) {
|
||||
self.centre = centre;
|
||||
self.radius = radius;
|
||||
}
|
||||
}
|
||||
|
||||
fn reset_camera(&mut self) {
|
||||
self.yaw = 0.35;
|
||||
self.pitch = -0.06;
|
||||
self.zoom = 1.0;
|
||||
self.pan = dvec2(0.0, 0.0);
|
||||
}
|
||||
|
||||
fn project(&self, point: [f32; 3], mesh_point: bool, rect: Rect) -> Option<(DVec2, f32)> {
|
||||
let scale = if mesh_point { self.mesh_scale } else { 1.0 } as f64;
|
||||
let x = point[0] as f64 * scale - self.centre[0] as f64;
|
||||
let y = point[1] as f64 * scale - self.centre[1] as f64;
|
||||
let z = point[2] as f64 * scale - self.centre[2] as f64;
|
||||
let (sy, cy) = self.yaw.sin_cos();
|
||||
let (sp, cp) = self.pitch.sin_cos();
|
||||
let rx = mirror_x(cy * x + sy * z, self.mirrored);
|
||||
let rz = -sy * x + cy * z;
|
||||
let ry = cp * y - sp * rz;
|
||||
let rz = sp * y + cp * rz;
|
||||
let fov = 35.0_f64.to_radians();
|
||||
let fit_distance = self.radius as f64 / (fov * 0.5).tan() * 1.2;
|
||||
let camera_z = fit_distance / self.zoom.max(0.08) - rz;
|
||||
if camera_z <= 0.01 {
|
||||
return None;
|
||||
}
|
||||
let focal = rect.size.y.max(1.0) * 0.5 / (fov * 0.5).tan();
|
||||
let centre = rect.pos + rect.size * 0.5 + self.pan;
|
||||
let screen = dvec2(centre.x + focal * rx / camera_z, centre.y - focal * ry / camera_z);
|
||||
Some((screen, camera_z as f32))
|
||||
}
|
||||
|
||||
fn projected_polyline(&self, points: &[[f32; 3]], rect: Rect) -> Vec<(DVec2, f32)> {
|
||||
points
|
||||
.iter()
|
||||
.filter_map(|point| self.project(*point, false, rect))
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
impl Widget for FabricBodyView {
|
||||
fn draw_walk(&mut self, cx: &mut Cx2d, _scope: &mut Scope, walk: Walk) -> DrawStep {
|
||||
let rect = cx.walk_turtle(walk);
|
||||
self.draw_bg.draw_abs(cx, rect);
|
||||
cx.push_clip_rect(rect);
|
||||
let Some(mesh) = self.mesh.as_ref() else {
|
||||
self.draw_text.color = Vec4f {
|
||||
x: 0.48,
|
||||
y: 0.55,
|
||||
z: 0.62,
|
||||
w: 1.0,
|
||||
};
|
||||
self.draw_text.draw_abs(
|
||||
cx,
|
||||
rect.pos + rect.size * 0.5 - dvec2(70.0, 5.0),
|
||||
"drop a photo to start",
|
||||
);
|
||||
cx.pop_clip_rect();
|
||||
cx.add_aligned_rect_area(&mut self.area, rect);
|
||||
return DrawStep::done();
|
||||
};
|
||||
let posed = self
|
||||
.posed
|
||||
.as_deref()
|
||||
.filter(|posed| posed.len() == mesh.vertices.len())
|
||||
.map(Vec::as_slice);
|
||||
let display_vertices = posed.unwrap_or(mesh.vertices.as_slice());
|
||||
|
||||
let mut points: Vec<(DVec2, f32)> = display_vertices
|
||||
.iter()
|
||||
.filter_map(|point| self.project(*point, true, rect))
|
||||
.collect();
|
||||
points.sort_by(|a, b| b.1.total_cmp(&a.1));
|
||||
let min_depth = points.iter().map(|point| point.1).fold(f32::INFINITY, f32::min);
|
||||
let max_depth = points
|
||||
.iter()
|
||||
.map(|point| point.1)
|
||||
.fold(f32::NEG_INFINITY, f32::max);
|
||||
let depth_span = (max_depth - min_depth).max(0.001);
|
||||
self.draw_point.begin_many_instances(cx);
|
||||
for (point, depth) in points {
|
||||
self.draw_point.depth = (depth - min_depth) / depth_span;
|
||||
self.draw_point.draw_abs(
|
||||
cx,
|
||||
Rect {
|
||||
pos: point - dvec2(1.25, 1.25),
|
||||
size: dvec2(2.5, 2.5),
|
||||
},
|
||||
);
|
||||
}
|
||||
self.draw_point.end_many_instances(cx);
|
||||
|
||||
let rings: Vec<(String, Vec<(DVec2, f32)>)> = self
|
||||
.rings
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(ring_index, ring)| {
|
||||
let points = match (
|
||||
posed,
|
||||
self.pose_mapping.ring_vertices.get(ring_index),
|
||||
) {
|
||||
(Some(posed), Some(indices)) if indices.len() == ring.points.len() => indices
|
||||
.iter()
|
||||
.filter_map(|&index| self.project(*posed.get(index)?, true, rect))
|
||||
.collect(),
|
||||
_ => self.projected_polyline(&ring.points, rect),
|
||||
};
|
||||
(ring.key.replace('_', " "), points)
|
||||
})
|
||||
.collect();
|
||||
let lines: Vec<(DVec2, DVec2)> = self
|
||||
.lines
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(line_index, line)| {
|
||||
let (from, to, mesh_points) = match (
|
||||
posed,
|
||||
self.pose_mapping.line_vertices.get(line_index),
|
||||
) {
|
||||
(Some(posed), Some([from, to])) =>
|
||||
(*posed.get(*from)?, *posed.get(*to)?, true),
|
||||
_ => (line.from, line.to, false),
|
||||
};
|
||||
Some((
|
||||
self.project(from, mesh_points, rect)?.0,
|
||||
self.project(to, mesh_points, rect)?.0,
|
||||
))
|
||||
})
|
||||
.collect();
|
||||
|
||||
self.draw_line.begin_many_instances(cx);
|
||||
self.draw_line.color = Vec4f {
|
||||
x: 1.0,
|
||||
y: 0.38,
|
||||
z: 0.18,
|
||||
w: 0.92,
|
||||
};
|
||||
for (_, points) in &rings {
|
||||
for pair in points.windows(2) {
|
||||
self.draw_line.segment(cx, pair[0].0, pair[1].0, 1.5);
|
||||
}
|
||||
if let (Some(first), Some(last)) = (points.first(), points.last()) {
|
||||
self.draw_line.segment(cx, last.0, first.0, 1.5);
|
||||
}
|
||||
}
|
||||
self.draw_line.color = Vec4f {
|
||||
x: 0.42,
|
||||
y: 0.78,
|
||||
z: 1.0,
|
||||
w: 0.9,
|
||||
};
|
||||
for (from, to) in lines {
|
||||
self.draw_line.segment(cx, from, to, 1.25);
|
||||
}
|
||||
self.draw_line.end_many_instances(cx);
|
||||
|
||||
self.draw_text.color = Vec4f {
|
||||
x: 1.0,
|
||||
y: 0.66,
|
||||
z: 0.48,
|
||||
w: 1.0,
|
||||
};
|
||||
for (key, points) in rings {
|
||||
if let Some(front) = points.iter().min_by(|a, b| a.1.total_cmp(&b.1)) {
|
||||
self.draw_text
|
||||
.draw_abs(cx, front.0 + dvec2(4.0, -5.0), &key);
|
||||
}
|
||||
}
|
||||
cx.pop_clip_rect();
|
||||
cx.add_aligned_rect_area(&mut self.area, rect);
|
||||
DrawStep::done()
|
||||
}
|
||||
|
||||
fn handle_event(&mut self, cx: &mut Cx, event: &Event, _scope: &mut Scope) {
|
||||
match event.hits(cx, self.area) {
|
||||
Hit::FingerDown(event) if event.device.is_primary_hit() => {
|
||||
if event.tap_count >= 2 {
|
||||
self.reset_camera();
|
||||
self.redraw(cx);
|
||||
return;
|
||||
}
|
||||
self.drag = Some(BodyDrag {
|
||||
from: event.abs,
|
||||
yaw: self.yaw,
|
||||
pitch: self.pitch,
|
||||
pan: self.pan,
|
||||
panning: event.modifiers.shift,
|
||||
});
|
||||
}
|
||||
Hit::FingerMove(event) => {
|
||||
if let Some(drag) = self.drag {
|
||||
let delta = event.abs - drag.from;
|
||||
if drag.panning {
|
||||
self.pan = drag.pan + delta;
|
||||
} else {
|
||||
self.yaw = drag.yaw - delta.x * 0.008;
|
||||
self.pitch = (drag.pitch + delta.y * 0.007).clamp(-1.35, 1.35);
|
||||
}
|
||||
self.redraw(cx);
|
||||
}
|
||||
}
|
||||
Hit::FingerUp(_) => self.drag = None,
|
||||
Hit::FingerScroll(event) => {
|
||||
self.zoom = (self.zoom * (-event.scroll.y * 0.004).exp()).clamp(0.15, 12.0);
|
||||
self.redraw(cx);
|
||||
}
|
||||
Hit::FingerHoverIn(_) | Hit::FingerHoverOver(_) => {
|
||||
cx.set_cursor(MouseCursor::Grab);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn nearest_vertex_mapping_uses_scaled_rest_mesh_positions() {
|
||||
let mesh = BodyMesh {
|
||||
vertices: vec![[0.0, 0.0, 0.0], [4.0, 0.0, 0.0], [9.0, 0.0, 0.0]],
|
||||
faces: Vec::new(),
|
||||
landmarks: None,
|
||||
};
|
||||
let measured = Measured {
|
||||
values: makepad_fabric_measure::Measurements::sample(),
|
||||
scale: 2.0,
|
||||
rings: vec![Ring {
|
||||
key: "test_ring",
|
||||
y_cm: 0.0,
|
||||
points: vec![[0.2, 0.0, 0.0], [7.5, 0.0, 0.0]],
|
||||
skin_perimeter_cm: 0.0,
|
||||
tape_perimeter_cm: 0.0,
|
||||
}],
|
||||
lines: vec![Line {
|
||||
key: "test_line",
|
||||
from: [7.5, 0.0, 0.0],
|
||||
to: [17.0, 0.0, 0.0],
|
||||
}],
|
||||
};
|
||||
let mapping = map_measurements_to_vertices(&mesh, &measured);
|
||||
assert_eq!(mapping.ring_vertices, vec![vec![0, 1]]);
|
||||
assert_eq!(mapping.line_vertices, vec![[1, 2]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mirror_transform_only_flips_horizontal_view_axis() {
|
||||
assert_eq!(mirror_x(12.5, false), 12.5);
|
||||
assert_eq!(mirror_x(12.5, true), -12.5);
|
||||
assert_eq!(mirror_x(-3.0, true), 3.0);
|
||||
}
|
||||
}
|
||||
303
apps/fabric/src/camera.rs
Normal file
|
|
@ -0,0 +1,303 @@
|
|||
use makepad_widgets::{
|
||||
makepad_platform::video::{
|
||||
CameraFrameLayout, CameraFrameRef, VideoFormatId, VideoInputId, VideoInputsEvent,
|
||||
VideoPixelFormat,
|
||||
},
|
||||
Cx, CxMediaApi,
|
||||
};
|
||||
use std::{
|
||||
cmp::Reverse,
|
||||
sync::{
|
||||
atomic::{AtomicBool, AtomicU64, Ordering},
|
||||
Arc, Mutex,
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
pub const SEND_MAX_WIDTH: usize = 640;
|
||||
const PREVIEW_MAX_WIDTH: usize = 320;
|
||||
const PREVIEW_INTERVAL: Duration = Duration::from_millis(100);
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct CameraRgbFrame {
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub rgb: Vec<u8>,
|
||||
pub serial: u64,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct PreviewSlot {
|
||||
frame: Option<CameraRgbFrame>,
|
||||
updated_at: Option<Instant>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct CameraMailboxInner {
|
||||
want: AtomicBool,
|
||||
serial: AtomicU64,
|
||||
frame: Mutex<Option<CameraRgbFrame>>,
|
||||
model_size: Mutex<Option<(u32, u32)>>,
|
||||
preview: Mutex<PreviewSlot>,
|
||||
}
|
||||
|
||||
/// A one-frame handoff from the camera callback to the model worker. The
|
||||
/// callback only converts a model-sized frame after the worker asks for one;
|
||||
/// the smaller preview is independent and limited to ten updates per second.
|
||||
#[derive(Clone, Default)]
|
||||
pub struct CameraMailbox {
|
||||
inner: Arc<CameraMailboxInner>,
|
||||
}
|
||||
|
||||
impl CameraMailbox {
|
||||
pub fn request(&self) {
|
||||
if let Ok(mut frame) = self.inner.frame.lock() {
|
||||
*frame = None;
|
||||
}
|
||||
self.inner.want.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
pub fn take(&self) -> Option<CameraRgbFrame> {
|
||||
self.inner.frame.lock().ok()?.take()
|
||||
}
|
||||
|
||||
pub fn peek_preview(&self) -> Option<CameraRgbFrame> {
|
||||
self.inner.preview.lock().ok()?.frame.clone()
|
||||
}
|
||||
|
||||
pub fn model_size(&self) -> Option<(u32, u32)> {
|
||||
*self.inner.model_size.lock().ok()?
|
||||
}
|
||||
|
||||
fn capture(&self, frame: &CameraFrameRef<'_>) {
|
||||
let wanted = self.inner.want.swap(false, Ordering::AcqRel);
|
||||
let preview_due = self
|
||||
.inner
|
||||
.preview
|
||||
.lock()
|
||||
.ok()
|
||||
.map(|slot| {
|
||||
slot.updated_at
|
||||
.map(|updated| updated.elapsed() >= PREVIEW_INTERVAL)
|
||||
.unwrap_or(true)
|
||||
})
|
||||
.unwrap_or(false);
|
||||
if !wanted && !preview_due {
|
||||
return;
|
||||
}
|
||||
|
||||
if wanted {
|
||||
if let Some((rgb, width, height)) = frame_to_rgb(frame) {
|
||||
let serial = self.inner.serial.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
if let Ok(mut size) = self.inner.model_size.lock() {
|
||||
*size = Some((width, height));
|
||||
}
|
||||
if let Ok(mut slot) = self.inner.frame.lock() {
|
||||
*slot = Some(CameraRgbFrame {
|
||||
width,
|
||||
height,
|
||||
rgb,
|
||||
serial,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if preview_due {
|
||||
if let Some((rgb, width, height)) = frame_to_rgb_max(frame, PREVIEW_MAX_WIDTH) {
|
||||
let serial = self.inner.serial.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
if let Ok(mut slot) = self.inner.preview.lock() {
|
||||
slot.frame = Some(CameraRgbFrame {
|
||||
width,
|
||||
height,
|
||||
rgb,
|
||||
serial,
|
||||
});
|
||||
slot.updated_at = Some(Instant::now());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Register the capture callback. Calling this also asks the platform to
|
||||
/// enumerate cameras, which produces `Event::VideoInputs` on the UI thread.
|
||||
pub fn install_camera(cx: &mut Cx, mailbox: CameraMailbox) {
|
||||
cx.camera_frame_input(0, move |frame| mailbox.capture(&frame));
|
||||
}
|
||||
|
||||
/// Choose the first device's smallest raw-YUV format at least 640x360.
|
||||
pub fn pick_camera(event: &VideoInputsEvent) -> Option<(VideoInputId, VideoFormatId)> {
|
||||
let device = event.descs.first()?;
|
||||
let format = device
|
||||
.formats
|
||||
.iter()
|
||||
.filter(|format| {
|
||||
format.width >= 640
|
||||
&& format.height >= 360
|
||||
&& matches!(
|
||||
format.pixel_format,
|
||||
VideoPixelFormat::NV12 | VideoPixelFormat::YUY2
|
||||
)
|
||||
})
|
||||
.min_by_key(|format| {
|
||||
(
|
||||
format.width.saturating_mul(format.height),
|
||||
if format.pixel_format == VideoPixelFormat::NV12 {
|
||||
0
|
||||
} else {
|
||||
1
|
||||
},
|
||||
Reverse((format.frame_rate.unwrap_or(0.0) * 1000.0) as u64),
|
||||
)
|
||||
})?;
|
||||
Some((device.input_id, format.format_id))
|
||||
}
|
||||
|
||||
/// Convert NV12 or YUY2 to packed RGB8, using an integer sampling step so
|
||||
/// the entire frame fits within [`SEND_MAX_WIDTH`].
|
||||
pub fn frame_to_rgb(frame: &CameraFrameRef<'_>) -> Option<(Vec<u8>, u32, u32)> {
|
||||
frame_to_rgb_max(frame, SEND_MAX_WIDTH)
|
||||
}
|
||||
|
||||
fn frame_to_rgb_max(
|
||||
frame: &CameraFrameRef<'_>,
|
||||
max_width: usize,
|
||||
) -> Option<(Vec<u8>, u32, u32)> {
|
||||
let (width, height) = (frame.width, frame.height);
|
||||
if width == 0 || height == 0 || max_width == 0 {
|
||||
return None;
|
||||
}
|
||||
let step = width.div_ceil(max_width).max(1);
|
||||
let out_width = width.div_ceil(step);
|
||||
let out_height = height.div_ceil(step);
|
||||
let mut rgb = Vec::with_capacity(out_width.checked_mul(out_height)?.checked_mul(3)?);
|
||||
|
||||
for out_y in 0..out_height {
|
||||
let source_y = (out_y * step).min(height - 1);
|
||||
for out_x in 0..out_width {
|
||||
let source_x = (out_x * step).min(width - 1);
|
||||
let (y, u, v) = match frame.layout {
|
||||
CameraFrameLayout::NV12 => nv12_pixel(frame, source_x, source_y)?,
|
||||
CameraFrameLayout::YUY2 => yuy2_pixel(frame, source_x, source_y)?,
|
||||
_ => return None,
|
||||
};
|
||||
rgb.extend_from_slice(&yuv_to_rgb(y, u, v));
|
||||
}
|
||||
}
|
||||
|
||||
Some((
|
||||
rgb,
|
||||
u32::try_from(out_width).ok()?,
|
||||
u32::try_from(out_height).ok()?,
|
||||
))
|
||||
}
|
||||
|
||||
fn nv12_pixel(frame: &CameraFrameRef<'_>, x: usize, y: usize) -> Option<(u8, u8, u8)> {
|
||||
if frame.plane_count < 2 {
|
||||
return None;
|
||||
}
|
||||
let y_plane = frame.planes[0];
|
||||
let uv_plane = frame.planes[1];
|
||||
let y_index = y
|
||||
.checked_mul(y_plane.row_stride)?
|
||||
.checked_add(x.checked_mul(y_plane.pixel_stride)?)?;
|
||||
let uv_index = (y / 2)
|
||||
.checked_mul(uv_plane.row_stride)?
|
||||
.checked_add((x / 2).checked_mul(uv_plane.pixel_stride)?)?;
|
||||
Some((
|
||||
*y_plane.bytes.get(y_index)?,
|
||||
*uv_plane.bytes.get(uv_index)?,
|
||||
*uv_plane.bytes.get(uv_index + 1)?,
|
||||
))
|
||||
}
|
||||
|
||||
fn yuy2_pixel(frame: &CameraFrameRef<'_>, x: usize, y: usize) -> Option<(u8, u8, u8)> {
|
||||
if frame.plane_count < 1 {
|
||||
return None;
|
||||
}
|
||||
let plane = frame.planes[0];
|
||||
let pixel_stride = plane.pixel_stride.max(2);
|
||||
let pair = y
|
||||
.checked_mul(plane.row_stride)?
|
||||
.checked_add((x / 2).checked_mul(pixel_stride.checked_mul(2)?)?)?;
|
||||
Some((
|
||||
*plane.bytes.get(pair + (x & 1) * pixel_stride)?,
|
||||
*plane.bytes.get(pair + 1)?,
|
||||
*plane.bytes.get(pair + pixel_stride + 1)?,
|
||||
))
|
||||
}
|
||||
|
||||
/// One BT.709, video-range YUV pixel to RGB8.
|
||||
pub(crate) fn yuv_to_rgb(y: u8, u: u8, v: u8) -> [u8; 3] {
|
||||
let c = i32::from(y) - 16;
|
||||
let d = i32::from(u) - 128;
|
||||
let e = i32::from(v) - 128;
|
||||
let clip = |value: i32| ((value + 128) >> 8).clamp(0, 255) as u8;
|
||||
[
|
||||
clip(298 * c + 459 * e),
|
||||
clip(298 * c - 55 * d - 136 * e),
|
||||
clip(298 * c + 541 * d),
|
||||
]
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use makepad_widgets::makepad_platform::video::{
|
||||
CameraColorMatrix, CameraFramePlaneRef,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn yuv_grey_red_and_blue() {
|
||||
let grey = yuv_to_rgb(126, 128, 128);
|
||||
assert!(grey.iter().all(|channel| (126i16 - i16::from(*channel)).abs() <= 2));
|
||||
|
||||
let red = yuv_to_rgb(81, 90, 240);
|
||||
assert!(red[0] > 220 && red[1] < 60 && red[2] < 60, "{red:?}");
|
||||
|
||||
let blue = yuv_to_rgb(41, 240, 110);
|
||||
assert!(blue[2] > 220 && blue[0] < 60 && blue[1] < 60, "{blue:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nv12_integer_downsample_keeps_the_whole_frame() {
|
||||
const WIDTH: usize = 64;
|
||||
const HEIGHT: usize = 32;
|
||||
let mut y_plane = vec![0u8; WIDTH * HEIGHT];
|
||||
for y in 0..HEIGHT {
|
||||
for x in 0..WIDTH {
|
||||
y_plane[y * WIDTH + x] = 16 + ((x + y) % 200) as u8;
|
||||
}
|
||||
}
|
||||
let uv_plane = vec![128u8; WIDTH * HEIGHT / 2];
|
||||
let frame = CameraFrameRef {
|
||||
timestamp_ns: 0,
|
||||
width: WIDTH,
|
||||
height: HEIGHT,
|
||||
layout: CameraFrameLayout::NV12,
|
||||
matrix: CameraColorMatrix::BT709,
|
||||
plane_count: 2,
|
||||
planes: [
|
||||
CameraFramePlaneRef {
|
||||
bytes: &y_plane,
|
||||
row_stride: WIDTH,
|
||||
pixel_stride: 1,
|
||||
},
|
||||
CameraFramePlaneRef {
|
||||
bytes: &uv_plane,
|
||||
row_stride: WIDTH,
|
||||
pixel_stride: 2,
|
||||
},
|
||||
CameraFramePlaneRef::empty(),
|
||||
],
|
||||
};
|
||||
|
||||
let (rgb, width, height) = frame_to_rgb_max(&frame, 16).unwrap();
|
||||
assert_eq!((width, height), (16, 8));
|
||||
assert_eq!(rgb.len(), 16 * 8 * 3);
|
||||
assert_eq!(&rgb[..3], &yuv_to_rgb(y_plane[0], 128, 128));
|
||||
let last_source = y_plane[28 * WIDTH + 60];
|
||||
assert_eq!(&rgb[rgb.len() - 3..], &yuv_to_rgb(last_source, 128, 128));
|
||||
}
|
||||
}
|
||||
73
apps/fabric/src/install.rs
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
use makepad_ai_hub::{
|
||||
local::{InstallState, LocalModels},
|
||||
registry::LicenseRestriction,
|
||||
};
|
||||
use makepad_ai_hub_ui::{ModelRowInstallState, ModelRowState};
|
||||
|
||||
pub const BODY_MODEL_ID: &str = "sam3dbody";
|
||||
pub const BODY_MODEL_ROLE: &str = "native-body";
|
||||
|
||||
pub fn body_model_row(models: &LocalModels) -> ModelRowState {
|
||||
let spec = models.spec(BODY_MODEL_ID);
|
||||
let bytes_from_spec = spec
|
||||
.map(|spec| spec.files.iter().filter_map(|file| file.size).sum())
|
||||
.unwrap_or(0);
|
||||
let (bytes_done, bytes_total, state) = match models.install_state(BODY_MODEL_ID) {
|
||||
InstallState::NotInstalled { bytes_total } => {
|
||||
(0, bytes_total.max(bytes_from_spec), ModelRowInstallState::NotInstalled)
|
||||
}
|
||||
InstallState::Partial {
|
||||
bytes_done,
|
||||
bytes_total,
|
||||
} => (bytes_done, bytes_total, ModelRowInstallState::NotInstalled),
|
||||
InstallState::Installed => (
|
||||
bytes_from_spec,
|
||||
bytes_from_spec,
|
||||
ModelRowInstallState::Installed,
|
||||
),
|
||||
};
|
||||
let license = spec.and_then(|spec| spec.license.as_ref());
|
||||
ModelRowState {
|
||||
model_id: BODY_MODEL_ID.to_string(),
|
||||
name: "SAM 3D Body".to_string(),
|
||||
bytes_total,
|
||||
bytes_done,
|
||||
state,
|
||||
license_name: license
|
||||
.map(|license| license.name.clone())
|
||||
.unwrap_or_else(|| "Licence unavailable".to_string()),
|
||||
restriction: license
|
||||
.map(|license| restriction_name(license.restriction).to_string())
|
||||
.unwrap_or_else(|| "restricted".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn body_model_status(models: &LocalModels, downloading: bool) -> String {
|
||||
if !models.license_acknowledged(BODY_MODEL_ID) {
|
||||
return "licence not accepted".to_string();
|
||||
}
|
||||
match models.install_state(BODY_MODEL_ID) {
|
||||
InstallState::Installed => "installed · 2.8 GB · Metal".to_string(),
|
||||
InstallState::NotInstalled { .. } => "not installed · 2.8 GB".to_string(),
|
||||
InstallState::Partial {
|
||||
bytes_done,
|
||||
bytes_total,
|
||||
} => {
|
||||
let percent = bytes_done.saturating_mul(100) / bytes_total.max(1);
|
||||
if downloading {
|
||||
format!("downloading {percent} %")
|
||||
} else {
|
||||
format!("not installed · {percent} % downloaded")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn restriction_name(restriction: LicenseRestriction) -> &'static str {
|
||||
match restriction {
|
||||
LicenseRestriction::None => "none",
|
||||
LicenseRestriction::NonCommercial => "non-commercial",
|
||||
LicenseRestriction::Community => "community",
|
||||
LicenseRestriction::Restricted => "restricted",
|
||||
}
|
||||
}
|
||||
1802
apps/fabric/src/main.rs
Normal file
398
apps/fabric/src/pattern_view.rs
Normal file
|
|
@ -0,0 +1,398 @@
|
|||
use crate::body_view::DrawFabricLine;
|
||||
use makepad_fabric_draft::{flatten, nest, offset, Layout as PatternLayout, Part, Pattern, Point};
|
||||
use makepad_widgets::*;
|
||||
|
||||
script_mod! {
|
||||
use mod.prelude.widgets_internal.*
|
||||
use mod.widgets.*
|
||||
|
||||
mod.widgets.FabricPatternViewBase = #(FabricPatternView::register_widget(vm))
|
||||
mod.widgets.FabricPatternView = set_type_default() do mod.widgets.FabricPatternViewBase {
|
||||
width: Fill
|
||||
height: Fill
|
||||
draw_bg +: {color: #x0d1218}
|
||||
draw_line +: {}
|
||||
draw_text +: {
|
||||
color: #xc5d0dc
|
||||
text_style: theme.font_regular{font_size: 8.5}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct PatternDrag {
|
||||
from: DVec2,
|
||||
pan: DVec2,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct Bounds {
|
||||
min: DVec2,
|
||||
max: DVec2,
|
||||
valid: bool,
|
||||
}
|
||||
|
||||
impl Bounds {
|
||||
fn include(&mut self, point: DVec2) {
|
||||
if !self.valid {
|
||||
self.min = point;
|
||||
self.max = point;
|
||||
self.valid = true;
|
||||
} else {
|
||||
self.min.x = self.min.x.min(point.x);
|
||||
self.min.y = self.min.y.min(point.y);
|
||||
self.max.x = self.max.x.max(point.x);
|
||||
self.max.y = self.max.y.max(point.y);
|
||||
}
|
||||
}
|
||||
|
||||
fn size(self) -> DVec2 {
|
||||
let size = self.max - self.min;
|
||||
dvec2(size.x.max(1.0), size.y.max(1.0))
|
||||
}
|
||||
|
||||
fn centre(self) -> DVec2 {
|
||||
(self.min + self.max) * 0.5
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Script, ScriptHook, Widget)]
|
||||
pub struct FabricPatternView {
|
||||
#[uid]
|
||||
uid: WidgetUid,
|
||||
#[source]
|
||||
source: ScriptObjectRef,
|
||||
#[walk]
|
||||
walk: Walk,
|
||||
#[layout]
|
||||
layout: Layout,
|
||||
#[redraw]
|
||||
#[area]
|
||||
area: Area,
|
||||
#[live]
|
||||
draw_bg: DrawColor,
|
||||
#[live]
|
||||
draw_line: DrawFabricLine,
|
||||
#[live]
|
||||
draw_text: DrawText,
|
||||
#[rust]
|
||||
pattern: Option<Pattern>,
|
||||
#[rust]
|
||||
nested: Option<PatternLayout>,
|
||||
#[rust]
|
||||
error: String,
|
||||
#[rust]
|
||||
bounds: Bounds,
|
||||
#[rust(1.0)]
|
||||
zoom: f64,
|
||||
/// The view aspect the current nest was chosen for.
|
||||
#[rust(1.0)]
|
||||
nest_aspect: f64,
|
||||
#[rust]
|
||||
pan: DVec2,
|
||||
#[rust]
|
||||
drag: Option<PatternDrag>,
|
||||
}
|
||||
|
||||
impl FabricPatternView {
|
||||
pub fn set_pattern(&mut self, cx: &mut Cx, pattern: Pattern) {
|
||||
self.pattern = Some(pattern);
|
||||
// Nested on the next draw, for the pane's shape.
|
||||
self.nested = None;
|
||||
self.error.clear();
|
||||
self.zoom = 1.0;
|
||||
self.pan = dvec2(0.0, 0.0);
|
||||
self.redraw(cx);
|
||||
}
|
||||
|
||||
pub fn set_error(&mut self, cx: &mut Cx, error: impl Into<String>) {
|
||||
self.pattern = None;
|
||||
self.nested = None;
|
||||
self.error = error.into();
|
||||
self.redraw(cx);
|
||||
}
|
||||
|
||||
/// Nest onto the fabric width whose finished layout has the pane's
|
||||
/// aspect ratio, so the pieces fill the view instead of a tall strip.
|
||||
fn ensure_nest(&mut self, rect: Rect) {
|
||||
let target = (rect.size.x - 24.0).max(1.0) / (rect.size.y - 24.0).max(1.0);
|
||||
if self.nested.is_some() && (self.nest_aspect / target).ln().abs() < 0.12 {
|
||||
return;
|
||||
}
|
||||
const WIDTHS: [f64; 9] = [
|
||||
900.0, 1200.0, 1500.0, 2000.0, 2500.0, 3000.0, 4000.0, 5000.0, 6500.0,
|
||||
];
|
||||
let best = {
|
||||
let Some(pattern) = &self.pattern else { return };
|
||||
let mut best: Option<(f64, PatternLayout, Bounds)> = None;
|
||||
for width in WIDTHS {
|
||||
let layout = nest(pattern, width);
|
||||
let bounds = pattern_bounds(pattern, &layout);
|
||||
let size = bounds.size();
|
||||
let aspect = size.x.max(1.0) / size.y.max(1.0);
|
||||
let score = (aspect / target).ln().abs();
|
||||
if best.as_ref().map_or(true, |(other, _, _)| score < *other) {
|
||||
best = Some((score, layout, bounds));
|
||||
}
|
||||
}
|
||||
best
|
||||
};
|
||||
if let Some((_, layout, bounds)) = best {
|
||||
self.nested = Some(layout);
|
||||
self.bounds = bounds;
|
||||
self.nest_aspect = target;
|
||||
}
|
||||
}
|
||||
|
||||
fn to_screen(&self, point: DVec2, rect: Rect) -> DVec2 {
|
||||
let size = self.bounds.size();
|
||||
let fit = ((rect.size.x - 24.0) / size.x)
|
||||
.min((rect.size.y - 24.0) / size.y)
|
||||
.max(0.0001);
|
||||
rect.pos + rect.size * 0.5
|
||||
+ self.pan
|
||||
+ (point - self.bounds.centre()) * fit * self.zoom
|
||||
}
|
||||
|
||||
fn part_offset<'a>(
|
||||
&'a self,
|
||||
layout: &'a PatternLayout,
|
||||
part_index: usize,
|
||||
) -> (Point, f64) {
|
||||
layout
|
||||
.placements
|
||||
.iter()
|
||||
.find(|placement| placement.part == part_index)
|
||||
.map(|placement| (placement.offset, placement.rotation_deg))
|
||||
.unwrap_or((Point::default(), 0.0))
|
||||
}
|
||||
|
||||
fn path_segments(
|
||||
&self,
|
||||
path: &makepad_fabric_draft::Path,
|
||||
offset_by: Point,
|
||||
rotation: f64,
|
||||
rect: Rect,
|
||||
) -> Vec<(DVec2, DVec2)> {
|
||||
let points = flatten(path, 1.0);
|
||||
let transformed: Vec<DVec2> = points
|
||||
.iter()
|
||||
.map(|point| self.to_screen(place(*point, offset_by, rotation), rect))
|
||||
.collect();
|
||||
let mut segments: Vec<_> = transformed
|
||||
.windows(2)
|
||||
.map(|pair| (pair[0], pair[1]))
|
||||
.collect();
|
||||
if path.closed {
|
||||
if let (Some(first), Some(last)) = (transformed.first(), transformed.last()) {
|
||||
segments.push((*last, *first));
|
||||
}
|
||||
}
|
||||
segments
|
||||
}
|
||||
}
|
||||
|
||||
impl Widget for FabricPatternView {
|
||||
fn draw_walk(&mut self, cx: &mut Cx2d, _scope: &mut Scope, walk: Walk) -> DrawStep {
|
||||
let rect = cx.walk_turtle(walk);
|
||||
self.draw_bg.draw_abs(cx, rect);
|
||||
cx.push_clip_rect(rect);
|
||||
self.ensure_nest(rect);
|
||||
let (Some(pattern), Some(layout)) = (&self.pattern, &self.nested) else {
|
||||
let message = if self.error.is_empty() {
|
||||
"pattern preview"
|
||||
} else {
|
||||
&self.error
|
||||
};
|
||||
self.draw_text.color = Vec4f {
|
||||
x: 0.52,
|
||||
y: 0.58,
|
||||
z: 0.65,
|
||||
w: 1.0,
|
||||
};
|
||||
self.draw_text
|
||||
.draw_abs(cx, rect.pos + dvec2(14.0, 16.0), message);
|
||||
cx.pop_clip_rect();
|
||||
cx.add_aligned_rect_area(&mut self.area, rect);
|
||||
return DrawStep::done();
|
||||
};
|
||||
|
||||
struct Stroke {
|
||||
from: DVec2,
|
||||
to: DVec2,
|
||||
color: Vec4f,
|
||||
width: f64,
|
||||
}
|
||||
let mut strokes = Vec::new();
|
||||
let mut labels = Vec::new();
|
||||
let cut_color = Vec4f {
|
||||
x: 0.93,
|
||||
y: 0.96,
|
||||
z: 0.99,
|
||||
w: 1.0,
|
||||
};
|
||||
let seam_color = Vec4f {
|
||||
x: 0.40,
|
||||
y: 0.49,
|
||||
z: 0.58,
|
||||
w: 0.72,
|
||||
};
|
||||
let mark_color = Vec4f {
|
||||
x: 1.0,
|
||||
y: 0.40,
|
||||
z: 0.18,
|
||||
w: 0.95,
|
||||
};
|
||||
|
||||
for (part_index, part) in pattern.parts.iter().enumerate() {
|
||||
let (part_offset, rotation) = self.part_offset(layout, part_index);
|
||||
let cut_path = offset(&part.outline, part.seam_allowance_mm);
|
||||
for (from, to) in self.path_segments(&cut_path, part_offset, rotation, rect) {
|
||||
strokes.push(Stroke {
|
||||
from,
|
||||
to,
|
||||
color: cut_color,
|
||||
width: 1.25,
|
||||
});
|
||||
}
|
||||
for (from, to) in self.path_segments(&part.outline, part_offset, rotation, rect) {
|
||||
strokes.push(Stroke {
|
||||
from,
|
||||
to,
|
||||
color: seam_color,
|
||||
width: 0.75,
|
||||
});
|
||||
}
|
||||
for path in &part.internal {
|
||||
for (from, to) in self.path_segments(path, part_offset, rotation, rect) {
|
||||
strokes.push(Stroke {
|
||||
from,
|
||||
to,
|
||||
color: seam_color,
|
||||
width: 0.75,
|
||||
});
|
||||
}
|
||||
}
|
||||
for notch in &part.notches {
|
||||
let at = self.to_screen(place(*notch, part_offset, rotation), rect);
|
||||
strokes.push(Stroke {
|
||||
from: at + dvec2(-3.0, -3.0),
|
||||
to: at + dvec2(3.0, 3.0),
|
||||
color: mark_color,
|
||||
width: 1.0,
|
||||
});
|
||||
strokes.push(Stroke {
|
||||
from: at + dvec2(-3.0, 3.0),
|
||||
to: at + dvec2(3.0, -3.0),
|
||||
color: mark_color,
|
||||
width: 1.0,
|
||||
});
|
||||
}
|
||||
let grain_from = self.to_screen(place(part.grainline.0, part_offset, rotation), rect);
|
||||
let grain_to = self.to_screen(place(part.grainline.1, part_offset, rotation), rect);
|
||||
strokes.push(Stroke {
|
||||
from: grain_from,
|
||||
to: grain_to,
|
||||
color: mark_color,
|
||||
width: 1.0,
|
||||
});
|
||||
let label_at = part
|
||||
.labels
|
||||
.first()
|
||||
.map(|label| label.at)
|
||||
.unwrap_or(part.outline.start);
|
||||
labels.push((
|
||||
self.to_screen(place(label_at, part_offset, rotation), rect),
|
||||
format!(
|
||||
"{} · {}",
|
||||
part.name,
|
||||
if part.on_fold {
|
||||
"cut on fold".to_string()
|
||||
} else {
|
||||
format!("cut {}", part.cut_count)
|
||||
}
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
self.draw_line.begin_many_instances(cx);
|
||||
for stroke in strokes {
|
||||
self.draw_line.color = stroke.color;
|
||||
self.draw_line
|
||||
.segment(cx, stroke.from, stroke.to, stroke.width);
|
||||
}
|
||||
self.draw_line.end_many_instances(cx);
|
||||
self.draw_text.color = Vec4f {
|
||||
x: 0.78,
|
||||
y: 0.84,
|
||||
z: 0.90,
|
||||
w: 1.0,
|
||||
};
|
||||
for (at, text) in labels {
|
||||
self.draw_text.draw_abs(cx, at + dvec2(4.0, -5.0), &text);
|
||||
}
|
||||
cx.pop_clip_rect();
|
||||
cx.add_aligned_rect_area(&mut self.area, rect);
|
||||
DrawStep::done()
|
||||
}
|
||||
|
||||
fn handle_event(&mut self, cx: &mut Cx, event: &Event, _scope: &mut Scope) {
|
||||
match event.hits(cx, self.area) {
|
||||
Hit::FingerDown(event) if event.device.is_primary_hit() => {
|
||||
self.drag = Some(PatternDrag {
|
||||
from: event.abs,
|
||||
pan: self.pan,
|
||||
});
|
||||
}
|
||||
Hit::FingerMove(event) => {
|
||||
if let Some(drag) = self.drag {
|
||||
self.pan = drag.pan + event.abs - drag.from;
|
||||
self.redraw(cx);
|
||||
}
|
||||
}
|
||||
Hit::FingerUp(_) => self.drag = None,
|
||||
Hit::FingerScroll(event) => {
|
||||
self.zoom = (self.zoom * (-event.scroll.y * 0.004).exp()).clamp(0.1, 30.0);
|
||||
self.redraw(cx);
|
||||
}
|
||||
Hit::FingerHoverIn(_) | Hit::FingerHoverOver(_) => {
|
||||
cx.set_cursor(MouseCursor::Grab)
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn place(point: Point, offset: Point, rotation_deg: f64) -> DVec2 {
|
||||
let angle = rotation_deg.to_radians();
|
||||
let (sin, cos) = angle.sin_cos();
|
||||
dvec2(
|
||||
point.x * cos - point.y * sin + offset.x,
|
||||
point.x * sin + point.y * cos + offset.y,
|
||||
)
|
||||
}
|
||||
|
||||
fn pattern_bounds(pattern: &Pattern, layout: &PatternLayout) -> Bounds {
|
||||
let mut bounds = Bounds::default();
|
||||
for (part_index, part) in pattern.parts.iter().enumerate() {
|
||||
let (part_offset, rotation) = layout
|
||||
.placements
|
||||
.iter()
|
||||
.find(|placement| placement.part == part_index)
|
||||
.map(|placement| (placement.offset, placement.rotation_deg))
|
||||
.unwrap_or((Point::default(), 0.0));
|
||||
include_part(&mut bounds, part, part_offset, rotation);
|
||||
}
|
||||
if !bounds.valid && layout.width_mm > 0.0 && layout.height_mm > 0.0 {
|
||||
bounds.include(dvec2(0.0, 0.0));
|
||||
bounds.include(dvec2(layout.width_mm, layout.height_mm));
|
||||
}
|
||||
bounds
|
||||
}
|
||||
|
||||
fn include_part(bounds: &mut Bounds, part: &Part, part_offset: Point, rotation: f64) {
|
||||
let cut = offset(&part.outline, part.seam_allowance_mm);
|
||||
for point in flatten(&cut, 1.0) {
|
||||
bounds.include(place(point, part_offset, rotation));
|
||||
}
|
||||
}
|
||||
676
apps/fabric/src/pipeline.rs
Normal file
|
|
@ -0,0 +1,676 @@
|
|||
use crate::{
|
||||
body_view::{map_measurements_to_vertices, BodyPoseMapping},
|
||||
camera::CameraMailbox,
|
||||
};
|
||||
use makepad_ai_body::model::BodyModel;
|
||||
use makepad_fabric_measure::{measure, BodyMesh, MeasureOptions, Measured};
|
||||
use makepad_widgets::image_cache::ImageBuffer;
|
||||
use makepad_widgets::makepad_platform::thread::SignalToUI;
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
path::{Path, PathBuf},
|
||||
sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
mpsc::{self, Receiver, Sender},
|
||||
Arc,
|
||||
},
|
||||
thread,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
const PHOTO_CROP_SIZE: usize = 512;
|
||||
const LIVE_CROP_SIZE: usize = 384;
|
||||
const SHAPE_ALPHA: f32 = 0.35;
|
||||
const POSE_ALPHA: f32 = 0.6;
|
||||
const SHAPE_RESET_GAP: Duration = Duration::from_secs(1);
|
||||
const FRAME_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
const FRAME_POLL: Duration = Duration::from_millis(5);
|
||||
|
||||
pub enum PipelineMessage {
|
||||
Stage(String),
|
||||
LiveFrame {
|
||||
fps: f32,
|
||||
model_ms: f32,
|
||||
pose_ms: f32,
|
||||
person: bool,
|
||||
bbox: Option<[f32; 4]>,
|
||||
},
|
||||
Done {
|
||||
measured: Box<Measured>,
|
||||
mesh: Arc<BodyMesh>,
|
||||
posed: Option<Arc<Vec<[f32; 3]>>>,
|
||||
pose_mapping: BodyPoseMapping,
|
||||
reset_pose: bool,
|
||||
},
|
||||
Failed(String),
|
||||
}
|
||||
|
||||
struct RunRequest {
|
||||
photo: PathBuf,
|
||||
weights: PathBuf,
|
||||
height_cm: Option<f32>,
|
||||
}
|
||||
|
||||
struct LiveRequest {
|
||||
weights: PathBuf,
|
||||
height_cm: Option<f32>,
|
||||
mailbox: CameraMailbox,
|
||||
}
|
||||
|
||||
enum WorkerRequest {
|
||||
Photo(RunRequest),
|
||||
Live(LiveRequest),
|
||||
}
|
||||
|
||||
pub struct Pipeline {
|
||||
request_tx: Sender<WorkerRequest>,
|
||||
message_rx: Receiver<PipelineMessage>,
|
||||
live_stop: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl Pipeline {
|
||||
pub fn new() -> Self {
|
||||
let (request_tx, request_rx) = mpsc::channel();
|
||||
let (message_tx, message_rx) = mpsc::channel();
|
||||
let live_stop = Arc::new(AtomicBool::new(true));
|
||||
let worker_stop = live_stop.clone();
|
||||
thread::Builder::new()
|
||||
.name("fabric-body-pipeline".to_string())
|
||||
.spawn(move || worker(request_rx, message_tx, worker_stop))
|
||||
.expect("spawn fabric body worker");
|
||||
Self {
|
||||
request_tx,
|
||||
message_rx,
|
||||
live_stop,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn run(
|
||||
&self,
|
||||
photo: PathBuf,
|
||||
weights: PathBuf,
|
||||
height_cm: Option<f32>,
|
||||
) -> Result<(), String> {
|
||||
self.request_tx
|
||||
.send(WorkerRequest::Photo(RunRequest {
|
||||
photo,
|
||||
weights,
|
||||
height_cm,
|
||||
}))
|
||||
.map_err(|_| "the body model worker stopped".to_string())
|
||||
}
|
||||
|
||||
pub fn start_live(
|
||||
&self,
|
||||
weights: PathBuf,
|
||||
height_cm: Option<f32>,
|
||||
mailbox: CameraMailbox,
|
||||
) -> Result<(), String> {
|
||||
self.live_stop.store(false, Ordering::Release);
|
||||
if self
|
||||
.request_tx
|
||||
.send(WorkerRequest::Live(LiveRequest {
|
||||
weights,
|
||||
height_cm,
|
||||
mailbox,
|
||||
}))
|
||||
.is_err()
|
||||
{
|
||||
self.live_stop.store(true, Ordering::Release);
|
||||
return Err("the body model worker stopped".to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn stop_live(&self) {
|
||||
self.live_stop.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
pub fn poll(&self) -> Vec<PipelineMessage> {
|
||||
self.message_rx.try_iter().collect()
|
||||
}
|
||||
}
|
||||
|
||||
fn emit(sender: &Sender<PipelineMessage>, message: PipelineMessage) -> bool {
|
||||
if sender.send(message).is_err() {
|
||||
return false;
|
||||
}
|
||||
SignalToUI::set_ui_signal();
|
||||
true
|
||||
}
|
||||
|
||||
fn worker(
|
||||
requests: Receiver<WorkerRequest>,
|
||||
messages: Sender<PipelineMessage>,
|
||||
live_stop: Arc<AtomicBool>,
|
||||
) {
|
||||
let mut loaded: Option<(PathBuf, BodyModel)> = None;
|
||||
while let Ok(request) = requests.recv() {
|
||||
let result = match request {
|
||||
WorkerRequest::Photo(request) => run_one(&request, &messages, &mut loaded),
|
||||
WorkerRequest::Live(request) => {
|
||||
run_live(&request, &messages, &mut loaded, &live_stop)
|
||||
}
|
||||
};
|
||||
if let Err(error) = result {
|
||||
if !emit(&messages, PipelineMessage::Failed(error)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn run_one(
|
||||
request: &RunRequest,
|
||||
messages: &Sender<PipelineMessage>,
|
||||
loaded: &mut Option<(PathBuf, BodyModel)>,
|
||||
) -> Result<(), String> {
|
||||
emit(messages, PipelineMessage::Stage("decoding photo…".to_string()));
|
||||
let (rgb, width, height) = decode_rgb(&request.photo)?;
|
||||
|
||||
ensure_model(&request.weights, messages, loaded)?;
|
||||
|
||||
emit(messages, PipelineMessage::Stage("inferring…".to_string()));
|
||||
let model = &mut loaded.as_mut().expect("model was loaded above").1;
|
||||
model
|
||||
.set_crop_size(PHOTO_CROP_SIZE)
|
||||
.map_err(|error| format!("could not configure the body model: {error}"))?;
|
||||
let packet = model
|
||||
.infer(&rgb, width, height, None)
|
||||
.map_err(|error| format!("body inference failed: {error}"))?;
|
||||
let person = packet
|
||||
.people
|
||||
.first()
|
||||
.ok_or_else(|| "no person found".to_string())?;
|
||||
let posed = Arc::new(posed_vertices(
|
||||
model,
|
||||
&person.shape,
|
||||
&person.expr,
|
||||
person.mhr,
|
||||
person.global_rot,
|
||||
)?);
|
||||
let vertices = model.rig().rest_vertices(&person.shape, &person.expr);
|
||||
let face_indices = model
|
||||
.weights
|
||||
.i64_shaped("head_pose.faces", &[36_874, 3])
|
||||
.map_err(|error| format!("could not read the body mesh faces: {error}"))?;
|
||||
let mesh = Arc::new(body_mesh_from_flat(&vertices, &face_indices)?);
|
||||
|
||||
emit(messages, PipelineMessage::Stage("measuring…".to_string()));
|
||||
let measured = measure(
|
||||
&mesh,
|
||||
&MeasureOptions {
|
||||
height_cm: request.height_cm,
|
||||
},
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
let pose_mapping = map_measurements_to_vertices(&mesh, &measured);
|
||||
emit(
|
||||
messages,
|
||||
PipelineMessage::Done {
|
||||
measured: Box::new(measured),
|
||||
mesh,
|
||||
posed: Some(posed),
|
||||
pose_mapping,
|
||||
reset_pose: true,
|
||||
},
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_model(
|
||||
weights: &Path,
|
||||
messages: &Sender<PipelineMessage>,
|
||||
loaded: &mut Option<(PathBuf, BodyModel)>,
|
||||
) -> Result<(), String> {
|
||||
if loaded
|
||||
.as_ref()
|
||||
.map(|(path, _)| path != weights)
|
||||
.unwrap_or(true)
|
||||
{
|
||||
emit(
|
||||
messages,
|
||||
PipelineMessage::Stage("loading model 2.8 GB…".to_string()),
|
||||
);
|
||||
let model = BodyModel::load(weights)
|
||||
.map_err(|error| format!("could not load the body model: {error}"))?;
|
||||
*loaded = Some((weights.to_path_buf(), model));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run_live(
|
||||
request: &LiveRequest,
|
||||
messages: &Sender<PipelineMessage>,
|
||||
loaded: &mut Option<(PathBuf, BodyModel)>,
|
||||
stop: &AtomicBool,
|
||||
) -> Result<(), String> {
|
||||
if stop.load(Ordering::Acquire) {
|
||||
return Ok(());
|
||||
}
|
||||
ensure_model(&request.weights, messages, loaded)?;
|
||||
let model = &mut loaded.as_mut().expect("model was loaded above").1;
|
||||
model
|
||||
.set_crop_size(LIVE_CROP_SIZE)
|
||||
.map_err(|error| format!("could not configure live body inference: {error}"))?;
|
||||
let face_indices = model
|
||||
.weights
|
||||
.i64_shaped("head_pose.faces", &[36_874, 3])
|
||||
.map_err(|error| format!("could not read the body mesh faces: {error}"))?;
|
||||
|
||||
let started = Instant::now();
|
||||
let mut previous_bbox = None;
|
||||
let mut smoother = ShapeSmoother::default();
|
||||
let mut pose_smoother = PoseSmoother::default();
|
||||
let mut fps = FpsCounter::default();
|
||||
let mut reset_pose = true;
|
||||
|
||||
while !stop.load(Ordering::Acquire) {
|
||||
request.mailbox.request();
|
||||
let wait_started = Instant::now();
|
||||
let frame = loop {
|
||||
if stop.load(Ordering::Acquire) {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(frame) = request.mailbox.take() {
|
||||
break Some(frame);
|
||||
}
|
||||
if wait_started.elapsed() >= FRAME_TIMEOUT {
|
||||
break None;
|
||||
}
|
||||
thread::sleep(FRAME_POLL);
|
||||
};
|
||||
let Some(frame) = frame else {
|
||||
if !emit(
|
||||
messages,
|
||||
PipelineMessage::Stage("no camera frames".to_string()),
|
||||
) {
|
||||
return Ok(());
|
||||
}
|
||||
continue;
|
||||
};
|
||||
|
||||
let model_started = Instant::now();
|
||||
let packet = model
|
||||
.infer(&frame.rgb, frame.width, frame.height, previous_bbox)
|
||||
.map_err(|error| format!("live body inference failed: {error}"))?;
|
||||
let model_ms = model_started.elapsed().as_secs_f32() * 1000.0;
|
||||
let now = started.elapsed();
|
||||
let person = packet.people.first();
|
||||
let bbox = person.map(|person| person.bbox);
|
||||
let mut pose_ms = 0.0;
|
||||
let mut posed = None;
|
||||
let smoothed_shape = if let Some(person) = person {
|
||||
previous_bbox = Some(expand_bbox(
|
||||
person.bbox,
|
||||
frame.width,
|
||||
frame.height,
|
||||
0.15,
|
||||
));
|
||||
let (mhr, global_rot) = pose_smoother.observe(person.mhr, person.global_rot);
|
||||
let pose_started = Instant::now();
|
||||
posed = Some(Arc::new(posed_vertices(
|
||||
model,
|
||||
&person.shape,
|
||||
&person.expr,
|
||||
mhr,
|
||||
global_rot,
|
||||
)?));
|
||||
pose_ms = pose_started.elapsed().as_secs_f32() * 1000.0;
|
||||
Some(smoother.observe_person(person.shape, now))
|
||||
} else {
|
||||
previous_bbox = None;
|
||||
smoother.observe_miss(now);
|
||||
pose_smoother.reset();
|
||||
None
|
||||
};
|
||||
let current_fps = fps.tick(now);
|
||||
if !emit(
|
||||
messages,
|
||||
PipelineMessage::LiveFrame {
|
||||
fps: current_fps,
|
||||
model_ms,
|
||||
pose_ms,
|
||||
person: person.is_some(),
|
||||
bbox,
|
||||
},
|
||||
) {
|
||||
return Ok(());
|
||||
}
|
||||
if stop.load(Ordering::Acquire) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let Some(shape) = smoothed_shape else {
|
||||
continue;
|
||||
};
|
||||
let expression = [0.0f32; 72];
|
||||
let vertices = model.rig().rest_vertices(&shape, &expression);
|
||||
let mesh = Arc::new(body_mesh_from_flat(&vertices, &face_indices)?);
|
||||
let measured = measure(
|
||||
&mesh,
|
||||
&MeasureOptions {
|
||||
height_cm: request.height_cm,
|
||||
},
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
let pose_mapping = map_measurements_to_vertices(&mesh, &measured);
|
||||
if !emit(
|
||||
messages,
|
||||
PipelineMessage::Done {
|
||||
measured: Box::new(measured),
|
||||
mesh,
|
||||
posed,
|
||||
pose_mapping,
|
||||
reset_pose,
|
||||
},
|
||||
) {
|
||||
return Ok(());
|
||||
}
|
||||
reset_pose = false;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct PoseSmoother {
|
||||
mhr: Option<[f32; 204]>,
|
||||
global_rot: Option<[f32; 3]>,
|
||||
}
|
||||
|
||||
impl PoseSmoother {
|
||||
fn observe(
|
||||
&mut self,
|
||||
mhr: [f32; 204],
|
||||
global_rot: [f32; 3],
|
||||
) -> ([f32; 204], [f32; 3]) {
|
||||
let global_rot = match self.global_rot {
|
||||
Some(previous) => std::array::from_fn(|index| {
|
||||
previous[index] + POSE_ALPHA * (global_rot[index] - previous[index])
|
||||
}),
|
||||
None => global_rot,
|
||||
};
|
||||
let mut mhr = match self.mhr {
|
||||
Some(previous) => std::array::from_fn(|index| {
|
||||
previous[index] + POSE_ALPHA * (mhr[index] - previous[index])
|
||||
}),
|
||||
None => mhr,
|
||||
};
|
||||
// The packet's 204 values are exactly the rig's [pose 136 | scales 68]
|
||||
// input. Global rotation is pose slots 3..6; keep the separately
|
||||
// smoothed copy authoritative before MhrRig::forward pads to 249.
|
||||
mhr[3..6].copy_from_slice(&global_rot);
|
||||
self.mhr = Some(mhr);
|
||||
self.global_rot = Some(global_rot);
|
||||
(mhr, global_rot)
|
||||
}
|
||||
|
||||
fn reset(&mut self) {
|
||||
self.mhr = None;
|
||||
self.global_rot = None;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ShapeSmoother {
|
||||
shape: Option<[f32; 45]>,
|
||||
last_person: Option<Duration>,
|
||||
}
|
||||
|
||||
impl ShapeSmoother {
|
||||
fn observe_person(&mut self, shape: [f32; 45], now: Duration) -> [f32; 45] {
|
||||
if self
|
||||
.last_person
|
||||
.is_some_and(|last| now.saturating_sub(last) > SHAPE_RESET_GAP)
|
||||
{
|
||||
self.shape = None;
|
||||
}
|
||||
self.last_person = Some(now);
|
||||
let smoothed = match self.shape {
|
||||
Some(previous) => std::array::from_fn(|index| {
|
||||
previous[index] + SHAPE_ALPHA * (shape[index] - previous[index])
|
||||
}),
|
||||
None => shape,
|
||||
};
|
||||
self.shape = Some(smoothed);
|
||||
smoothed
|
||||
}
|
||||
|
||||
fn observe_miss(&mut self, now: Duration) {
|
||||
if self
|
||||
.last_person
|
||||
.is_some_and(|last| now.saturating_sub(last) > SHAPE_RESET_GAP)
|
||||
{
|
||||
self.shape = None;
|
||||
self.last_person = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FpsCounter {
|
||||
samples: VecDeque<Duration>,
|
||||
}
|
||||
|
||||
impl FpsCounter {
|
||||
fn tick(&mut self, now: Duration) -> f32 {
|
||||
self.samples.push_back(now);
|
||||
while self.samples.len() > 10 {
|
||||
self.samples.pop_front();
|
||||
}
|
||||
let Some(first) = self.samples.front().copied() else {
|
||||
return 0.0;
|
||||
};
|
||||
let seconds = now.saturating_sub(first).as_secs_f32();
|
||||
if seconds <= f32::EPSILON {
|
||||
0.0
|
||||
} else {
|
||||
(self.samples.len().saturating_sub(1)) as f32 / seconds
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn expand_bbox(
|
||||
bbox: [f32; 4],
|
||||
width: u32,
|
||||
height: u32,
|
||||
amount: f32,
|
||||
) -> [f32; 4] {
|
||||
let box_width = (bbox[2] - bbox[0]).max(0.0);
|
||||
let box_height = (bbox[3] - bbox[1]).max(0.0);
|
||||
[
|
||||
(bbox[0] - box_width * amount).clamp(0.0, width as f32),
|
||||
(bbox[1] - box_height * amount).clamp(0.0, height as f32),
|
||||
(bbox[2] + box_width * amount).clamp(0.0, width as f32),
|
||||
(bbox[3] + box_height * amount).clamp(0.0, height as f32),
|
||||
]
|
||||
}
|
||||
|
||||
fn posed_vertices(
|
||||
model: &BodyModel,
|
||||
shape: &[f32; 45],
|
||||
expression: &[f32; 72],
|
||||
mut mhr: [f32; 204],
|
||||
global_rot: [f32; 3],
|
||||
) -> Result<Vec<[f32; 3]>, String> {
|
||||
// BodyPerson::mhr is already model_params(): pose 0..136 followed by
|
||||
// scales 136..204. The root translation remains zero and the model's
|
||||
// global rotation occupies 3..6. forward() pads the 45 identity slots
|
||||
// to the rig's 249-wide internal vector, then returns rig-space cm.
|
||||
mhr[3..6].copy_from_slice(&global_rot);
|
||||
let output = model.rig().forward(shape, &mhr, expression, true);
|
||||
vertices_from_flat(&output.verts)
|
||||
}
|
||||
|
||||
fn decode_rgb(path: &Path) -> Result<(Vec<u8>, u32, u32), String> {
|
||||
let bytes = std::fs::read(path)
|
||||
.map_err(|error| format!("could not read {}: {error}", path.display()))?;
|
||||
let extension = path
|
||||
.extension()
|
||||
.and_then(|extension| extension.to_str())
|
||||
.unwrap_or_default()
|
||||
.to_ascii_lowercase();
|
||||
let image = match extension.as_str() {
|
||||
"jpg" | "jpeg" => ImageBuffer::from_jpg(&bytes),
|
||||
"png" => ImageBuffer::from_png(&bytes),
|
||||
_ => return Err("choose a JPG or PNG photo".to_string()),
|
||||
}
|
||||
.map_err(|error| format!("could not decode {}: {error}", path.display()))?;
|
||||
let pixel_count = image
|
||||
.width
|
||||
.checked_mul(image.height)
|
||||
.ok_or_else(|| "photo dimensions are too large".to_string())?;
|
||||
if image.data.len() < pixel_count {
|
||||
return Err("decoded photo has too few pixels".to_string());
|
||||
}
|
||||
let width = u32::try_from(image.width).map_err(|_| "photo is too wide".to_string())?;
|
||||
let height = u32::try_from(image.height).map_err(|_| "photo is too tall".to_string())?;
|
||||
Ok((argb_to_rgb(&image.data[..pixel_count]), width, height))
|
||||
}
|
||||
|
||||
pub(crate) fn argb_to_rgb(pixels: &[u32]) -> Vec<u8> {
|
||||
let mut rgb = Vec::with_capacity(pixels.len() * 3);
|
||||
for pixel in pixels {
|
||||
rgb.push((pixel >> 16) as u8);
|
||||
rgb.push((pixel >> 8) as u8);
|
||||
rgb.push(*pixel as u8);
|
||||
}
|
||||
rgb
|
||||
}
|
||||
|
||||
pub(crate) fn faces_i64_to_u32(
|
||||
values: &[i64],
|
||||
vertex_count: usize,
|
||||
) -> Result<Vec<[u32; 3]>, String> {
|
||||
if values.len() % 3 != 0 {
|
||||
return Err("body face index buffer is not made of triangles".to_string());
|
||||
}
|
||||
values
|
||||
.chunks_exact(3)
|
||||
.enumerate()
|
||||
.map(|(face_index, triangle)| {
|
||||
let mut face = [0; 3];
|
||||
for corner in 0..3 {
|
||||
let index = usize::try_from(triangle[corner]).map_err(|_| {
|
||||
format!("body face {face_index} contains a negative vertex index")
|
||||
})?;
|
||||
if index >= vertex_count {
|
||||
return Err(format!(
|
||||
"body face {face_index} references vertex {index}, but there are {vertex_count} vertices"
|
||||
));
|
||||
}
|
||||
face[corner] = u32::try_from(index)
|
||||
.map_err(|_| format!("body vertex index {index} exceeds u32"))?;
|
||||
}
|
||||
Ok(face)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn body_mesh_from_flat(
|
||||
vertices: &[f32],
|
||||
face_indices: &[i64],
|
||||
) -> Result<BodyMesh, String> {
|
||||
let vertices = vertices_from_flat(vertices)?;
|
||||
let faces = faces_i64_to_u32(face_indices, vertices.len())?;
|
||||
Ok(BodyMesh {
|
||||
vertices,
|
||||
faces,
|
||||
landmarks: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn vertices_from_flat(vertices: &[f32]) -> Result<Vec<[f32; 3]>, String> {
|
||||
if vertices.is_empty() || vertices.len() % 3 != 0 {
|
||||
return Err("body vertex buffer is empty or malformed".to_string());
|
||||
}
|
||||
Ok(vertices
|
||||
.chunks_exact(3)
|
||||
.map(|point| [point[0], point[1], point[2]])
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn converts_aarrggbb_to_rgb() {
|
||||
assert_eq!(
|
||||
argb_to_rgb(&[0xff_12_34_56, 0x00_ab_cd_ef]),
|
||||
vec![0x12, 0x34, 0x56, 0xab, 0xcd, 0xef]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_a_tiny_checked_mesh() {
|
||||
let mesh = body_mesh_from_flat(
|
||||
&[0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0],
|
||||
&[0, 1, 2],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(mesh.vertices.len(), 3);
|
||||
assert_eq!(mesh.faces, vec![[0, 1, 2]]);
|
||||
assert!(faces_i64_to_u32(&[0, 1, 3], 3).is_err());
|
||||
assert!(faces_i64_to_u32(&[-1, 1, 2], 3).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn live_shape_ema_converges_and_resets_after_a_gap() {
|
||||
let mut smoother = ShapeSmoother::default();
|
||||
assert_eq!(
|
||||
smoother.observe_person([0.0; 45], Duration::ZERO),
|
||||
[0.0; 45]
|
||||
);
|
||||
let first = smoother.observe_person([10.0; 45], Duration::from_millis(250));
|
||||
assert!((first[0] - 3.5).abs() < 0.0001);
|
||||
let second = smoother.observe_person([10.0; 45], Duration::from_millis(500));
|
||||
assert!((second[0] - 5.775).abs() < 0.0001);
|
||||
|
||||
smoother.observe_miss(Duration::from_millis(1_501));
|
||||
let reset = smoother.observe_person([8.0; 45], Duration::from_millis(1_750));
|
||||
assert_eq!(reset, [8.0; 45]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn live_pose_ema_smooths_parameters_and_keeps_global_slots_in_sync() {
|
||||
let mut smoother = PoseSmoother::default();
|
||||
let mut initial_mhr = [0.0; 204];
|
||||
initial_mhr[20] = 2.0;
|
||||
let (initial, initial_rot) = smoother.observe(initial_mhr, [1.0, 2.0, 3.0]);
|
||||
assert_eq!(initial[3..6], initial_rot);
|
||||
|
||||
let mut next_mhr = [10.0; 204];
|
||||
next_mhr[20] = 12.0;
|
||||
let (smoothed, smoothed_rot) = smoother.observe(next_mhr, [3.0, 4.0, 5.0]);
|
||||
assert!((smoothed[20] - 8.0).abs() < 0.0001);
|
||||
assert_eq!(smoothed_rot, [2.2, 3.2, 4.2]);
|
||||
assert_eq!(smoothed[3..6], smoothed_rot);
|
||||
|
||||
smoother.reset();
|
||||
let (reset, _) = smoother.observe([7.0; 204], [0.5, 1.0, 1.5]);
|
||||
assert_eq!(reset[20], 7.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bbox_expansion_is_fifteen_percent_and_clamped() {
|
||||
let expanded = expand_bbox([100.0, 50.0, 300.0, 250.0], 640, 360, 0.15);
|
||||
for (actual, expected) in expanded.into_iter().zip([70.0, 20.0, 330.0, 280.0]) {
|
||||
assert!((actual - expected).abs() < 0.0001, "{expanded:?}");
|
||||
}
|
||||
assert_eq!(
|
||||
expand_bbox([5.0, 10.0, 635.0, 350.0], 640, 360, 0.15),
|
||||
[0.0, 0.0, 640.0, 360.0]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fps_counter_tracks_recent_frame_cadence() {
|
||||
let mut counter = FpsCounter::default();
|
||||
assert_eq!(counter.tick(Duration::ZERO), 0.0);
|
||||
for quarter in 1..=4 {
|
||||
let fps = counter.tick(Duration::from_millis(quarter * 250));
|
||||
assert!((fps - 4.0).abs() < 0.0001, "{fps}");
|
||||
}
|
||||
}
|
||||
}
|
||||
25
apps/files/Cargo.toml
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
[package]
|
||||
name = "makepad-files"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[[bin]]
|
||||
name = "files"
|
||||
path = "src/main.rs"
|
||||
|
||||
[features]
|
||||
default = ["chat"]
|
||||
chat = ["dep:makepad-ai-hub"]
|
||||
demo = []
|
||||
|
||||
[dependencies]
|
||||
makepad-widgets = { path = "../../widgets" }
|
||||
makepad-wm-theme = { path = "../../libs/wm_theme" }
|
||||
makepad-wm-api = { path = "../../libs/wm_api" }
|
||||
# The AI services wire: bounded read and confirmed mutation tools on the bus
|
||||
# (hosted port, id-correlated runner — see ai_service.rs).
|
||||
makepad-ai-services = { path = "../../libs/ai/services" }
|
||||
makepad-strict-json = { path = "../../libs/strict_json" }
|
||||
# The ask panel's local model: an in-process Qwen GGUF on makepad-ggml, loaded
|
||||
# only when the panel is first opened.
|
||||
makepad-ai-hub = { path = "../../libs/ai/hub", default-features = false, features = ["llm"], optional = true }
|
||||
64
apps/files/build.rs
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
#[path = "../../libs/ai/models/paint/src/png.rs"]
|
||||
#[allow(dead_code)]
|
||||
mod png;
|
||||
|
||||
use std::{env, fs, path::Path};
|
||||
|
||||
const WIDTH: u32 = 256;
|
||||
const HEIGHT: u32 = 128;
|
||||
|
||||
const PALETTES: [(&str, [u8; 3], [u8; 3], [u8; 3]); 6] = [
|
||||
("aurora-vignette.png", [28, 31, 78], [69, 175, 170], [227, 151, 232]),
|
||||
("canyon-vignette.png", [86, 35, 38], [221, 119, 70], [255, 211, 128]),
|
||||
("lagoon-vignette.png", [12, 55, 77], [28, 154, 166], [151, 232, 207]),
|
||||
("meadow-vignette.png", [36, 68, 45], [124, 167, 79], [234, 213, 126]),
|
||||
("twilight-vignette.png", [36, 25, 65], [103, 71, 141], [238, 144, 116]),
|
||||
("cinema-still.png", [19, 24, 39], [56, 75, 105], [235, 176, 91]),
|
||||
];
|
||||
|
||||
fn mix(a: u8, b: u8, amount: u32) -> u32 {
|
||||
(u32::from(a) * (255 - amount) + u32::from(b) * amount) / 255
|
||||
}
|
||||
|
||||
fn picture(top: [u8; 3], bottom: [u8; 3], accent: [u8; 3], seed: u32) -> Vec<u8> {
|
||||
let mut pixels = Vec::with_capacity((WIDTH * HEIGHT * 3) as usize);
|
||||
let glow_x = 36 + seed * 31;
|
||||
let glow_y = 22 + (seed * 17) % 54;
|
||||
for y in 0..HEIGHT {
|
||||
for x in 0..WIDTH {
|
||||
let blend = (y * 190 / (HEIGHT - 1) + x * 65 / (WIDTH - 1)).min(255);
|
||||
let edge_x = (x as i32 * 2 - (WIDTH - 1) as i32).unsigned_abs();
|
||||
let edge_y = (y as i32 * 2 - (HEIGHT - 1) as i32).unsigned_abs();
|
||||
let vignette = edge_x * edge_x * 18 / ((WIDTH - 1) * (WIDTH - 1))
|
||||
+ edge_y * edge_y * 24 / ((HEIGHT - 1) * (HEIGHT - 1));
|
||||
let glow_dx = (x as i32 - glow_x as i32).unsigned_abs();
|
||||
let glow_dy = (y as i32 - glow_y as i32).unsigned_abs();
|
||||
let glow = 72u32.saturating_sub(glow_dx * glow_dx / 360 + glow_dy * glow_dy / 90);
|
||||
let texture = ((x * 17 + y * 31 + seed * 43) & 7) as i32 - 3;
|
||||
|
||||
for channel in 0..3 {
|
||||
let base = mix(top[channel], bottom[channel], blend);
|
||||
let lit = (base * (72 - glow) + u32::from(accent[channel]) * glow) / 72;
|
||||
pixels.push((lit as i32 - vignette as i32 + texture).clamp(0, 255) as u8);
|
||||
}
|
||||
}
|
||||
}
|
||||
pixels
|
||||
}
|
||||
|
||||
fn write_if_changed(path: &Path, bytes: &[u8]) {
|
||||
if fs::read(path).ok().as_deref() != Some(bytes) {
|
||||
fs::write(path, bytes).expect("write generated files demo picture");
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
println!("cargo:rerun-if-changed=../../libs/ai/models/paint/src/png.rs");
|
||||
println!("cargo:rerun-if-changed=build.rs");
|
||||
let out_dir = env::var_os("OUT_DIR").expect("Cargo did not set OUT_DIR");
|
||||
for (seed, (name, top, bottom, accent)) in PALETTES.iter().copied().enumerate() {
|
||||
let pixels = picture(top, bottom, accent, seed as u32);
|
||||
let encoded = png::encode_png(WIDTH, HEIGHT, png::PngColor::Rgb, &pixels);
|
||||
write_if_changed(&Path::new(&out_dir).join(name), &encoded);
|
||||
}
|
||||
}
|
||||
BIN
apps/files/demos/amusement-ride.jpg
Normal file
|
After Width: | Height: | Size: 307 KiB |
BIN
apps/files/demos/royal-esplanade-panorama.jpg
Normal file
|
After Width: | Height: | Size: 374 KiB |
BIN
apps/files/demos/rubber-duck-illustration.png
Normal file
|
After Width: | Height: | Size: 143 KiB |
|
Before Width: | Height: | Size: 427 B After Width: | Height: | Size: 427 B |
|
Before Width: | Height: | Size: 297 B After Width: | Height: | Size: 297 B |
|
Before Width: | Height: | Size: 199 B After Width: | Height: | Size: 199 B |
|
Before Width: | Height: | Size: 130 B After Width: | Height: | Size: 130 B |
|
Before Width: | Height: | Size: 160 B After Width: | Height: | Size: 160 B |
|
Before Width: | Height: | Size: 130 B After Width: | Height: | Size: 130 B |
|
Before Width: | Height: | Size: 247 B After Width: | Height: | Size: 247 B |
|
Before Width: | Height: | Size: 175 B After Width: | Height: | Size: 175 B |
|
Before Width: | Height: | Size: 342 B After Width: | Height: | Size: 342 B |
|
Before Width: | Height: | Size: 183 B After Width: | Height: | Size: 183 B |
|
Before Width: | Height: | Size: 230 B After Width: | Height: | Size: 230 B |
|
Before Width: | Height: | Size: 218 B After Width: | Height: | Size: 218 B |
|
Before Width: | Height: | Size: 305 B After Width: | Height: | Size: 305 B |
|
Before Width: | Height: | Size: 109 B After Width: | Height: | Size: 109 B |
|
Before Width: | Height: | Size: 252 B After Width: | Height: | Size: 252 B |
|
Before Width: | Height: | Size: 197 B After Width: | Height: | Size: 197 B |
|
Before Width: | Height: | Size: 128 B After Width: | Height: | Size: 128 B |
|
Before Width: | Height: | Size: 127 B After Width: | Height: | Size: 127 B |
|
Before Width: | Height: | Size: 359 B After Width: | Height: | Size: 359 B |
|
Before Width: | Height: | Size: 186 B After Width: | Height: | Size: 186 B |
|
Before Width: | Height: | Size: 152 B After Width: | Height: | Size: 152 B |
|
Before Width: | Height: | Size: 159 B After Width: | Height: | Size: 159 B |
|
Before Width: | Height: | Size: 188 B After Width: | Height: | Size: 188 B |
|
Before Width: | Height: | Size: 141 B After Width: | Height: | Size: 141 B |
|
Before Width: | Height: | Size: 398 B After Width: | Height: | Size: 398 B |
|
Before Width: | Height: | Size: 185 B After Width: | Height: | Size: 185 B |
|
Before Width: | Height: | Size: 256 B After Width: | Height: | Size: 256 B |
|
Before Width: | Height: | Size: 148 B After Width: | Height: | Size: 148 B |
|
Before Width: | Height: | Size: 166 B After Width: | Height: | Size: 166 B |
|
Before Width: | Height: | Size: 305 B After Width: | Height: | Size: 305 B |
|
Before Width: | Height: | Size: 120 B After Width: | Height: | Size: 120 B |
|
Before Width: | Height: | Size: 152 B After Width: | Height: | Size: 152 B |
|
Before Width: | Height: | Size: 140 B After Width: | Height: | Size: 140 B |
|
Before Width: | Height: | Size: 160 B After Width: | Height: | Size: 160 B |
|
Before Width: | Height: | Size: 92 B After Width: | Height: | Size: 92 B |
|
Before Width: | Height: | Size: 93 B After Width: | Height: | Size: 93 B |
|
Before Width: | Height: | Size: 713 B After Width: | Height: | Size: 713 B |
250
apps/files/src/ai_service.rs
Normal file
|
|
@ -0,0 +1,250 @@
|
|||
//! The file browser on the desktop's AI bus.
|
||||
//!
|
||||
//! Hosted by the window manager, the app opens one [`AiServicePort`] with
|
||||
//! the manifest from `chat_tools::service_manifest` and answers the calls
|
||||
//! that come back through it. The tools are the same seven the app's own
|
||||
//! panel has; what is new is how they run: every call carries the
|
||||
//! engine's `call_id`, the answer carries it back, and the person (or the
|
||||
//! router) can give up on a call mid-walk. The old panel's runner is
|
||||
//! order-only and cannot do either, so this one sits beside it rather
|
||||
//! than inside it, and the two never share a job.
|
||||
//!
|
||||
//! One worker thread, one job at a time. A job in flight is cancelled by
|
||||
//! a flag the walk checks on every entry; a job still queued behind it is
|
||||
//! cancelled before it starts. Progress from the walk and the finished
|
||||
//! results come back on channels the UI drains on its signal.
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
path::PathBuf,
|
||||
sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
mpsc::{channel, Receiver, Sender},
|
||||
Arc,
|
||||
},
|
||||
};
|
||||
|
||||
use makepad_ai_services::wire::{ServiceCall, ToolResult};
|
||||
use makepad_strict_json as json;
|
||||
use makepad_widgets::makepad_platform::thread::{SignalToUI, ThreadOptions, ThreadSpawner};
|
||||
|
||||
use crate::chat_tools::{run_with, ToolJob, ToolOutcome};
|
||||
|
||||
#[cfg(test)]
|
||||
use std::thread;
|
||||
|
||||
/// One call on its way to the worker.
|
||||
struct ServiceJob {
|
||||
call_id: String,
|
||||
job: ToolJob,
|
||||
cancel: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
/// What the worker sends back, in the order it happened.
|
||||
pub enum ServiceReply {
|
||||
Result { result: ToolResult, mutated: bool },
|
||||
Progress { call_id: String, note: String, permille: u16 },
|
||||
}
|
||||
|
||||
/// The bus's tool worker: correlated by call id, cancellable.
|
||||
pub struct ServiceRunner {
|
||||
jobs: Sender<ServiceJob>,
|
||||
replies: Receiver<ServiceReply>,
|
||||
/// The cancel flag of every call not yet answered.
|
||||
live: HashMap<String, Arc<AtomicBool>>,
|
||||
}
|
||||
|
||||
impl ServiceRunner {
|
||||
pub fn new(spawner: &ThreadSpawner) -> Self {
|
||||
let (jobs, job_rx) = channel::<ServiceJob>();
|
||||
let (reply_tx, replies) = channel::<ServiceReply>();
|
||||
if let Ok(handle) = spawner.spawn_worker(
|
||||
ThreadOptions { name: Some("files-ai-service".into()), ..Default::default() },
|
||||
move || {
|
||||
while let Ok(ServiceJob { call_id, job, cancel }) = job_rx.recv() {
|
||||
let (result, mutated) = if cancel.load(Ordering::Relaxed) {
|
||||
// Given up on while it waited its turn: never started.
|
||||
(ToolResult::cancelled(&call_id), false)
|
||||
} else {
|
||||
let progress_tx = reply_tx.clone();
|
||||
let progress_id = call_id.clone();
|
||||
let progress = move |permille: u16| {
|
||||
let _ = progress_tx.send(ServiceReply::Progress {
|
||||
call_id: progress_id.clone(),
|
||||
note: "measuring…".to_string(),
|
||||
permille,
|
||||
});
|
||||
SignalToUI::set_ui_signal();
|
||||
};
|
||||
let outcome = run_with(&job, &cancel, &progress);
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
// The walk stopped early on the flag: what it has
|
||||
// is a floor nobody asked for any more.
|
||||
(ToolResult::cancelled(&call_id), false)
|
||||
} else {
|
||||
let mutated = outcome.mutated;
|
||||
(result_for(&call_id, outcome), mutated)
|
||||
}
|
||||
};
|
||||
if reply_tx.send(ServiceReply::Result { result, mutated }).is_err() {
|
||||
return;
|
||||
}
|
||||
SignalToUI::set_ui_signal();
|
||||
}
|
||||
},
|
||||
) {
|
||||
handle.detach();
|
||||
}
|
||||
Self { jobs, replies, live: HashMap::new() }
|
||||
}
|
||||
|
||||
/// Queue one call. `cwd` is the folder the person is looking at — what
|
||||
/// a relative path is read from — and `home` the jail.
|
||||
pub fn submit(&mut self, call: &ServiceCall, cwd: PathBuf, home: PathBuf) {
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
self.live.insert(call.call_id.clone(), cancel.clone());
|
||||
let job = ToolJob {
|
||||
name: call.tool.clone(),
|
||||
args: flat_args(&call.args),
|
||||
cwd,
|
||||
home,
|
||||
};
|
||||
let _ = self.jobs.send(ServiceJob { call_id: call.call_id.clone(), job, cancel });
|
||||
}
|
||||
|
||||
/// Give up on one call, running or queued. Unknown ids are nothing.
|
||||
pub fn cancel(&mut self, call_id: &str) {
|
||||
if let Some(flag) = self.live.get(call_id) {
|
||||
flag.store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/// Everything the worker sent since the last drain. A result retires
|
||||
/// its call's flag.
|
||||
pub fn drain(&mut self) -> Vec<ServiceReply> {
|
||||
let replies: Vec<ServiceReply> = self.replies.try_iter().collect();
|
||||
for reply in &replies {
|
||||
if let ServiceReply::Result { result, .. } = reply {
|
||||
self.live.remove(&result.call_id);
|
||||
}
|
||||
}
|
||||
replies
|
||||
}
|
||||
}
|
||||
|
||||
/// A tool's outcome as the wire says it. The tools already sort their own
|
||||
/// failures into "refused" (the jail, an unknown name) and "could not"
|
||||
/// (the disk said no); the wire keeps that distinction.
|
||||
fn result_for(call_id: &str, outcome: ToolOutcome) -> ToolResult {
|
||||
if !outcome.is_error {
|
||||
return ToolResult::ok(call_id, outcome.text, outcome.note);
|
||||
}
|
||||
if outcome.note.starts_with("refused") || outcome.note.starts_with("unknown tool") {
|
||||
ToolResult::refused(call_id, outcome.text)
|
||||
} else {
|
||||
ToolResult::failed(call_id, outcome.text)
|
||||
}
|
||||
}
|
||||
|
||||
/// The call's JSON argument object as the tools read it: one string per
|
||||
/// key. Numbers and booleans become their text; nested values are dropped,
|
||||
/// since no tool here takes one.
|
||||
pub fn flat_args(args: &str) -> Vec<(String, String)> {
|
||||
let Ok(json::Value::Obj(fields)) = json::parse(args.as_bytes()) else {
|
||||
return Vec::new();
|
||||
};
|
||||
fields
|
||||
.into_iter()
|
||||
.filter_map(|(key, value)| {
|
||||
let text = match value {
|
||||
json::Value::Str(s) => s,
|
||||
json::Value::Int(i) => i.to_string(),
|
||||
json::Value::F64(f) => f.to_string(),
|
||||
json::Value::Bool(b) => b.to_string(),
|
||||
_ => return None,
|
||||
};
|
||||
Some((key, text))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
// Native worker tests use std deadlines while exercising blocking threads.
|
||||
#[allow(clippy::disallowed_types, clippy::disallowed_methods)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use makepad_ai_services::wire::ToolOutcome as Outcome;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
fn call(id: &str, tool: &str, args: &str) -> ServiceCall {
|
||||
ServiceCall { call_id: id.into(), tool: tool.into(), args: args.into() }
|
||||
}
|
||||
|
||||
fn wait_for(runner: &mut ServiceRunner, n: usize) -> Vec<ToolResult> {
|
||||
let deadline = Instant::now() + Duration::from_secs(20);
|
||||
let mut out = Vec::new();
|
||||
while out.len() < n && Instant::now() < deadline {
|
||||
for reply in runner.drain() {
|
||||
if let ServiceReply::Result { result: r, .. } = reply {
|
||||
out.push(r);
|
||||
}
|
||||
}
|
||||
thread::sleep(Duration::from_millis(5));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_args_become_the_tools_flat_pairs() {
|
||||
let args = flat_args(r#"{"path":"~/Downloads","top":3,"deep":{"x":1},"quick":true}"#);
|
||||
assert_eq!(
|
||||
args,
|
||||
vec![
|
||||
("path".to_string(), "~/Downloads".to_string()),
|
||||
("top".to_string(), "3".to_string()),
|
||||
("quick".to_string(), "true".to_string()),
|
||||
]
|
||||
);
|
||||
assert!(flat_args("not json").is_empty());
|
||||
assert!(flat_args("[1,2]").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn results_carry_their_own_call_ids_and_a_queued_cancel_never_runs() {
|
||||
let home = crate::model::home_dir();
|
||||
let spawner = makepad_widgets::Cx::new(Box::new(|_, _| {})).thread_spawner();
|
||||
let mut runner = ServiceRunner::new(&spawner);
|
||||
// An unknown tool is refused; a jail escape is refused; both keep
|
||||
// their ids whatever order the worker answers in.
|
||||
runner.submit(&call("a", "stat", r#"{"path":"/etc/passwd"}"#), home.clone(), home.clone());
|
||||
runner.submit(&call("b", "rm_rf", r#"{"path":"~"}"#), home.clone(), home.clone());
|
||||
runner.submit(&call("c", "stat", r#"{"path":"~"}"#), home.clone(), home.clone());
|
||||
// Cancelled while it is still queued: it must come back cancelled
|
||||
// without the tool ever running.
|
||||
runner.cancel("c");
|
||||
assert_eq!(runner.live.len(), 3);
|
||||
let results = wait_for(&mut runner, 3);
|
||||
assert_eq!(results.len(), 3);
|
||||
let by_id = |id: &str| results.iter().find(|r| r.call_id == id).unwrap();
|
||||
assert_eq!(by_id("a").outcome, Outcome::Refused);
|
||||
assert!(by_id("a").text.contains("refused"));
|
||||
assert_eq!(by_id("b").outcome, Outcome::Refused);
|
||||
assert!(by_id("b").text.contains("no tool called"));
|
||||
assert_eq!(by_id("c").outcome, Outcome::Cancelled);
|
||||
assert_eq!(runner.live.len(), 0);
|
||||
// An unknown id is nothing.
|
||||
runner.cancel("zzz");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_folder_walk_stops_on_the_flag() {
|
||||
let cancel = AtomicBool::new(true);
|
||||
let started = Instant::now();
|
||||
let (bytes, files, complete) = crate::chat_tools::measure_for_test(
|
||||
std::path::Path::new(env!("CARGO_MANIFEST_DIR")),
|
||||
&cancel,
|
||||
);
|
||||
assert_eq!((bytes, files, complete), (0, 0, false));
|
||||
assert!(started.elapsed() < Duration::from_secs(1));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
//! The sidebar's bookmarks: folders the user keeps, in their own section.
|
||||
//!
|
||||
//! Bookmarks are the one piece of mpfiles state that outlives the process, so
|
||||
//! Bookmarks are the one piece of files state that outlives the process, so
|
||||
//! the format is the one a person can fix in an editor when it goes wrong: one
|
||||
//! absolute path per line, in the order the sidebar shows them. That is also
|
||||
//! what GNOME Files stores (`~/.config/gtk-3.0/bookmarks`), minus the URI
|
||||
|
|
@ -18,9 +18,9 @@ use std::{
|
|||
/// last slot would be saved and never shown, which is worse than refusing it.
|
||||
pub const MAX_BOOKMARKS: usize = 12;
|
||||
|
||||
/// The bookmarks file for a given home directory.
|
||||
pub fn config_path(home: &Path) -> PathBuf {
|
||||
home.join(".config").join("mpfiles").join("bookmarks")
|
||||
/// The bookmarks file for a given Makepad home directory.
|
||||
pub fn config_path(makepad_home: &Path) -> PathBuf {
|
||||
makepad_home.join("files/bookmarks")
|
||||
}
|
||||
|
||||
/// The bookmark list, in sidebar order, and where it is persisted.
|
||||
|
|
@ -160,7 +160,7 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn survives_a_round_trip_through_a_real_file() {
|
||||
let home = std::env::temp_dir().join("mpfiles-test-bookmarks");
|
||||
let home = std::env::temp_dir().join("files-test-bookmarks");
|
||||
let _ = fs::remove_dir_all(&home);
|
||||
fs::create_dir_all(&home).unwrap();
|
||||
|
||||
|
|
@ -17,7 +17,7 @@ pub use makepad_ai_hub::local_llm::ChatEvent;
|
|||
/// Where the weights live, relative to the checkout this was built from.
|
||||
pub const MODEL_FILE: &str = "local/models/Qwen3.5-9B-UD-Q4_K_XL.gguf";
|
||||
/// The environment variable that overrides it.
|
||||
pub const MODEL_ENV: &str = "MPFILES_CHAT_MODEL";
|
||||
pub const MODEL_ENV: &str = "MAKEPAD_FILES_CHAT_MODEL";
|
||||
|
||||
pub struct ChatAgent {
|
||||
session: HubChatSession,
|
||||
|
|
@ -57,7 +57,7 @@ impl ChatAgent {
|
|||
|
||||
/// Where the weights are, or `None` when this machine has none.
|
||||
///
|
||||
/// `MPFILES_CHAT_MODEL` wins; otherwise the file is looked for relative to the
|
||||
/// `MAKEPAD_FILES_CHAT_MODEL` wins; otherwise the file is looked for relative to the
|
||||
/// working directory, then up from the binary (which finds `target/release`
|
||||
/// runs from anywhere), then in the checkout this binary was compiled in.
|
||||
pub fn model_path() -> Option<PathBuf> {
|
||||