392 lines
17 KiB
Text
392 lines
17 KiB
Text
/**
|
|
Recipe prototypes for the asset creator's generation kinds.
|
|
|
|
`text.expand` deliberately uses the existing `mod.flow.Llm` node instead of
|
|
adding an `Expand` prototype: expansion is a pure text turn, and `Llm` exposes
|
|
the system instruction that makes the rewrite inspectable in each template.
|
|
*/
|
|
/** Ask a vision model a question about an image. Default face: `mod.flow.ui.GenFace`. */
|
|
mod.flow.Vision = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "vision"
|
|
type_name: "Vision"
|
|
/** 16..512 */ max_tokens: 512
|
|
/** Empty selects the hub's best model for the vision domain. */ model: ""
|
|
ports: { in: {prompt: @text, image: @image}, out: {text: @text} }
|
|
text: fn() { self.out(@text) }
|
|
/** Generic parameter and text-result face. */ ui: mod.flow.ui.GenFace
|
|
}
|
|
|
|
/** Edit a primary image with an instruction. Default face: `mod.flow.ui.ImageFace`. */
|
|
mod.flow.ImageEdit = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "edit"
|
|
type_name: "ImageEdit"
|
|
/** one of: 1.0, 0.85, 0.7, 0.55, 0.4, 0.25 */ strength: 1.0
|
|
/** 0 is the deterministic default seed. */ seed: 0
|
|
/** Empty selects the hub's best model for the edit domain. */ model: ""
|
|
ports: {
|
|
in: {
|
|
prompt: @text
|
|
image: @image
|
|
/** Optional additional reference image; up to three total. */ reference_1: @image
|
|
/** Optional additional reference image; up to three total. */ reference_2: @image
|
|
/** Optional additional reference image; up to three total. */ reference_3: @image
|
|
}
|
|
out: { image: @image }
|
|
}
|
|
image: fn() { self.out(@image) }
|
|
/** Image preview with generic edit parameters. */ ui: mod.flow.ui.ImageFace
|
|
}
|
|
|
|
/** Render an image constrained by a control image. Default face: `mod.flow.ui.ImageFace`. */
|
|
mod.flow.Control = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "control"
|
|
type_name: "Control"
|
|
/** 1..200; the control backend default is 30. */ steps: 30
|
|
/** 0..2000; Canny default is 50. */ canny_low: 50.0
|
|
/** 0..2000; Canny default is 200. */ canny_high: 200.0
|
|
/** 0 is the deterministic default seed. */ seed: 0
|
|
/** Empty selects depth or Canny control by hub affinity. */ model: ""
|
|
ports: { in: {prompt: @text, control: @image}, out: {image: @image} }
|
|
image: fn() { self.out(@image) }
|
|
/** Image preview with generic control parameters. */ ui: mod.flow.ui.ImageFace
|
|
}
|
|
|
|
/** Fill or extend a masked region of an image. Default face: `mod.flow.ui.ImageFace`. */
|
|
mod.flow.Inpaint = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "inpaint"
|
|
type_name: "Inpaint"
|
|
/** 1..200; the inpaint backend default is 50. */ steps: 50
|
|
/** 0..100; the inpaint backend default is 30. */ guidance: 30.0
|
|
/** 0 is the deterministic default seed. */ seed: 0
|
|
/** Empty selects the hub's best model for the inpaint domain. */ model: ""
|
|
ports: {
|
|
in: {
|
|
prompt: @text
|
|
image: @image
|
|
/** White marks the region to regenerate; black is kept. */ mask: @image
|
|
}
|
|
out: { image: @image }
|
|
}
|
|
image: fn() { self.out(@image) }
|
|
/** Image preview with generic inpaint parameters. */ ui: mod.flow.ui.ImageFace
|
|
}
|
|
|
|
/** Extract a soft alpha matte from an image. Default face: `mod.flow.ui.ImageFace`. */
|
|
mod.flow.Matte = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "matte"
|
|
type_name: "Matte"
|
|
/** Empty selects the hub's best model for the matte domain. */ model: ""
|
|
ports: { in: {image: @image}, out: {image: @image} }
|
|
image: fn() { self.out(@image) }
|
|
/** Image preview with generic matte parameters. */ ui: mod.flow.ui.ImageFace
|
|
}
|
|
|
|
/** Estimate a metric depth map from an image. Default face: `mod.flow.ui.ImageFace`. */
|
|
mod.flow.Depth = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "depth"
|
|
type_name: "Depth"
|
|
/** Empty selects the hub's best model for the depth domain. */ model: ""
|
|
ports: { in: {image: @image}, out: {image: @image} }
|
|
image: fn() { self.out(@image) }
|
|
/** Image preview with generic depth parameters. */ ui: mod.flow.ui.ImageFace
|
|
}
|
|
|
|
/** Generate a video, optionally from a first image keyframe. Default face: `mod.flow.ui.VideoFace`. */
|
|
mod.flow.Video = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "video"
|
|
type_name: "Video"
|
|
/** 640..960 step 32 paired sizes: 640x352, 864x480, 960x544 */ width: 640
|
|
/** 352..544 step 32 paired sizes: 640x352, 864x480, 960x544 */ height: 352
|
|
/** 5..4096 step 1. Requested native frames; H3 rounds up to 17n+5. Larger clips require more VRAM. */ frames: 39
|
|
/** Denoising steps. */ steps: 30
|
|
/** one of: h264, h265, hevc */ codec: "h264"
|
|
/** Decode and mux the jointly generated audio track. */ audio: true
|
|
/** one of: 1, 2, 4 */ interpolate: 1
|
|
/** 0 is the deterministic default seed. */ seed: 0
|
|
/** Empty selects the hub's best model for the video domain. */ model: ""
|
|
ports: {
|
|
in: {
|
|
prompt: @text
|
|
image: @image
|
|
/** Optional last-frame keyframe, for a closed loop back to `image`. */ last_frame: @image
|
|
}
|
|
out: { video: @video }
|
|
}
|
|
video: fn() { self.out(@video) }
|
|
/** Video settings with an editable frame count. */ ui: mod.flow.ui.VideoFace
|
|
}
|
|
|
|
/** Upscale, interpolate, or add motion metadata to a video. Default face: `mod.flow.ui.GenFace`. */
|
|
mod.flow.VideoEnhance = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "enhance"
|
|
type_name: "VideoEnhance"
|
|
/** one of: 1, 2, 4 */ upscale: 2
|
|
/** one of: 1, 2, 4 */ interpolate: 2
|
|
/** Append the `mkfl` motion-field sidecar. */ flow_map: true
|
|
/** Empty selects the hub's best model for the enhance domain. */ model: ""
|
|
ports: { in: {video: @video}, out: {video: @video} }
|
|
video: fn() { self.out(@video) }
|
|
/** Generic parameter and video-result face. */ ui: mod.flow.ui.GenFace
|
|
}
|
|
|
|
/** Generate a sound effect from a description. Default face: `mod.flow.ui.GenFace`. */
|
|
mod.flow.Sfx = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "audio"
|
|
type_name: "Sfx"
|
|
/** 0.5..120; the current creator default is 4 seconds. */ seconds: 4.0
|
|
/** 1..200; the SA3 default is 8. */ steps: 8
|
|
/** 0 is the deterministic default seed. */ seed: 0
|
|
/** Empty selects the hub's best model for the audio domain. */ model: ""
|
|
ports: { in: {prompt: @text}, out: {audio: @audio} }
|
|
audio: fn() { self.out(@audio) }
|
|
/** Generic parameter and audio-result face. */ ui: mod.flow.ui.GenFace
|
|
}
|
|
|
|
/** Generate a song from a production brief and optional structured lyrics and reference audio. Default face: `mod.flow.ui.GenFace`. */
|
|
mod.flow.Music = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "music"
|
|
type_name: "Music"
|
|
/** one of: 60, 120, 180, 240, 300; accepted range 5..300 */ seconds: 180
|
|
/** 0..1; 0.8 matches the backend's default reference cadence. */ strength: 0.8
|
|
/** 0 is the deterministic default seed. */ seed: 0
|
|
/** Empty selects the hub's best model for the music domain. */ model: ""
|
|
ports: {
|
|
in: {
|
|
prompt: @text
|
|
/** Optional section-tagged lyrics; empty renders instrumental. */ lyrics: @text
|
|
audio: @audio
|
|
}
|
|
out: { audio: @audio }
|
|
}
|
|
audio: fn() { self.out(@audio) }
|
|
/** Generic parameter and audio-result face. */ ui: mod.flow.ui.GenFace
|
|
}
|
|
|
|
/** Sing a full song from style tags and section-tagged lyrics, planning an editable melody-and-chords score (ABC) first; feed an edited score back through `score` to re-sing it. Default face: `mod.flow.ui.GenFace`. */
|
|
mod.flow.Song = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "music"
|
|
type_name: "Song"
|
|
/** one of: full, melody, off; full plans melody and chords, melody only the melody (covers), off no score. */ score_mode: "full"
|
|
/** An ABC score to sing instead of planning one (usually an edited score of an earlier version); empty plans a new score. */ score: ""
|
|
/** 0 is the deterministic default seed. */ seed: 0
|
|
/** A score-planning music model. */ model: "yue2-3b"
|
|
ports: {
|
|
in: {
|
|
/** Style tags: genre, mood, instruments, vocal. */ prompt: @text
|
|
/** Section-tagged lyrics: [Verse], [Chorus], [Bridge]; the song's length follows them. */ lyrics: @text
|
|
}
|
|
out: { audio: @audio sheet: @text }
|
|
}
|
|
audio: fn() { self.out(@audio) }
|
|
sheet: fn() { self.out(@sheet) }
|
|
/** Generic parameter and audio-result face. */ ui: mod.flow.ui.GenFace
|
|
}
|
|
|
|
/** Speak text, optionally cloning a reference-audio voice. Default face: `mod.flow.ui.GenFace`. */
|
|
mod.flow.Speech = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "speech"
|
|
type_name: "Speech"
|
|
/** Empty uses the selected backend's default voice. */ voice: ""
|
|
/** 0.25..4 */ speed: 1.0
|
|
/** Empty uses the model's default language. */ language: ""
|
|
/** Eight values in 0..1.2; empty is neutral/reference affect. */ emotion: []
|
|
/** 0 is the deterministic default seed. */ seed: 0
|
|
/** Empty selects the hub's best model for the speech domain. */ model: ""
|
|
ports: { in: {text: @text, audio: @audio}, out: {audio: @audio} }
|
|
audio: fn() { self.out(@audio) }
|
|
/** Generic parameter and audio-result face. */ ui: mod.flow.ui.GenFace
|
|
}
|
|
|
|
/** Reconstruct a GLB mesh from a reference image. Default face: `mod.flow.ui.GenFace`. */
|
|
mod.flow.Mesh = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "mesh"
|
|
type_name: "Mesh"
|
|
/** 0 for raw decode, otherwise 16..512; creator chains use 512. */ remesh_resolution: 512
|
|
/** Keep TRELLIS's own baked texture. */ texture: true
|
|
/** one of: 12000, 20000, 40000, 80000, 160000; creator object default is 12000. */ decimation_target: 12000
|
|
/** one of: 1024, 2048, 4096 */ texture_size: 1024
|
|
/** 0 is the deterministic default seed. */ seed: 0
|
|
/** Empty selects the hub's best model for the mesh domain. */ model: ""
|
|
ports: { in: {prompt: @text, image: @image}, out: {mesh: @mesh} }
|
|
mesh: fn() { self.out(@mesh) }
|
|
/** Generic parameter and mesh-result face. */ ui: mod.flow.ui.GenFace
|
|
}
|
|
|
|
/** Paint PBR materials onto a mesh from a reference image. Default face: `mod.flow.ui.GenFace`. */
|
|
mod.flow.Paint = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "paint"
|
|
type_name: "Paint"
|
|
/** one of: 1024, 2048, 4096 */ texture_size: 1024
|
|
/** 0 is the deterministic default seed. */ seed: 0
|
|
/** Empty selects the hub's best model for the paint domain. */ model: ""
|
|
ports: { in: {prompt: @text, mesh: @mesh, reference_image: @image}, out: {mesh: @mesh} }
|
|
mesh: fn() { self.out(@mesh) }
|
|
/** Generic parameter and mesh-result face. */ ui: mod.flow.ui.GenFace
|
|
}
|
|
|
|
/** Skin and rig a GLB mesh. Default face: `mod.flow.ui.GenFace`. */
|
|
mod.flow.Rig = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "rig"
|
|
type_name: "Rig"
|
|
/** 0 is the deterministic default seed. */ seed: 0
|
|
/** Empty selects the hub's best model for the rig domain. */ model: ""
|
|
ports: { in: {prompt: @text, mesh: @mesh}, out: {mesh: @mesh} }
|
|
mesh: fn() { self.out(@mesh) }
|
|
/** Generic parameter and mesh-result face. */ ui: mod.flow.ui.GenFace
|
|
}
|
|
|
|
/** Animate a rigged GLB mesh. Default face: `mod.flow.ui.GenFace`. */
|
|
mod.flow.Motion = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "motion"
|
|
type_name: "Motion"
|
|
/** one of: playable, prompt */ motion_mode: "playable"
|
|
/** 0 is the deterministic default seed. */ seed: 0
|
|
/** Empty selects the hub's best model for the motion domain. */ model: ""
|
|
ports: { in: {prompt: @text, mesh: @mesh}, out: {mesh: @mesh} }
|
|
mesh: fn() { self.out(@mesh) }
|
|
/** Generic parameter and mesh-result face. */ ui: mod.flow.ui.GenFace
|
|
}
|
|
|
|
/** Reconstruct one object as a Gaussian-splat PLY. Default face: `mod.flow.ui.GenFace`. */
|
|
mod.flow.Splat = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "splat"
|
|
type_name: "Splat"
|
|
/** 1..200; the TripoSplat default is 20. */ steps: 20
|
|
/** The TripoSplat default is 3. */ guidance: 3.0
|
|
/** 32768..262144 step 32 */ gaussians: 262144
|
|
/** 0 is the deterministic default seed. */ seed: 0
|
|
/** Empty selects the hub's best model for the splat domain. */ model: ""
|
|
ports: { in: {prompt: @text, image: @image}, out: {mesh: @mesh} }
|
|
mesh: fn() { self.out(@mesh) }
|
|
/** Generic parameter and mesh-result face. */ ui: mod.flow.ui.GenFace
|
|
}
|
|
|
|
/** Generate a walkable world as a Gaussian-splat PLY. Default face: `mod.flow.ui.GenFace`. */
|
|
mod.flow.World = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "world"
|
|
type_name: "World"
|
|
/** 0 is the deterministic default seed. */ seed: 0
|
|
/** Empty selects the hub's best model for the world domain. */ model: ""
|
|
ports: { in: {prompt: @text, image: @image}, out: {mesh: @mesh} }
|
|
mesh: fn() { self.out(@mesh) }
|
|
/** Generic parameter and mesh-result face. */ ui: mod.flow.ui.GenFace
|
|
}
|
|
|
|
/** Recover structured human pose data from an image. Default face: `mod.flow.ui.GenFace`. */
|
|
mod.flow.Body = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "body"
|
|
type_name: "Body"
|
|
/** `hands`, `detect`, and `persons=N` may be combined. */ prompt: ""
|
|
/** Empty selects the hub's best model for the body domain. */ model: ""
|
|
ports: { in: {image: @image}, out: {json: @json} }
|
|
json: fn() { self.out(@json) }
|
|
/** Generic parameter and structured-result face. */ ui: mod.flow.ui.GenFace
|
|
}
|
|
|
|
/** Segment one or more prompted subjects from an image. Default face: `mod.flow.ui.ImageFace`. */
|
|
mod.flow.Segment = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "segment"
|
|
type_name: "Segment"
|
|
/** Multiplex prompt such as `person:1` or `cat:2`. */ prompt: "person:1"
|
|
/** Empty selects the hub's best model for the segment domain. */ model: ""
|
|
ports: { in: {image: @image}, out: {mask: @image, cutout: @image} }
|
|
mask: fn() { self.out(@mask) }
|
|
cutout: fn() { self.out(@cutout) }
|
|
/** Image preview with generic segmentation parameters. */ ui: mod.flow.ui.ImageFace
|
|
}
|
|
|
|
/** Transcribe speech audio into a timed JSON transcript. Default face: `mod.flow.ui.GenFace`. */
|
|
mod.flow.Stt = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "stt"
|
|
type_name: "Stt"
|
|
/** Optional BCP-47 language hint; empty enables automatic detection. */ language: ""
|
|
/** Empty selects the hub's best model for speech transcription. */ model: ""
|
|
ports: { in: {audio: @audio}, out: {json: @json} }
|
|
json: fn() { self.out(@json) }
|
|
/** Generic parameter and transcript-result face. */ ui: mod.flow.ui.GenFace
|
|
}
|
|
|
|
/** Detect beats and downbeats in audio. Default face: `mod.flow.ui.GenFace`. */
|
|
mod.flow.Beats = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "beats"
|
|
type_name: "Beats"
|
|
/** Empty selects the hub's best beat-analysis model. */ model: ""
|
|
ports: { in: {audio: @audio}, out: {json: @json} }
|
|
json: fn() { self.out(@json) }
|
|
/** Generic parameter and analysis-result face. */ ui: mod.flow.ui.GenFace
|
|
}
|
|
|
|
/** Separate stereo music into drums, bass, other, and vocals: one audio port per stem, plus the packed bundle. Default face: `mod.flow.ui.GenFace`. */
|
|
mod.flow.Stems = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "stems"
|
|
type_name: "Stems"
|
|
/** Empty selects the hub's best source-separation model. */ model: ""
|
|
ports: { in: {audio: @audio}, out: {drums: @audio, bass: @audio, other: @audio, vocals: @audio, stems: @bytes} }
|
|
drums: fn() { self.out(@drums) }
|
|
bass: fn() { self.out(@bass) }
|
|
other: fn() { self.out(@other) }
|
|
vocals: fn() { self.out(@vocals) }
|
|
stems: fn() { self.out(@stems) }
|
|
/** Generic parameter and stem-bundle face. */ ui: mod.flow.ui.GenFace
|
|
}
|
|
|
|
/** Transcribe polyphonic notes to JSON and MIDI. Default face: `mod.flow.ui.GenFace`. */
|
|
mod.flow.Notes = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "notes"
|
|
type_name: "Notes"
|
|
/** Empty selects the hub's best note-transcription model. */ model: ""
|
|
ports: { in: {audio: @audio}, out: {json: @json, midi: @bytes} }
|
|
json: fn() { self.out(@json) }
|
|
midi: fn() { self.out(@midi) }
|
|
/** Generic parameter and transcription-result face. */ ui: mod.flow.ui.GenFace
|
|
}
|
|
|
|
/** Apply a vision answer to an existing asset's annotation record. Default face: `mod.flow.ui.GenFace`. */
|
|
mod.flow.Annotate = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "vision"
|
|
type_name: "Annotate"
|
|
/** The current nine-line annotation record needs about 200 tokens. */ max_tokens: 200
|
|
/** Empty selects the hub's best model for the vision domain. */ model: ""
|
|
ports: { in: {prompt: @text, image: @image}, out: {json: @json} }
|
|
json: fn() { self.out(@json) }
|
|
/** Generic parameter and annotation-receipt face. */ ui: mod.flow.ui.GenFace
|
|
}
|
|
|
|
/** Transcribe a scanned page into HTML text with the hub's OCR domain. Default face: `mod.flow.ui.GenFace`. */
|
|
mod.flow.Ocr = mod.flow.Gen{
|
|
kind: @gen
|
|
domain: "ocr"
|
|
type_name: "Ocr"
|
|
/** A full page transcribes into about twelve thousand tokens. */ max_tokens: 12384
|
|
/** Empty selects the hub's best model for the OCR domain. */ model: ""
|
|
ports: { in: {prompt: @text, image: @image}, out: {text: @text} }
|
|
text: fn() { self.out(@text) }
|
|
/** Generic parameter and text-result face. */ ui: mod.flow.ui.GenFace
|
|
}
|
|
|
|
mod.flow.freeze_module()
|