import Foundation import Speech import AVFoundation import CoreMedia // The STT half of makepad-system-speech's Apple bridge: SpeechAnalyzer (macOS 26 / // iOS 26) over caller-recorded 16 kHz mono PCM. Symbols are prefixed `mss_` so // this static library can coexist with any other Swift bridge in the binary. struct MssSegment { var text: UnsafeMutablePointer? var start_ms: Int64 var end_ms: Int64 } func mssResolveLocale(_ tag: String) -> Locale { let defaults: [String: String] = [ "en": "en-US", "fr": "fr-FR", "de": "de-DE", "es": "es-ES", "it": "it-IT", "pt": "pt-BR", "nl": "nl-NL", "zh": "zh-CN", "ja": "ja-JP", "ko": "ko-KR", "ru": "ru-RU", "yue": "yue-CN", ] return Locale(identifier: defaults[tag] ?? tag.replacingOccurrences(of: "_", with: "-")) } func mssRunAsyncSync(_ body: @escaping @Sendable () async throws -> T) throws -> T { let sem = DispatchSemaphore(value: 0) let box = UnsafeMutablePointer>.allocate(capacity: 1) box.initialize(to: .failure(NSError(domain: "uninit", code: 0))) DispatchQueue.global(qos: .userInitiated).async { Task { do { box.pointee = .success(try await body()) } catch { box.pointee = .failure(error) } sem.signal() } } sem.wait() let r = box.pointee box.deallocate() switch r { case .success(let v): return v case .failure(let e): throw e } } /// Recognize `sampleCount` floats of 16 kHz mono PCM. Returns 0 and an owned /// array of `MssSegment` (release with `mss_stt_free_segments`), or a /// negative code: -1 recognition failed, -2 locale unsupported. @_cdecl("mss_stt_transcribe") func mss_stt_transcribe( _ samples: UnsafePointer, _ sampleCount: Int64, _ lang: UnsafePointer, _ wantTimestamps: Int32, _ outCount: UnsafeMutablePointer, _ outSegments: UnsafeMutablePointer ) -> Int32 { let requestedLocale = mssResolveLocale(String(cString: lang)) let count = Int(sampleCount) outCount.pointee = 0 outSegments.pointee = nil if count == 0 { return 0 } let samplesCopy = Array(UnsafeBufferPointer(start: samples, count: count)) do { let segments: [(String, Int64, Int64)] = try mssRunAsyncSync { let locale = await SpeechTranscriber.supportedLocale(equivalentTo: requestedLocale) ?? requestedLocale let preset = SpeechTranscriber.Preset( transcriptionOptions: [], reportingOptions: [], attributeOptions: wantTimestamps != 0 ? [.audioTimeRange] : [] ) let transcriber = SpeechTranscriber(locale: locale, preset: preset) let analyzer = SpeechAnalyzer(modules: [transcriber]) let inputFormat = AVAudioFormat( commonFormat: .pcmFormatFloat32, sampleRate: 16000, channels: 1, interleaved: false )! // SpeechAnalyzer's file entry point is the one that finishes // deterministically after the last sample; write the PCM to a // temporary CAF and hand it that. let tempURL = URL(fileURLWithPath: NSTemporaryDirectory()) .appendingPathComponent("makepad_mss_\(UUID().uuidString).caf") defer { try? FileManager.default.removeItem(at: tempURL) } let outFile = try AVAudioFile(forWriting: tempURL, settings: inputFormat.settings) let allFrames = AVAudioFrameCount(count) let outBuf = AVAudioPCMBuffer(pcmFormat: inputFormat, frameCapacity: allFrames)! outBuf.frameLength = allFrames samplesCopy.withUnsafeBufferPointer { buf in memcpy(outBuf.floatChannelData![0], buf.baseAddress!, count * MemoryLayout.size) } try outFile.write(from: outBuf) let inFile = try AVAudioFile(forReading: tempURL) try await analyzer.start(inputAudioFile: inFile, finishAfterFile: true) var segs: [(String, Int64, Int64)] = [] for try await response in transcriber.results { let text = String(response.text.characters) if response.isFinal { let range = response.range let startMs = Int64(CMTimeGetSeconds(range.start) * 1000) let endMs = Int64(CMTimeGetSeconds(CMTimeAdd(range.start, range.duration)) * 1000) if !text.isEmpty { segs.append((text, startMs, endMs)) } } } return segs } outCount.pointee = Int32(segments.count) if segments.isEmpty { return 0 } let ptr = UnsafeMutablePointer.allocate(capacity: segments.count) for (i, (text, startMs, endMs)) in segments.enumerated() { ptr[i] = MssSegment(text: strdup(text), start_ms: startMs, end_ms: endMs) } outSegments.pointee = OpaquePointer(ptr) return 0 } catch { NSLog("[makepad-system-speech] stt transcribe: %@", error.localizedDescription) return -1 } } @_cdecl("mss_stt_free_segments") func mss_stt_free_segments(_ ptr: OpaquePointer?, _ count: Int32) { guard let rawPtr = ptr else { return } let typed = UnsafeMutablePointer(rawPtr) for i in 0..) -> Int32 { let requestedLocale = mssResolveLocale(String(cString: lang)) do { try mssRunAsyncSync { let locale = await SpeechTranscriber.supportedLocale(equivalentTo: requestedLocale) ?? requestedLocale let supported = await SpeechTranscriber.supportedLocales let bcp47 = locale.identifier(.bcp47) guard supported.contains(where: { $0.identifier(.bcp47) == bcp47 }) else { throw NSError(domain: "mss", code: -2, userInfo: [NSLocalizedDescriptionKey: "Unsupported: \(bcp47)"]) } let installed = await SpeechTranscriber.installedLocales if !installed.contains(where: { $0.identifier(.bcp47) == bcp47 }) { let preset = SpeechTranscriber.Preset( transcriptionOptions: [], reportingOptions: [], attributeOptions: []) let t = SpeechTranscriber(locale: locale, preset: preset) if let dl = try await AssetInventory.assetInstallationRequest(supporting: [t]) { try await dl.downloadAndInstall() } } } return 0 } catch let e as NSError where e.code == -2 { return -2 } catch { NSLog("[makepad-system-speech] stt prepare: %@", error.localizedDescription) return -1 } }