import Foundation // @preconcurrency:AVFAudio 的 AVAudioConverterInputBlock 标了 @Sendable 但 // AVAudioPCMBuffer 未标 Sendable,decodeToMonoFloat 同步喂 buffer 是安全的,压掉误报。 @preconcurrency import AVFoundation /// 端侧问诊语音转写服务(SenseVoice,经 sherpa-mnn 跑在 MNN 后端)。 /// /// 「记录问诊」录完整段录音后调它做**离线转写**(非流式,无实时字幕), /// 转写稿再交本地 LLM(DiaryAssistService.organizeConsultation)整理成问诊小结。 /// /// 守红线: /// - 全程本机,无任何网络(§1 / §10 #1)。 /// - 与 LLM/VL 经 `AIRuntime.runExclusiveForASR` 闸门**互斥**:转写前卸掉常驻文本/视觉模型腾内存, /// 避免两个模型同时常驻冲过单 App 内存上限被 jetsam 杀(§3.1 OOM 防护,见 [[airuntime-llm-vl-oom-gate]])。 /// - 模型或引擎未就绪时本服务抛错,调用方自动回退系统端侧识别(SFSpeech),App 不卡死(§10 #5)。 /// /// 模型与 LLM 的 `ModelKind` 解耦,自管 `Models/SenseVoice/` 目录(model.mnn + tokens.txt)。 /// 转换/接入步骤见 docs/release/sensevoice-integration.md。 struct SenseVoiceASRService { static let shared = SenseVoiceASRService() private init() {} enum ASRError: Error, LocalizedError { case modelNotInstalled case engineUnavailable case decodeFailed case empty var errorDescription: String? { switch self { case .modelNotInstalled: return String(appLoc: "问诊转写模型未就绪") case .engineUnavailable: return String(appLoc: "本机暂不支持本地语音转写") case .decodeFailed: return String(appLoc: "录音解码失败") case .empty: return String(appLoc: "没识别到语音内容") } } } // MARK: - 模型位置(独立于 LLM 的 ModelKind,问诊 ASR 自管目录) /// `Application Support/Models/SenseVoice/`,与 LLM 模型同根目录、互不干扰。 nonisolated static var modelDir: URL { ModelStore.shared.rootURL.appendingPathComponent("SenseVoice", isDirectory: true) } /// MNNConvert 产出的 SenseVoice 图。 nonisolated static var modelFile: URL { modelDir.appendingPathComponent("model.mnn") } /// tokens.txt(id↔token 映射)。 nonisolated static var tokensFile: URL { modelDir.appendingPathComponent("tokens.txt") } /// 模型两件套是否都已就位(下载 / 旁路导入后)。 nonisolated static var isModelInstalled: Bool { let fm = FileManager.default return fm.fileExists(atPath: modelFile.path) && fm.fileExists(atPath: tokensFile.path) } /// 端侧 SenseVoice 是否可用 = 引擎已链接(sherpa-mnn)**且**模型已就位。 /// false 时「记录问诊」自动回退系统端侧识别(SFSpeech)。 nonisolated static var isAvailable: Bool { SenseVoiceBridge.isAvailable() && isModelInstalled } // MARK: - 转写 /// 把一段录音(m4a/wav/caf 等)离线转写成文字。失败抛错,调用方回退。 /// language:"auto" 自动判别(中英日韩粤),也可固定 "zh"。 func transcribe(audioFileURL: URL, language: String = "auto") async throws -> String { guard SenseVoiceBridge.isAvailable() else { throw ASRError.engineUnavailable } guard Self.isModelInstalled else { throw ASRError.modelNotInstalled } let modelPath = Self.modelFile.path let tokensPath = Self.tokensFile.path // 解码 + 解码推理都重(CPU),全部放进闸门内的后台线程:与 LLM/VL 串行,且先卸常驻模型腾内存。 let raw = try await AIRuntime.shared.runExclusiveForASR { try await Self.runOnBackground { let decoded = try Self.decodeToMonoFloat(url: audioFileURL) guard !decoded.samples.isEmpty else { throw ASRError.decodeFailed } guard let bridge = SenseVoiceBridge(modelPath: modelPath, tokensPath: tokensPath, language: language) else { throw ASRError.engineUnavailable } let text = decoded.samples.withUnsafeBufferPointer { buf -> String? in guard let base = buf.baseAddress else { return nil } return bridge.transcribeSamples(base, count: Int32(buf.count), sampleRate: Int32(decoded.sampleRate)) } return text ?? "" } } let cleaned = Self.cleanTranscript(raw) guard !cleaned.isEmpty else { throw ASRError.empty } return cleaned } // MARK: - 纯函数(单测覆盖) /// 清洗 SenseVoice 输出:去掉可能残留的 `<|zh|><|NEUTRAL|><|Speech|><|woitn|>` 等标签后 trim。 /// sherpa 多数情况下已剥成纯文本,这里再兜一层,确保给 LLM 的转写稿干净。 nonisolated static func cleanTranscript(_ raw: String) -> String { let stripped = raw.replacingOccurrences( of: "<\\|[^|]*\\|>", with: "", options: .regularExpression) return stripped.trimmingCharacters(in: .whitespacesAndNewlines) } // MARK: - 音频解码(任意容器 → 16kHz 单声道 float32) private struct Decoded { let samples: [Float]; let sampleRate: Int } /// 用 AVAudioConverter 把录音重采样/混音成 16kHz 单声道 float32(SenseVoice 期望的输入域)。 /// 一次性整段转换:本调用已在闸门内、LLM 已卸,瞬时内存可控。 nonisolated private static func decodeToMonoFloat(url: URL, targetSampleRate: Double = 16000) throws -> Decoded { let file = try AVAudioFile(forReading: url) let inFormat = file.processingFormat let frameCount = AVAudioFrameCount(file.length) guard frameCount > 0 else { return Decoded(samples: [], sampleRate: Int(targetSampleRate)) } guard let targetFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: targetSampleRate, channels: 1, interleaved: false), let converter = AVAudioConverter(from: inFormat, to: targetFormat), let inBuffer = AVAudioPCMBuffer(pcmFormat: inFormat, frameCapacity: frameCount) else { throw ASRError.decodeFailed } try file.read(into: inBuffer) // 目标采样率通常低于源(48k/44.1k→16k),输出更短;源若低于 16k 则 ratio>1,按比例 + 余量留足。 let ratio = targetSampleRate / inFormat.sampleRate let outCapacity = AVAudioFrameCount(Double(frameCount) * ratio) + 1024 guard let outBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: outCapacity) else { throw ASRError.decodeFailed } var fed = false var convError: NSError? let status = converter.convert(to: outBuffer, error: &convError) { _, inStatus in if fed { inStatus.pointee = .endOfStream // 已喂完整段,通知 flush 余下重采样样本 return nil } fed = true inStatus.pointee = .haveData return inBuffer } if let convError { throw convError } guard status != .error else { throw ASRError.decodeFailed } guard let channel = outBuffer.floatChannelData?[0] else { return Decoded(samples: [], sampleRate: Int(targetSampleRate)) } let n = Int(outBuffer.frameLength) let samples = Array(UnsafeBufferPointer(start: channel, count: n)) return Decoded(samples: samples, sampleRate: Int(targetSampleRate)) } /// 把一段阻塞的同步工作放到后台 QoS 队列跑(转写解码同步阻塞,绝不能占住 actor/主线程)。 nonisolated private static func runOnBackground( _ work: @escaping @Sendable () throws -> T ) async throws -> T { try await withCheckedThrowingContinuation { cont in DispatchQueue.global(qos: .userInitiated).async { do { cont.resume(returning: try work()) } catch { cont.resume(throwing: error) } } } } }