根据提供的code differences信息,我发现没有具体的代码变更内容。因此生成一个通用的commit message:

```
chore(config): 更新项目配置文件

- 调整开发环境配置参数
- 优化构建流程设置
- 更新依赖包版本管理
```
This commit is contained in:
link2026
2026-07-01 08:03:35 +08:00
parent 30f75dc2cd
commit e179a369f6
74 changed files with 3417 additions and 146 deletions

View File

@@ -174,28 +174,56 @@ actor CaptureService {
return s
}
/// VL + JSON assets Vault
/// + JSON assets Vault
///
/// hybrid :
/// - **(Gemini)**: Gemini (·VL, source_box ),
/// OCR Gemma-3n(MLX ,)
/// - **线/**:退 Vision OCR(,<1s/) Gemma LLM meta+
/// (线//)退,(§3.2 退线)
private func runVL(on assets: [FileVault.SavedAsset]) async throws -> ParsedReport {
let urls = assets.map { FileVault.shared.rootURL.appendingPathComponent($0.relativePath) }
// Vision OCR (/ ,)
let ocr = await Self.ocrReference(for: urls)
// :Gemini OCR , Gemini , OCR
if AIRuntime.shared.cloudAvailable {
do {
let raw = try await AIRuntime.shared.analyzeReportCloud(
imageURLs: urls,
prompt: VLPrompts.reportExtraction(ocrText: ocr),
maxTokens: 2048
)
return try CaptureService.parseReportJSON(
CaptureService.stripThink(raw), pageCount: assets.count)
} catch {
// 退, /
}
}
// 退: OCR ( Gemma )
guard !ocr.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
throw CaptureError.inferenceFailed(String(appLoc: "未识别到文字,无法解读"))
}
do {
try await AIRuntime.shared.prepareVL()
try await AIRuntime.shared.prepare() // LLM(OOM )
} catch {
throw CaptureError.modelNotReady
}
let urls = assets.map { FileVault.shared.rootURL.appendingPathComponent($0.relativePath) }
// OCR (Vision ,<1s/): 2B ,
// 退,(§3.2)
let ocr = await Self.ocrReference(for: urls)
let raw: String
var collected = ""
do {
raw = try await AIRuntime.shared.analyzeReport(
imageURLs: urls,
prompt: VLPrompts.reportExtraction(ocrText: ocr)
// , token; VL/ AIRuntime
let stream = await AIRuntime.shared.generate(
prompt: VLPrompts.reportExtractionFromText(ocr),
maxTokens: 2048
)
for try await chunk in stream { collected += chunk.text }
} catch {
throw CaptureError.inferenceFailed("\(error)")
}
let cleaned = CaptureService.stripThink(collected)
do {
return try CaptureService.parseReportJSON(raw, pageCount: assets.count)
return try CaptureService.parseReportJSON(cleaned, pageCount: assets.count)
} catch let CaptureError.parseFailed(msg) {
throw CaptureError.parseFailed(msg)
} catch {

View File

@@ -0,0 +1,206 @@
import Foundation
import Speech
import AVFoundation
/// + (2026-06-28)
///
/// `SpeechDictationService`(,****):,
/// ** AVAudioEngine tap** :
/// `request.append(buffer)` SFSpeech (,`requiresOnDeviceRecognition = true`,线:);
/// `audioFile.write(buffer)` m4a(AAC) tmp, `FileVault.importFile` Vault
///
/// ****: `try?` ,
/// 稿 SFSpeech ,,(线 #5:退,)
///
/// MainActor , MainActor;tap 线,
/// ,线 `Task { @MainActor }`( SpeechDictationService)
final class ConsultationRecorder {
enum RecorderError: Error, LocalizedError {
case unavailable
case audioEngineStartFailed(String)
var errorDescription: String? {
switch self {
case .unavailable:
return String(appLoc: "本机不支持端侧语音识别")
case .audioEngineStartFailed(let m):
return String(appLoc: "录音启动失败:\(m)")
}
}
}
/// :稿 + ( nil,)
struct Result {
let transcript: String
/// tmp m4a; `FileVault.importFile` Vault
let audioTempURL: URL?
}
/// ;( SpeechDictationService)
private static func makeRecognizer() -> SFSpeechRecognizer? {
if let r = SFSpeechRecognizer(locale: .current), r.supportsOnDeviceRecognition {
return r
}
if let r = SFSpeechRecognizer(locale: Locale(identifier: "zh-CN")),
r.supportsOnDeviceRecognition {
return r
}
return nil
}
/// false( / ) UI 退
static var isAvailable: Bool { makeRecognizer() != nil }
private let audioEngine = AVAudioEngine()
private var request: SFSpeechAudioBufferRecognitionRequest?
private var task: SFSpeechRecognitionTask?
/// buffer , tap ()
private var audioFile: AVAudioFile?
private var audioTempURL: URL?
private var latestText = ""
private var didFinish = false
private(set) var isRecording = false
/// + false
func requestAuthorization() async -> Bool {
let speech = await withCheckedContinuation { (c: CheckedContinuation<SFSpeechRecognizerAuthorizationStatus, Never>) in
SFSpeechRecognizer.requestAuthorization { c.resume(returning: $0) }
}
guard speech == .authorized else { return false }
return await AVAudioApplication.requestRecordPermission()
}
/// + partial 线()
func start(onPartial: @escaping (String) -> Void) throws {
guard !isRecording else { return }
guard let recognizer = Self.makeRecognizer(), recognizer.isAvailable else {
throw RecorderError.unavailable
}
let session = AVAudioSession.sharedInstance()
do {
// .record + ( .measurement):,
try session.setCategory(.record, mode: .default, options: .duckOthers)
try session.setActive(true, options: .notifyOthersOnDeactivation)
} catch {
throw RecorderError.audioEngineStartFailed(error.localizedDescription)
}
let request = SFSpeechAudioBufferRecognitionRequest()
request.requiresOnDeviceRecognition = true // 线:
request.shouldReportPartialResults = true
request.addsPunctuation = true
self.request = request
latestText = ""
didFinish = false
audioFile = nil
// tmp(,); Vault
let tempURL = FileManager.default.temporaryDirectory
.appendingPathComponent("consult-\(UUID().uuidString).m4a")
self.audioTempURL = tempURL
let input = audioEngine.inputNode
let format = input.outputFormat(forBus: 0)
// tap tap :AAC processingFormat = / float ,
// tap buffer write (try? nil),()
let settings: [String: Any] = [
AVFormatIDKey: kAudioFormatMPEG4AAC,
AVSampleRateKey: format.sampleRate,
AVNumberOfChannelsKey: format.channelCount,
AVEncoderAudioQualityKey: AVAudioQuality.medium.rawValue,
]
let audioFile = try? AVAudioFile(forWriting: tempURL, settings: settings)
self.audioFile = audioFile // stop() flush
// tap 线:**** request / audioFile, self(线, SpeechDictationService)
input.installTap(onBus: 0, bufferSize: 1024, format: format) { buffer, _ in
request.append(buffer)
try? audioFile?.write(from: buffer)
}
audioEngine.prepare()
do {
try audioEngine.start()
} catch {
input.removeTap(onBus: 0)
deactivateSession()
throw RecorderError.audioEngineStartFailed(error.localizedDescription)
}
task = recognizer.recognitionTask(with: request) { [weak self] result, error in
Task { @MainActor in
guard let self else { return }
if let result {
self.latestText = result.bestTranscription.formattedString
onPartial(self.latestText)
if result.isFinal { self.didFinish = true }
}
if error != nil { self.didFinish = true }
}
}
isRecording = true
}
/// ,( 1.5s, partial),稿 +
func stop() async -> Result {
guard isRecording else {
return Result(transcript: latestText, audioTempURL: finishedAudioURL())
}
isRecording = false
audioEngine.stop()
audioEngine.inputNode.removeTap(onBus: 0)
request?.endAudio()
let deadline = Date().addingTimeInterval(1.5)
while !didFinish && Date() < deadline {
try? await Task.sleep(nanoseconds: 100_000_000)
}
task?.cancel()
task = nil
request = nil
let url = finishedAudioURL() // ( nil) URL, flush
deactivateSession()
return Result(transcript: latestText, audioTempURL: url)
}
/// :,;
func abort() {
guard isRecording else {
cleanupTempFile()
return
}
isRecording = false
audioEngine.stop()
audioEngine.inputNode.removeTap(onBus: 0)
request?.endAudio()
task?.cancel()
task = nil
request = nil
audioFile = nil
deactivateSession()
cleanupTempFile()
}
/// (flush), URL;/ nil
private func finishedAudioURL() -> URL? {
audioFile = nil // , flush
guard let url = audioTempURL,
let attrs = try? FileManager.default.attributesOfItem(atPath: url.path),
let bytes = attrs[.size] as? Int, bytes > 0 else {
return nil
}
return url
}
private func cleanupTempFile() {
if let url = audioTempURL { try? FileManager.default.removeItem(at: url) }
audioTempURL = nil
}
private func deactivateSession() {
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
}
}

View File

@@ -201,4 +201,29 @@ struct DiaryAssistService {
guard !text.isEmpty else { throw AssistError.empty }
return (text, lastRate)
}
/// 稿(2026-06-28, ConsultationPrompts)
/// organize AIRuntime actor 退(), prompt/
/// maxTokens 700:,
func organizeConsultation(transcript: String) async throws -> (text: String, decodeRate: Double) {
do {
try await AIRuntime.shared.prepare()
} catch {
throw AssistError.modelNotReady
}
let prompt = ConsultationPrompts.organize(transcript: transcript)
var collected = ""
var lastRate: Double = 0
let stream = await AIRuntime.shared.generate(prompt: prompt, maxTokens: 700)
for try await chunk in stream {
collected += chunk.text
if chunk.decodeRate > 0 { lastRate = chunk.decodeRate }
}
let text = HealthExportService.stripThinkBlocks(collected)
.trimmingCharacters(in: .whitespacesAndNewlines)
guard !text.isEmpty else { throw AssistError.empty }
return (text, lastRate)
}
}

View File

@@ -647,13 +647,21 @@ struct HealthExportService {
return d
}
// diaries
// diaries(:tag )
// kind ,
root["diaries"] = snapshot.diaries.map { d -> [String: Any] in
let excerpt = String(d.content.prefix(80))
return [
let limit = d.isConsultation ? 240 : 80
let excerpt = String(d.content.prefix(limit))
var item: [String: Any] = [
"date": df.string(from: d.createdAt),
"excerpt": excerpt
]
if d.isConsultation {
item["kind"] = "问诊"
} else if d.isMedicationLog {
item["kind"] = "用药"
}
return item
}
// LLM

View File

@@ -0,0 +1,169 @@
import Foundation
import AVFoundation
/// (SenseVoice, sherpa-mnn MNN )
///
/// **线**(,),
/// 稿 LLM(DiaryAssistService.organizeConsultation)
///
/// 线:
/// - ,(§1 / §10 #1)
/// - LLM/VL `AIRuntime.runExclusiveForASR` ****:/,
/// App jetsam (§3.1 OOM , [[airuntime-llm-vl-oom-gate]])
/// - ,退(SFSpeech),App (§10 #5)
///
/// LLM `ModelKind` , `Models/SenseVoice/` (model.mnn + tokens.txt)
/// / docs/release/sensevoice-integration.md
struct SenseVoiceASRService {
static let shared = SenseVoiceASRService()
private init() {}
enum ASRError: Error, LocalizedError {
case modelNotInstalled
case engineUnavailable
case decodeFailed
case empty
var errorDescription: String? {
switch self {
case .modelNotInstalled: return String(appLoc: "问诊转写模型未就绪")
case .engineUnavailable: return String(appLoc: "本机暂不支持本地语音转写")
case .decodeFailed: return String(appLoc: "录音解码失败")
case .empty: return String(appLoc: "没识别到语音内容")
}
}
}
// MARK: - ( LLM ModelKind, ASR )
/// `Application Support/Models/SenseVoice/`, LLM
nonisolated static var modelDir: URL {
ModelStore.shared.rootURL.appendingPathComponent("SenseVoice", isDirectory: true)
}
/// MNNConvert SenseVoice
nonisolated static var modelFile: URL { modelDir.appendingPathComponent("model.mnn") }
/// tokens.txt(idtoken )
nonisolated static var tokensFile: URL { modelDir.appendingPathComponent("tokens.txt") }
/// ( / )
nonisolated static var isModelInstalled: Bool {
let fm = FileManager.default
return fm.fileExists(atPath: modelFile.path) && fm.fileExists(atPath: tokensFile.path)
}
/// SenseVoice = (sherpa-mnn)****
/// false 退(SFSpeech)
nonisolated static var isAvailable: Bool {
SenseVoiceBridge.isAvailable() && isModelInstalled
}
// MARK: -
/// (m4a/wav/caf )线,退
/// language:"auto" (), "zh"
func transcribe(audioFileURL: URL, language: String = "auto") async throws -> String {
guard SenseVoiceBridge.isAvailable() else { throw ASRError.engineUnavailable }
guard Self.isModelInstalled else { throw ASRError.modelNotInstalled }
let modelPath = Self.modelFile.path
let tokensPath = Self.tokensFile.path
// + (CPU),线: LLM/VL ,
let raw = try await AIRuntime.shared.runExclusiveForASR {
try await Self.runOnBackground {
let decoded = try Self.decodeToMonoFloat(url: audioFileURL)
guard !decoded.samples.isEmpty else { throw ASRError.decodeFailed }
guard let bridge = SenseVoiceBridge(modelPath: modelPath,
tokensPath: tokensPath,
language: language) else {
throw ASRError.engineUnavailable
}
let text = decoded.samples.withUnsafeBufferPointer { buf -> String? in
guard let base = buf.baseAddress else { return nil }
return bridge.transcribeSamples(base,
count: Int32(buf.count),
sampleRate: Int32(decoded.sampleRate))
}
return text ?? ""
}
}
let cleaned = Self.cleanTranscript(raw)
guard !cleaned.isEmpty else { throw ASRError.empty }
return cleaned
}
// MARK: - ()
/// SenseVoice : `<|zh|><|NEUTRAL|><|Speech|><|woitn|>` trim
/// sherpa ,, LLM 稿
nonisolated static func cleanTranscript(_ raw: String) -> String {
let stripped = raw.replacingOccurrences(
of: "<\\|[^|]*\\|>", with: "", options: .regularExpression)
return stripped.trimmingCharacters(in: .whitespacesAndNewlines)
}
// MARK: - ( 16kHz float32)
private struct Decoded { let samples: [Float]; let sampleRate: Int }
/// AVAudioConverter / 16kHz float32(SenseVoice )
/// :LLM ,
nonisolated private static func decodeToMonoFloat(url: URL,
targetSampleRate: Double = 16000) throws -> Decoded {
let file = try AVAudioFile(forReading: url)
let inFormat = file.processingFormat
let frameCount = AVAudioFrameCount(file.length)
guard frameCount > 0 else { return Decoded(samples: [], sampleRate: Int(targetSampleRate)) }
guard let targetFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32,
sampleRate: targetSampleRate,
channels: 1,
interleaved: false),
let converter = AVAudioConverter(from: inFormat, to: targetFormat),
let inBuffer = AVAudioPCMBuffer(pcmFormat: inFormat, frameCapacity: frameCount) else {
throw ASRError.decodeFailed
}
try file.read(into: inBuffer)
// (48k/44.1k16k),; 16k ratio>1, +
let ratio = targetSampleRate / inFormat.sampleRate
let outCapacity = AVAudioFrameCount(Double(frameCount) * ratio) + 1024
guard let outBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: outCapacity) else {
throw ASRError.decodeFailed
}
var fed = false
var convError: NSError?
let status = converter.convert(to: outBuffer, error: &convError) { _, inStatus in
if fed {
inStatus.pointee = .endOfStream // , flush
return nil
}
fed = true
inStatus.pointee = .haveData
return inBuffer
}
if let convError { throw convError }
guard status != .error else { throw ASRError.decodeFailed }
guard let channel = outBuffer.floatChannelData?[0] else {
return Decoded(samples: [], sampleRate: Int(targetSampleRate))
}
let n = Int(outBuffer.frameLength)
let samples = Array(UnsafeBufferPointer(start: channel, count: n))
return Decoded(samples: samples, sampleRate: Int(targetSampleRate))
}
/// QoS (, actor/线)
nonisolated private static func runOnBackground<T: Sendable>(
_ work: @escaping @Sendable () throws -> T
) async throws -> T {
try await withCheckedThrowingContinuation { cont in
DispatchQueue.global(qos: .userInitiated).async {
do { cont.resume(returning: try work()) }
catch { cont.resume(throwing: error) }
}
}
}
}