```
feat(AI): 集成MNN推理引擎替换MLX作为主AI运行时 - 引入MNN(alibaba) + Arm SME2 + CPU作为主AI运行时,支持A19/iPhone17的 SME2和A17的NEON加速 - 添加MLX Swift作为兜底GPU推理方案,实现双后端切换机制 - 使用单一Qwen3.5-2B多模态模型(1.2GB),替代原有的LLM+VL分离架构 - 实现InferenceEngine.current引擎选择逻辑,真机默认MNN,模拟器回退MLX - 更新AIAgent架构,通过MNNLLMBridge(ObjC++) → MNNBackend进行推理 - 修改队列机制防止并发推理导致OOM,使用信号量闸门控制显存占用 - 更新文档中的技术栈说明、模块边界和周次交付计划 ```
This commit is contained in:
@@ -33,7 +33,9 @@ struct ParsedReport: Sendable {
|
||||
var isEmpty: Bool { indicators.isEmpty }
|
||||
|
||||
/// 占位空结果,失败回退时给 UI。
|
||||
static func empty(date: Date = .now) -> ParsedReport {
|
||||
/// nonisolated:本工程默认 MainActor 隔离,而 CaptureService(actor)里的 extractReportMeta
|
||||
/// 需要在 actor 上下文构造空结果 —— 纯值工厂,标 nonisolated 才能跨隔离调用(Swift 6)。
|
||||
nonisolated static func empty(date: Date = .now) -> ParsedReport {
|
||||
ParsedReport(
|
||||
title: "",
|
||||
typeRaw: ReportType.other.rawValue,
|
||||
@@ -78,6 +80,40 @@ actor CaptureService {
|
||||
try await runVL(on: assets)
|
||||
}
|
||||
|
||||
/// 报告归档「轻量 meta 提取」:**只保存原图,不逐项识别指标**。
|
||||
/// 逐项多模态识别在 2B 上又慢又易 OOM(jetsam 杀进程 = 用户说的「死机」),
|
||||
/// 故归档链路改为:Vision OCR(本地,<1s/页)→ 文本 LLM 只抽 {title,type,date,institution}(~50 token)。
|
||||
/// 全程容错:OCR 空 / 模型未就绪 / 解析失败都返回 (空 meta, recognized:false),绝不抛、绝不阻断保存原图(§3.2)。
|
||||
/// 返回的 indicators 恒为空 —— 归档不建指标。
|
||||
func extractReportMeta(assets: [FileVault.SavedAsset]) async -> (meta: ParsedReport, recognized: Bool) {
|
||||
let urls = assets.map { FileVault.shared.rootURL.appendingPathComponent($0.relativePath) }
|
||||
let ocr = await Self.ocrReference(for: urls)
|
||||
guard !ocr.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
|
||||
return (.empty(), false)
|
||||
}
|
||||
do {
|
||||
try await AIRuntime.shared.prepare() // 文本 LLM(轻);OOM 闸门已处理 VL 卸载
|
||||
} catch {
|
||||
return (.empty(), false)
|
||||
}
|
||||
var collected = ""
|
||||
do {
|
||||
// meta 输出极小,256 token 足够,远小于逐项识别的 2048 —— 这是不卡死的关键。
|
||||
let stream = await AIRuntime.shared.generate(prompt: VLPrompts.reportMetaFromText(ocr),
|
||||
maxTokens: 256)
|
||||
for try await chunk in stream { collected += chunk.text }
|
||||
} catch {
|
||||
return (.empty(), false)
|
||||
}
|
||||
let cleaned = CaptureService.stripThink(collected)
|
||||
guard var parsed = try? CaptureService.parseReportJSON(cleaned, pageCount: assets.count) else {
|
||||
return (.empty(), false)
|
||||
}
|
||||
// 归档只存 meta + 原图,丢弃模型可能附带的任何指标。
|
||||
parsed.indicators = []
|
||||
return (parsed, true)
|
||||
}
|
||||
|
||||
/// 「拍照识别」OCR 链路:把 Vision OCR 出的纯文本交给 LLM(Qwen3-1.7B)结构化抽指标。
|
||||
/// 不建 Report、不留图;失败抛 `CaptureError`,UI 回退手动录入(§3.2)。
|
||||
/// 调用方(MainActor)先做 OCR,再把文本传进来——OCR 不需进 actor,也避免 UIImage 跨 actor。
|
||||
@@ -169,8 +205,17 @@ actor CaptureService {
|
||||
private static func ocrReference(for urls: [URL]) async -> String {
|
||||
var pages: [String] = []
|
||||
for (idx, url) in urls.prefix(4).enumerated() {
|
||||
guard let src = CGImageSourceCreateWithURL(url as CFURL, nil),
|
||||
let cg = CGImageSourceCreateImageAtIndex(src, 0, nil) else { continue }
|
||||
guard let src = CGImageSourceCreateWithURL(url as CFURL, nil) else { continue }
|
||||
// OCR 不需要全分辨率:一张 4000px 体检照全量解码 ≈48MB,正赶在 VL 推理前叠加,
|
||||
// 易触发 jetsam。降到 ≤3000px 既省内存又加速 Vision,医检报告字号此分辨率仍清晰;
|
||||
// 且原图仍完整交给 VL 自行读取,OCR 仅当数字「抄写员」辅助,降采样不影响最终可用信息。
|
||||
let thumbOptions: [CFString: Any] = [
|
||||
kCGImageSourceCreateThumbnailFromImageAlways: true,
|
||||
kCGImageSourceCreateThumbnailWithTransform: true,
|
||||
kCGImageSourceShouldCacheImmediately: true,
|
||||
kCGImageSourceThumbnailMaxPixelSize: 3000
|
||||
]
|
||||
guard let cg = CGImageSourceCreateThumbnailAtIndex(src, 0, thumbOptions as CFDictionary) else { continue }
|
||||
guard let text = try? await OCRService.recognizeText(in: cg),
|
||||
!text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { continue }
|
||||
pages.append(urls.count > 1 ? "【第 \(idx + 1) 页】\n\(text)" : text)
|
||||
|
||||
@@ -450,6 +450,8 @@ struct HealthExportService {
|
||||
var reports: [Report]
|
||||
var diaries: [DiaryEntry]
|
||||
var profile: UserProfile
|
||||
/// 药品库(用户「我有哪些药」清单)→ AI 背景 current_meds。空 → 不写该字段。
|
||||
var medications: [Medication] = []
|
||||
/// 相关指标的趋势行(确定性计算,不进 LLM)。空 → 不渲染「## 指标趋势」段。
|
||||
var trends: [ExportTrend] = []
|
||||
}
|
||||
@@ -530,6 +532,9 @@ struct HealthExportService {
|
||||
// —— Profile(单例) ——
|
||||
let profile = UserProfileStore.loadOrCreate(in: ctx)
|
||||
|
||||
// —— 药品库(全量,作为 AI 背景 current_meds) ——
|
||||
let medications = (try? ctx.fetch(FetchDescriptor<Medication>())) ?? []
|
||||
|
||||
// —— 趋势(确定性,不进 LLM) ——
|
||||
// 用全量 in-window 还原完整序列;裁剪后的 indicators 决定哪些 series 相关。
|
||||
let trends = ExportTrendBuilder.build(
|
||||
@@ -546,6 +551,7 @@ struct HealthExportService {
|
||||
reports: reports,
|
||||
diaries: diaries,
|
||||
profile: profile,
|
||||
medications: medications,
|
||||
trends: trends
|
||||
)
|
||||
}
|
||||
@@ -561,6 +567,7 @@ struct HealthExportService {
|
||||
let indicators = (try? ctx.fetch(indicatorDesc)) ?? []
|
||||
let diaries = (try? ctx.fetch(diaryDesc)) ?? []
|
||||
let profile = UserProfileStore.loadOrCreate(in: ctx)
|
||||
let medications = (try? ctx.fetch(FetchDescriptor<Medication>())) ?? []
|
||||
|
||||
let dates = indicators.map(\.capturedAt) + diaries.map(\.createdAt)
|
||||
let fromDate = dates.min() ?? Date()
|
||||
@@ -581,6 +588,7 @@ struct HealthExportService {
|
||||
reports: [],
|
||||
diaries: diaries,
|
||||
profile: profile,
|
||||
medications: medications,
|
||||
trends: trends
|
||||
)
|
||||
}
|
||||
@@ -611,7 +619,11 @@ struct HealthExportService {
|
||||
if !profile.allergies.isEmpty { profDict["allergies"] = profile.allergies }
|
||||
if !profile.chronicConditions.isEmpty { profDict["chronic"] = profile.chronicConditions }
|
||||
if !profile.familyHistory.isEmpty { profDict["family_history"] = profile.familyHistory }
|
||||
if !profile.currentMedications.isEmpty { profDict["current_meds"] = profile.currentMedications }
|
||||
// current_meds 改读药品库(Medication);旧 profile.currentMedications 已停用。
|
||||
let medNames = snapshot.medications.map { m in
|
||||
m.detailLine.isEmpty ? m.name : "\(m.name) \(m.detailLine)"
|
||||
}
|
||||
if !medNames.isEmpty { profDict["current_meds"] = medNames }
|
||||
root["profile"] = profDict
|
||||
|
||||
// symptoms
|
||||
@@ -681,7 +693,8 @@ struct HealthExportService {
|
||||
/// 检索结果是否「实质为空」:无症状/指标/报告/日记,且 profile 也没有任何可写字段。
|
||||
/// 为真时跳过 LLM,改用确定性「无记录」摘要,避免小模型凭先验编造病例。
|
||||
static func isEffectivelyEmpty(_ s: Snapshot) -> Bool {
|
||||
guard s.symptoms.isEmpty, s.indicators.isEmpty, s.reports.isEmpty, s.diaries.isEmpty else {
|
||||
guard s.symptoms.isEmpty, s.indicators.isEmpty, s.reports.isEmpty,
|
||||
s.diaries.isEmpty, s.medications.isEmpty else {
|
||||
return false
|
||||
}
|
||||
let p = s.profile
|
||||
@@ -693,7 +706,6 @@ struct HealthExportService {
|
||||
&& p.allergies.isEmpty
|
||||
&& p.chronicConditions.isEmpty
|
||||
&& p.familyHistory.isEmpty
|
||||
&& p.currentMedications.isEmpty
|
||||
}
|
||||
|
||||
/// 无真实记录时的确定性摘要:6 段全「无记录」,主诉仅照搬本人原话,不做任何推断。
|
||||
|
||||
@@ -80,20 +80,24 @@ enum ReminderService {
|
||||
let title = reminder.title.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let body = reminder.note.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let h = reminder.hour, m = reminder.minute
|
||||
let slots: [Slot]
|
||||
switch reminder.frequency {
|
||||
case .daily:
|
||||
slots = [Slot(suffix: "daily", dc: DateComponents(hour: h, minute: m))]
|
||||
case .weekly:
|
||||
slots = reminder.weekdays.map { wd in
|
||||
Slot(suffix: "w\(wd)", dc: DateComponents(hour: h, minute: m, weekday: wd))
|
||||
// 多选频率:把每个选中频率展开成槽,合并调度(suffix 各不冲突,可单独取消)。
|
||||
var slots: [Slot] = []
|
||||
for f in reminder.frequencies {
|
||||
switch f {
|
||||
case .daily:
|
||||
slots.append(Slot(suffix: "daily", dc: DateComponents(hour: h, minute: m)))
|
||||
case .weekly:
|
||||
slots += reminder.weekdays.map { wd in
|
||||
Slot(suffix: "w\(wd)", dc: DateComponents(hour: h, minute: m, weekday: wd))
|
||||
}
|
||||
case .monthly:
|
||||
slots += reminder.monthlyDays.map { d in
|
||||
Slot(suffix: "m\(d)", dc: DateComponents(day: d, hour: h, minute: m))
|
||||
}
|
||||
case .yearly:
|
||||
slots.append(Slot(suffix: "yearly",
|
||||
dc: DateComponents(month: reminder.month, day: reminder.dayOfMonth, hour: h, minute: m)))
|
||||
}
|
||||
case .monthly:
|
||||
slots = [Slot(suffix: "monthly",
|
||||
dc: DateComponents(day: reminder.dayOfMonth, hour: h, minute: m))]
|
||||
case .yearly:
|
||||
slots = [Slot(suffix: "yearly",
|
||||
dc: DateComponents(month: reminder.month, day: reminder.dayOfMonth, hour: h, minute: m))]
|
||||
}
|
||||
await schedule(
|
||||
idBase: "\(customIdPrefix)\(reminder.id.uuidString)",
|
||||
@@ -146,11 +150,13 @@ enum ReminderService {
|
||||
}
|
||||
}
|
||||
|
||||
/// 取消某个 idBase 下所有可能后缀的 pending 通知(daily/monthly/yearly + 7 个 weekday,不漏)。
|
||||
/// 取消某个 idBase 下所有可能后缀的 pending 通知,不漏:
|
||||
/// daily / yearly / 旧版 monthly + 7 个 weekday(w1...w7)+ 31 个月内日(m1...m31)。
|
||||
private static func cancelBase(_ idBase: String) {
|
||||
let center = UNUserNotificationCenter.current()
|
||||
var ids = ["\(idBase).daily", "\(idBase).monthly", "\(idBase).yearly"]
|
||||
ids += (1...7).map { "\(idBase).w\($0)" }
|
||||
ids += (1...31).map { "\(idBase).m\($0)" }
|
||||
center.removePendingNotificationRequests(withIdentifiers: ids)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,16 @@ final class SpeechDictationService {
|
||||
}
|
||||
}
|
||||
|
||||
/// 把已有文字 `prefix` 与新听写片段 `partial` 合并成一段。纯函数,方便单测、与录音生命周期解耦。
|
||||
/// 规则:空片段保留原文;空前缀直接用片段;前缀已以空白(空格/换行)结尾则直接拼,
|
||||
/// 否则中间补一个空格——避免「已有内容新听写」黏成一坨,也不会在换行后多塞空格。
|
||||
static func merge(prefix: String, partial: String) -> String {
|
||||
if partial.isEmpty { return prefix }
|
||||
if prefix.isEmpty { return partial }
|
||||
if prefix.last?.isWhitespace == true { return prefix + partial }
|
||||
return prefix + " " + partial
|
||||
}
|
||||
|
||||
/// 优先系统语言;系统语言不支持端侧时兜底中文(demo 机即使系统是英文也能用)。
|
||||
private static func makeRecognizer() -> SFSpeechRecognizer? {
|
||||
if let r = SFSpeechRecognizer(locale: .current), r.supportsOnDeviceRecognition {
|
||||
|
||||
Reference in New Issue
Block a user