```
feat(AI): 集成MNN推理引擎替换MLX作为主AI运行时 - 引入MNN(alibaba) + Arm SME2 + CPU作为主AI运行时,支持A19/iPhone17的 SME2和A17的NEON加速 - 添加MLX Swift作为兜底GPU推理方案,实现双后端切换机制 - 使用单一Qwen3.5-2B多模态模型(1.2GB),替代原有的LLM+VL分离架构 - 实现InferenceEngine.current引擎选择逻辑,真机默认MNN,模拟器回退MLX - 更新AIAgent架构,通过MNNLLMBridge(ObjC++) → MNNBackend进行推理 - 修改队列机制防止并发推理导致OOM,使用信号量闸门控制显存占用 - 更新文档中的技术栈说明、模块边界和周次交付计划 ```
This commit is contained in:
@@ -8,9 +8,12 @@ struct CaptureReviewForm: View {
|
||||
@State var parsed: ParsedReport
|
||||
let assets: [FileVault.SavedAsset]
|
||||
let warning: String?
|
||||
/// 归档模式:只存原图 + 基本信息(标题/类型/日期/机构),隐藏指标区与摘要。
|
||||
/// 报告归档不再逐项识别(逐项多模态在 2B 上易 OOM 卡死),见 CaptureService.extractReportMeta。
|
||||
var metaOnly: Bool = false
|
||||
let onSave: (ParsedReport) -> Void
|
||||
let onCancel: () -> Void
|
||||
/// 「重新识别」回调。assets 为空(写图失败)时传 nil,banner 上不显示该按钮。
|
||||
/// 「重新识别信息」回调。assets 为空(写图失败)时传 nil,banner 上不显示该按钮。
|
||||
var onReanalyze: (() -> Void)? = nil
|
||||
|
||||
var body: some View {
|
||||
@@ -23,7 +26,9 @@ struct CaptureReviewForm: View {
|
||||
pageThumbnails
|
||||
}
|
||||
metaSection
|
||||
indicatorSection
|
||||
if !metaOnly {
|
||||
indicatorSection
|
||||
}
|
||||
Spacer(minLength: 8)
|
||||
actions
|
||||
}
|
||||
@@ -68,20 +73,26 @@ struct CaptureReviewForm: View {
|
||||
private var pageThumbnails: some View {
|
||||
VStack(alignment: .leading, spacing: 8) {
|
||||
sectionLabel(String(appLoc: "已保存 \(assets.count) 页(端侧加密)"))
|
||||
if metaOnly {
|
||||
Text("原图已加密保存,详情页随时可翻看放大。系统只识别报告日期与机构作为标签,不逐项录入数值。")
|
||||
.font(.tjScaled( 11))
|
||||
.foregroundStyle(Tj.Palette.text3)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
}
|
||||
ScrollView(.horizontal, showsIndicators: false) {
|
||||
HStack(spacing: 10) {
|
||||
ForEach(Array(assets.enumerated()), id: \.offset) { _, asset in
|
||||
if let img = try? FileVault.shared.loadImage(relativePath: asset.relativePath) {
|
||||
Image(uiImage: img)
|
||||
.resizable()
|
||||
.scaledToFill()
|
||||
.frame(width: 84, height: 110)
|
||||
.clipShape(RoundedRectangle(cornerRadius: 8, style: .continuous))
|
||||
.overlay(
|
||||
RoundedRectangle(cornerRadius: 8, style: .continuous)
|
||||
.strokeBorder(Tj.Palette.line, lineWidth: 1)
|
||||
)
|
||||
VaultImage(relativePath: asset.relativePath, maxPixel: 400) { img in
|
||||
Image(uiImage: img).resizable().scaledToFill()
|
||||
} placeholder: { _ in
|
||||
Tj.Palette.paper
|
||||
}
|
||||
.frame(width: 84, height: 110)
|
||||
.clipShape(RoundedRectangle(cornerRadius: 8, style: .continuous))
|
||||
.overlay(
|
||||
RoundedRectangle(cornerRadius: 8, style: .continuous)
|
||||
.strokeBorder(Tj.Palette.line, lineWidth: 1)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -117,9 +128,11 @@ struct CaptureReviewForm: View {
|
||||
labeledField(String(appLoc: "机构(可选)")) {
|
||||
TextField("如:协和医院", text: $parsed.institution)
|
||||
}
|
||||
labeledField(String(appLoc: "摘要(可选)")) {
|
||||
TextField("一句话总结", text: $parsed.summary, axis: .vertical)
|
||||
.lineLimit(1...3)
|
||||
if !metaOnly {
|
||||
labeledField(String(appLoc: "摘要(可选)")) {
|
||||
TextField("一句话总结", text: $parsed.summary, axis: .vertical)
|
||||
.lineLimit(1...3)
|
||||
}
|
||||
}
|
||||
}
|
||||
.padding(12)
|
||||
|
||||
@@ -62,7 +62,7 @@ struct UnifiedCaptureFlow: View {
|
||||
switch phase {
|
||||
case .idle: return String(appLoc: "拍摄报告")
|
||||
case .analyzing: return String(appLoc: "本地识别中…")
|
||||
case .editing: return String(appLoc: "核对识别结果")
|
||||
case .editing: return String(appLoc: "核对报告信息")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,6 +86,7 @@ struct UnifiedCaptureFlow: View {
|
||||
parsed: parsed,
|
||||
assets: assets,
|
||||
warning: warning,
|
||||
metaOnly: true, // 归档只存原图 + meta,不逐项识别(§见 CaptureService.extractReportMeta)
|
||||
onSave: { final in saveAll(parsed: final, assets: assets) },
|
||||
onCancel: cancelAll,
|
||||
onReanalyze: assets.isEmpty ? nil : { reanalyze(assets: assets) }
|
||||
@@ -152,9 +153,7 @@ struct UnifiedCaptureFlow: View {
|
||||
phase = .analyzing(images: images, assets: nil)
|
||||
let timeout = analyzeTimeoutSeconds
|
||||
analyzeTask = Task {
|
||||
// Step 1: 先把图写进 Vault。
|
||||
// 在 UI 这一层写,而不是塞进 CaptureService.analyze —— 这样取消/失败回退时,
|
||||
// assets 已经在 phase 里,cancelAll 能清理孤儿,editingFallback 也不必再补写。
|
||||
// Step 1: 先把图写进 Vault(归档的核心价值就是「把原图存下来」,先保证它)。
|
||||
let assets = images.compactMap { try? FileVault.shared.writeJPEG($0) }
|
||||
// 极端情况:用户在写图过程中按了「取消」,View 已 dismiss、cancelAll 看到的
|
||||
// phase 还是 .analyzing(_, nil),清不到这批刚写完的图 — 这里手动收尾。
|
||||
@@ -167,7 +166,7 @@ struct UnifiedCaptureFlow: View {
|
||||
phase = .editing(
|
||||
parsed: .empty(),
|
||||
assets: [],
|
||||
warning: String(appLoc: "图片保存失败,手动录入并保留文本")
|
||||
warning: String(appLoc: "图片保存失败,请重试")
|
||||
)
|
||||
}
|
||||
return
|
||||
@@ -179,49 +178,40 @@ struct UnifiedCaptureFlow: View {
|
||||
}
|
||||
}
|
||||
|
||||
// Step 2: VL 推理(timeout 哨兵到点 cancel 父任务,VLSession 在下一个 token break)。
|
||||
// Step 2: 轻量 meta 提取(OCR + 文本 LLM,只抽日期/机构/类型/标题)。
|
||||
// 不再跑多模态逐项识别 —— 那在 2B 上又慢又会 OOM 卡死。watchdog 到点 cancel。
|
||||
let watchdog = Task {
|
||||
try? await Task.sleep(for: .seconds(timeout))
|
||||
analyzeTask?.cancel()
|
||||
}
|
||||
defer { watchdog.cancel() }
|
||||
|
||||
do {
|
||||
let parsed = try await CaptureService.shared.reanalyze(assets: assets)
|
||||
if Task.isCancelled {
|
||||
await editingFallback(assets: assets,
|
||||
msg: String(appLoc: "识别超时(>\(timeout)s),先手动录入"))
|
||||
return
|
||||
}
|
||||
let (meta, recognized) = await CaptureService.shared.extractReportMeta(assets: assets)
|
||||
if Task.isCancelled {
|
||||
await MainActor.run {
|
||||
phase = .editing(
|
||||
parsed: parsed,
|
||||
assets: assets,
|
||||
warning: parsed.isEmpty ? String(appLoc: "识别没有读出指标,请手动补充") : nil
|
||||
)
|
||||
phase = .editing(parsed: .empty(), assets: assets,
|
||||
warning: String(appLoc: "识别超时,已保存原图,请手动填写信息"))
|
||||
}
|
||||
} catch let CaptureError.parseFailed(msg) {
|
||||
await editingFallback(assets: assets, msg: String(appLoc: "VL 输出无法解析:\(msg)"))
|
||||
} catch let CaptureError.inferenceFailed(msg) {
|
||||
await editingFallback(assets: assets,
|
||||
msg: Task.isCancelled
|
||||
? String(appLoc: "识别超时(>\(timeout)s),先手动录入")
|
||||
: String(appLoc: "推理失败:\(msg)"))
|
||||
} catch CaptureError.modelNotReady {
|
||||
await editingFallback(assets: assets, msg: String(appLoc: "VL 模型未就绪,先手动录入"))
|
||||
} catch {
|
||||
await editingFallback(assets: assets,
|
||||
msg: String(appLoc: "未知错误:\(error.localizedDescription)"))
|
||||
return
|
||||
}
|
||||
await MainActor.run {
|
||||
phase = .editing(
|
||||
parsed: meta,
|
||||
assets: assets,
|
||||
warning: recognized ? nil
|
||||
: String(appLoc: "未能自动识别报告信息,已保存原图,可手动填写日期 / 机构")
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 「重新识别」:复用已存 assets,不再写图,只重跑 VL。
|
||||
/// 「重新识别信息」:复用已存 assets,不再写图,只重跑一次轻量 meta 提取。
|
||||
private func reanalyze(assets: [FileVault.SavedAsset]) {
|
||||
analyzeTask?.cancel()
|
||||
// 这里没有原始 UIImage,AnalyzingView 显示首张缩略图即可
|
||||
// 这里没有原始 UIImage,AnalyzingView 只把首张缩略图模糊后当背景,降采样到 600px 足够,
|
||||
// 避免为「重新识别」把整页全分辨率原图(数十 MB)载进内存。
|
||||
let thumbnails: [UIImage] = assets.compactMap {
|
||||
try? FileVault.shared.loadImage(relativePath: $0.relativePath)
|
||||
try? FileVault.shared.loadDownsampledImage(relativePath: $0.relativePath, maxPixelSize: 600)
|
||||
}
|
||||
phase = .analyzing(images: thumbnails, assets: assets)
|
||||
let timeout = analyzeTimeoutSeconds
|
||||
@@ -232,40 +222,19 @@ struct UnifiedCaptureFlow: View {
|
||||
}
|
||||
defer { watchdog.cancel() }
|
||||
|
||||
do {
|
||||
let parsed = try await CaptureService.shared.reanalyze(assets: assets)
|
||||
if Task.isCancelled {
|
||||
await editingFallback(assets: assets,
|
||||
msg: String(appLoc: "识别超时(>\(timeout)s),保留旧编辑"))
|
||||
return
|
||||
}
|
||||
let (meta, recognized) = await CaptureService.shared.extractReportMeta(assets: assets)
|
||||
if Task.isCancelled {
|
||||
await MainActor.run {
|
||||
phase = .editing(
|
||||
parsed: parsed,
|
||||
assets: assets,
|
||||
warning: parsed.isEmpty ? String(appLoc: "重新识别没有读出新指标") : nil
|
||||
)
|
||||
phase = .editing(parsed: .empty(), assets: assets,
|
||||
warning: String(appLoc: "识别超时,已保留原图"))
|
||||
}
|
||||
} catch CaptureError.modelNotReady {
|
||||
await editingFallback(assets: assets, msg: String(appLoc: "VL 模型未就绪"))
|
||||
} catch let CaptureError.parseFailed(msg) {
|
||||
await editingFallback(assets: assets, msg: String(appLoc: "VL 输出无法解析:\(msg)"))
|
||||
} catch let CaptureError.inferenceFailed(msg) {
|
||||
await editingFallback(assets: assets,
|
||||
msg: Task.isCancelled
|
||||
? String(appLoc: "识别超时(>\(timeout)s)")
|
||||
: String(appLoc: "推理失败:\(msg)"))
|
||||
} catch {
|
||||
await editingFallback(assets: assets,
|
||||
msg: String(appLoc: "未知错误:\(error.localizedDescription)"))
|
||||
return
|
||||
}
|
||||
await MainActor.run {
|
||||
phase = .editing(parsed: meta, assets: assets,
|
||||
warning: recognized ? nil
|
||||
: String(appLoc: "未能自动识别报告信息,可手动填写"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// reanalyze 失败时回到 editing,保留 assets 但清空 parsed。
|
||||
private func editingFallback(assets: [FileVault.SavedAsset], msg: String) async {
|
||||
await MainActor.run {
|
||||
phase = .editing(parsed: .empty(), assets: assets, warning: msg)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user