feat(AI): 集成MNN推理引擎替换MLX作为主AI运行时

- 引入MNN(alibaba) + Arm SME2 + CPU作为主AI运行时,支持A19/iPhone17的
  SME2和A17的NEON加速
- 添加MLX Swift作为兜底GPU推理方案,实现双后端切换机制
- 使用单一Qwen3.5-2B多模态模型(1.2GB),替代原有的LLM+VL分离架构
- 实现InferenceEngine.current引擎选择逻辑,真机默认MNN,模拟器回退MLX
- 更新AIAgent架构,通过MNNLLMBridge(ObjC++) → MNNBackend进行推理
- 修改队列机制防止并发推理导致OOM,使用信号量闸门控制显存占用
- 更新文档中的技术栈说明、模块边界和周次交付计划
```
This commit is contained in:
link2026
2026-06-15 09:24:59 +08:00
parent 6c6a950140
commit 9d856fcfc4
37 changed files with 2605 additions and 430 deletions

View File

@@ -8,9 +8,12 @@ struct CaptureReviewForm: View {
@State var parsed: ParsedReport
let assets: [FileVault.SavedAsset]
let warning: String?
/// : + (///),
/// ( 2B OOM ), CaptureService.extractReportMeta
var metaOnly: Bool = false
let onSave: (ParsedReport) -> Void
let onCancel: () -> Void
/// assets () nil,banner
/// assets () nil,banner
var onReanalyze: (() -> Void)? = nil
var body: some View {
@@ -23,7 +26,9 @@ struct CaptureReviewForm: View {
pageThumbnails
}
metaSection
indicatorSection
if !metaOnly {
indicatorSection
}
Spacer(minLength: 8)
actions
}
@@ -68,20 +73,26 @@ struct CaptureReviewForm: View {
private var pageThumbnails: some View {
VStack(alignment: .leading, spacing: 8) {
sectionLabel(String(appLoc: "已保存 \(assets.count) 页(端侧加密)"))
if metaOnly {
Text("原图已加密保存,详情页随时可翻看放大。系统只识别报告日期与机构作为标签,不逐项录入数值。")
.font(.tjScaled( 11))
.foregroundStyle(Tj.Palette.text3)
.fixedSize(horizontal: false, vertical: true)
}
ScrollView(.horizontal, showsIndicators: false) {
HStack(spacing: 10) {
ForEach(Array(assets.enumerated()), id: \.offset) { _, asset in
if let img = try? FileVault.shared.loadImage(relativePath: asset.relativePath) {
Image(uiImage: img)
.resizable()
.scaledToFill()
.frame(width: 84, height: 110)
.clipShape(RoundedRectangle(cornerRadius: 8, style: .continuous))
.overlay(
RoundedRectangle(cornerRadius: 8, style: .continuous)
.strokeBorder(Tj.Palette.line, lineWidth: 1)
)
VaultImage(relativePath: asset.relativePath, maxPixel: 400) { img in
Image(uiImage: img).resizable().scaledToFill()
} placeholder: { _ in
Tj.Palette.paper
}
.frame(width: 84, height: 110)
.clipShape(RoundedRectangle(cornerRadius: 8, style: .continuous))
.overlay(
RoundedRectangle(cornerRadius: 8, style: .continuous)
.strokeBorder(Tj.Palette.line, lineWidth: 1)
)
}
}
}
@@ -117,9 +128,11 @@ struct CaptureReviewForm: View {
labeledField(String(appLoc: "机构(可选)")) {
TextField("如:协和医院", text: $parsed.institution)
}
labeledField(String(appLoc: "摘要(可选)")) {
TextField("一句话总结", text: $parsed.summary, axis: .vertical)
.lineLimit(1...3)
if !metaOnly {
labeledField(String(appLoc: "摘要(可选)")) {
TextField("一句话总结", text: $parsed.summary, axis: .vertical)
.lineLimit(1...3)
}
}
}
.padding(12)

View File

@@ -62,7 +62,7 @@ struct UnifiedCaptureFlow: View {
switch phase {
case .idle: return String(appLoc: "拍摄报告")
case .analyzing: return String(appLoc: "本地识别中…")
case .editing: return String(appLoc: "核对识别结果")
case .editing: return String(appLoc: "核对报告信息")
}
}
@@ -86,6 +86,7 @@ struct UnifiedCaptureFlow: View {
parsed: parsed,
assets: assets,
warning: warning,
metaOnly: true, // + meta,(§ CaptureService.extractReportMeta)
onSave: { final in saveAll(parsed: final, assets: assets) },
onCancel: cancelAll,
onReanalyze: assets.isEmpty ? nil : { reanalyze(assets: assets) }
@@ -152,9 +153,7 @@ struct UnifiedCaptureFlow: View {
phase = .analyzing(images: images, assets: nil)
let timeout = analyzeTimeoutSeconds
analyzeTask = Task {
// Step 1: Vault
// UI , CaptureService.analyze /退,
// assets phase ,cancelAll ,editingFallback
// Step 1: Vault(,)
let assets = images.compactMap { try? FileVault.shared.writeJPEG($0) }
// :,View dismisscancelAll
// phase .analyzing(_, nil),
@@ -167,7 +166,7 @@ struct UnifiedCaptureFlow: View {
phase = .editing(
parsed: .empty(),
assets: [],
warning: String(appLoc: "图片保存失败,手动录入并保留文本")
warning: String(appLoc: "图片保存失败,请重试")
)
}
return
@@ -179,49 +178,40 @@ struct UnifiedCaptureFlow: View {
}
}
// Step 2: VL (timeout cancel ,VLSession token break)
// Step 2: meta (OCR + LLM,///)
// 2B OOM watchdog cancel
let watchdog = Task {
try? await Task.sleep(for: .seconds(timeout))
analyzeTask?.cancel()
}
defer { watchdog.cancel() }
do {
let parsed = try await CaptureService.shared.reanalyze(assets: assets)
if Task.isCancelled {
await editingFallback(assets: assets,
msg: String(appLoc: "识别超时(>\(timeout)s),先手动录入"))
return
}
let (meta, recognized) = await CaptureService.shared.extractReportMeta(assets: assets)
if Task.isCancelled {
await MainActor.run {
phase = .editing(
parsed: parsed,
assets: assets,
warning: parsed.isEmpty ? String(appLoc: "识别没有读出指标,请手动补充") : nil
)
phase = .editing(parsed: .empty(), assets: assets,
warning: String(appLoc: "识别超时,已保存原图,请手动填写信息"))
}
} catch let CaptureError.parseFailed(msg) {
await editingFallback(assets: assets, msg: String(appLoc: "VL 输出无法解析:\(msg)"))
} catch let CaptureError.inferenceFailed(msg) {
await editingFallback(assets: assets,
msg: Task.isCancelled
? String(appLoc: "识别超时(>\(timeout)s),先手动录入")
: String(appLoc: "推理失败:\(msg)"))
} catch CaptureError.modelNotReady {
await editingFallback(assets: assets, msg: String(appLoc: "VL 模型未就绪,先手动录入"))
} catch {
await editingFallback(assets: assets,
msg: String(appLoc: "未知错误:\(error.localizedDescription)"))
return
}
await MainActor.run {
phase = .editing(
parsed: meta,
assets: assets,
warning: recognized ? nil
: String(appLoc: "未能自动识别报告信息,已保存原图,可手动填写日期 / 机构")
)
}
}
}
/// : assets,, VL
/// : assets,, meta
private func reanalyze(assets: [FileVault.SavedAsset]) {
analyzeTask?.cancel()
// UIImage,AnalyzingView
// UIImage,AnalyzingView , 600px ,
// ( MB)
let thumbnails: [UIImage] = assets.compactMap {
try? FileVault.shared.loadImage(relativePath: $0.relativePath)
try? FileVault.shared.loadDownsampledImage(relativePath: $0.relativePath, maxPixelSize: 600)
}
phase = .analyzing(images: thumbnails, assets: assets)
let timeout = analyzeTimeoutSeconds
@@ -232,40 +222,19 @@ struct UnifiedCaptureFlow: View {
}
defer { watchdog.cancel() }
do {
let parsed = try await CaptureService.shared.reanalyze(assets: assets)
if Task.isCancelled {
await editingFallback(assets: assets,
msg: String(appLoc: "识别超时(>\(timeout)s),保留旧编辑"))
return
}
let (meta, recognized) = await CaptureService.shared.extractReportMeta(assets: assets)
if Task.isCancelled {
await MainActor.run {
phase = .editing(
parsed: parsed,
assets: assets,
warning: parsed.isEmpty ? String(appLoc: "重新识别没有读出新指标") : nil
)
phase = .editing(parsed: .empty(), assets: assets,
warning: String(appLoc: "识别超时,已保留原图"))
}
} catch CaptureError.modelNotReady {
await editingFallback(assets: assets, msg: String(appLoc: "VL 模型未就绪"))
} catch let CaptureError.parseFailed(msg) {
await editingFallback(assets: assets, msg: String(appLoc: "VL 输出无法解析:\(msg)"))
} catch let CaptureError.inferenceFailed(msg) {
await editingFallback(assets: assets,
msg: Task.isCancelled
? String(appLoc: "识别超时(>\(timeout)s)")
: String(appLoc: "推理失败:\(msg)"))
} catch {
await editingFallback(assets: assets,
msg: String(appLoc: "未知错误:\(error.localizedDescription)"))
return
}
await MainActor.run {
phase = .editing(parsed: meta, assets: assets,
warning: recognized ? nil
: String(appLoc: "未能自动识别报告信息,可手动填写"))
}
}
}
/// reanalyze editing, assets parsed
private func editingFallback(assets: [FileVault.SavedAsset], msg: String) async {
await MainActor.run {
phase = .editing(parsed: .empty(), assets: assets, warning: msg)
}
}