9d856fcfc4
feat(AI): 集成MNN推理引擎替换MLX作为主AI运行时 - 引入MNN(alibaba) + Arm SME2 + CPU作为主AI运行时，支持A19/iPhone17的 SME2和A17的NEON加速 - 添加MLX Swift作为兜底GPU推理方案，实现双后端切换机制 - 使用单一Qwen3.5-2B多模态模型(1.2GB)，替代原有的LLM+VL分离架构 - 实现InferenceEngine.current引擎选择逻辑，真机默认MNN，模拟器回退MLX - 更新AIAgent架构，通过MNNLLMBridge(ObjC++) → MNNBackend进行推理 - 修改队列机制防止并发推理导致OOM，使用信号量闸门控制显存占用 - 更新文档中的技术栈说明、模块边界和周次交付计划 ```
453 lines
19 KiB
Swift
453 lines
19 KiB
Swift
import SwiftUI
|
||
import SwiftData
|
||
import UIKit
|
||
|
||
/// 「拍药盒入库」流程:拍药盒/说明书(最多 5 张,选一张识别)→ Vision OCR → LLM 结构化 → 核对 → 存入药品库(连同原图)。
|
||
/// 入口:「记录 · 药品库」与「记录 · 健康日记 · 拍药盒」。
|
||
/// 两个入口确认后都走 `MedicationArchiver`:每条药建一个 `Medication`(挂原图),不写日记、不写当前用药。
|
||
/// 服用流水改由「写日记 · 用药」生成带 `medicationTag` 的 DiaryEntry。只识别入库,不做用药提醒/剂量建议(§1)。
|
||
///
|
||
/// 状态机:
|
||
/// ```
|
||
/// idle(相机/相册) ─拍到第1张→ collecting(复看:删/继续拍≤5/选一张/开始识别)
|
||
/// │ 开始识别
|
||
/// ▼
|
||
/// recognizing(选中单张 OCR + LLM) ─→ confirm(核对一种药) ─onSave→ 关闭
|
||
/// │ 失败/没读出 ───────────────► confirm(空行 + 警示)
|
||
/// ```
|
||
struct MedicationScanFlow: View {
|
||
/// 用户确认后回传(结构化药品, 原图)。入库由调用方做(走 MedicationArchiver.archive(medications:))。
|
||
let onSave: ([ParsedMedication], [UIImage]) -> Void
|
||
let onClose: () -> Void
|
||
|
||
/// 一种药最多关联 5 张原图(正面/背面/说明书…)。
|
||
static let maxImages = 5
|
||
|
||
@State private var phase: Phase = .idle
|
||
/// 已拍/已选的原图,跨 collecting → recognizing → confirm 一直留着,确认时全部作为该药原图落库。
|
||
@State private var images: [UIImage] = []
|
||
/// 识别用的照片索引(在多张里单选一张)。一次只记一种药 → 只 OCR 这一张;删图时校正。
|
||
@State private var recognizeIndex = 0
|
||
/// 在 collecting 复看页「继续拍/继续选」时弹相机或相册。
|
||
@State private var showMoreCapture = false
|
||
/// 识别任务句柄:识别中点「取消」要能立刻中断,不留后台推理。
|
||
@State private var recognitionTask: Task<Void, Never>?
|
||
|
||
enum Phase {
|
||
case idle
|
||
case collecting
|
||
case recognizing
|
||
case confirm(items: [EditableMedication], warning: String?)
|
||
}
|
||
|
||
struct EditableMedication: Identifiable {
|
||
let id = UUID()
|
||
var name: String
|
||
var strength: String
|
||
var usage: String
|
||
var include: Bool = true
|
||
}
|
||
|
||
private var remainingSlots: Int { max(0, Self.maxImages - images.count) }
|
||
|
||
var body: some View {
|
||
content
|
||
.background(Tj.Palette.sand.ignoresSafeArea())
|
||
}
|
||
|
||
@ViewBuilder
|
||
private var content: some View {
|
||
switch phase {
|
||
case .idle:
|
||
// 不整体 ignoresSafeArea:相机内部已全屏黑底,忽略安全区会让「取消」顶进灵动岛。
|
||
initialCaptureEntry
|
||
|
||
case .collecting:
|
||
collectingView
|
||
.fullScreenCover(isPresented: $showMoreCapture) { moreCaptureSheet }
|
||
|
||
case .recognizing:
|
||
recognizingView
|
||
|
||
case .confirm(let items, let warning):
|
||
NavigationStack {
|
||
MedicationConfirmView(
|
||
items: items,
|
||
warning: warning,
|
||
onSave: { saveItems($0) },
|
||
onRetake: { images = []; phase = .idle }
|
||
)
|
||
.navigationTitle("核对药品")
|
||
.navigationBarTitleDisplayMode(.inline)
|
||
.toolbar {
|
||
ToolbarItem(placement: .topBarLeading) {
|
||
Button("取消") { onClose() }
|
||
.foregroundStyle(Tj.Palette.text)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// MARK: - 入口:拍照(真机)/ 相册(模拟器)
|
||
|
||
/// 首张:进入即拍/选。拿到第一张就转 collecting 复看。
|
||
@ViewBuilder
|
||
private var initialCaptureEntry: some View {
|
||
#if targetEnvironment(simulator)
|
||
PhotoPickerSheet(
|
||
onFinish: { picked in
|
||
appendImages(picked)
|
||
if images.isEmpty { onClose() } else { phase = .collecting }
|
||
},
|
||
onCancel: onClose
|
||
)
|
||
#else
|
||
SingleShotCameraView(
|
||
onCapture: { appendImages([$0]); phase = .collecting },
|
||
onCancel: onClose
|
||
)
|
||
#endif
|
||
}
|
||
|
||
/// collecting 复看页里「继续拍/继续选」弹出的二次采集。
|
||
@ViewBuilder
|
||
private var moreCaptureSheet: some View {
|
||
#if targetEnvironment(simulator)
|
||
PhotoPickerSheet(
|
||
onFinish: { picked in appendImages(picked); showMoreCapture = false },
|
||
onCancel: { showMoreCapture = false }
|
||
)
|
||
#else
|
||
SingleShotCameraView(
|
||
onCapture: { appendImages([$0]); showMoreCapture = false },
|
||
onCancel: { showMoreCapture = false }
|
||
)
|
||
#endif
|
||
}
|
||
|
||
private func appendImages(_ new: [UIImage]) {
|
||
guard remainingSlots > 0 else { return }
|
||
images.append(contentsOf: new.prefix(remainingSlots))
|
||
}
|
||
|
||
// MARK: - 复看(已拍 N 张:删 / 继续拍 / 开始识别)
|
||
|
||
private var collectingView: some View {
|
||
VStack(spacing: 0) {
|
||
ScrollView {
|
||
LazyVGrid(columns: [GridItem(.adaptive(minimum: 96), spacing: 12)], spacing: 12) {
|
||
ForEach(Array(images.enumerated()), id: \.offset) { idx, img in
|
||
let isPick = idx == recognizeIndex
|
||
ZStack(alignment: .topTrailing) {
|
||
Image(uiImage: img)
|
||
.resizable()
|
||
.scaledToFill()
|
||
.frame(width: 96, height: 96)
|
||
.clipShape(RoundedRectangle(cornerRadius: Tj.Radius.md, style: .continuous))
|
||
.overlay(
|
||
RoundedRectangle(cornerRadius: Tj.Radius.md, style: .continuous)
|
||
.strokeBorder(isPick ? Tj.Palette.ink : Color.clear, lineWidth: 3)
|
||
)
|
||
.overlay(alignment: .bottomLeading) {
|
||
if isPick {
|
||
Text("识别此张")
|
||
.font(.tjScaled(10, weight: .semibold))
|
||
.foregroundStyle(Tj.Palette.paper)
|
||
.padding(.horizontal, 6)
|
||
.padding(.vertical, 3)
|
||
.background(Capsule().fill(Tj.Palette.ink))
|
||
.padding(5)
|
||
}
|
||
}
|
||
// 点图把它选为「识别用」那张(单选)。
|
||
.onTapGesture { recognizeIndex = idx }
|
||
Button {
|
||
images.remove(at: idx)
|
||
// 校正识别索引:删选中前面的图要左移;删到越界则收回末尾。
|
||
if images.isEmpty {
|
||
recognizeIndex = 0
|
||
phase = .idle
|
||
} else if idx < recognizeIndex {
|
||
recognizeIndex -= 1
|
||
} else if recognizeIndex >= images.count {
|
||
recognizeIndex = images.count - 1
|
||
}
|
||
} label: {
|
||
Image(systemName: "xmark.circle.fill")
|
||
.font(.tjScaled(20))
|
||
.foregroundStyle(.white, .black.opacity(0.5))
|
||
.padding(4)
|
||
}
|
||
.buttonStyle(.plain)
|
||
}
|
||
}
|
||
|
||
if remainingSlots > 0 {
|
||
Button { showMoreCapture = true } label: {
|
||
VStack(spacing: 6) {
|
||
Image(systemName: "plus")
|
||
.font(.tjScaled(22, weight: .medium))
|
||
Text("继续拍")
|
||
.font(.tjScaled(12))
|
||
}
|
||
.foregroundStyle(Tj.Palette.text2)
|
||
.frame(width: 96, height: 96)
|
||
.background(
|
||
RoundedRectangle(cornerRadius: Tj.Radius.md, style: .continuous)
|
||
.strokeBorder(Tj.Palette.line, style: StrokeStyle(lineWidth: 1, dash: [4]))
|
||
)
|
||
}
|
||
.buttonStyle(.plain)
|
||
}
|
||
}
|
||
.padding(18)
|
||
}
|
||
|
||
VStack(spacing: 8) {
|
||
Text("已拍 \(images.count)/\(Self.maxImages) 张 · 可拍正面、背面、说明书")
|
||
.font(.tjScaled(12))
|
||
.foregroundStyle(Tj.Palette.text3)
|
||
if images.count > 1 {
|
||
Text("点照片选「识别此张」· 一次记一种药")
|
||
.font(.tjScaled(11))
|
||
.foregroundStyle(Tj.Palette.ink)
|
||
}
|
||
Text("照片与文字均不离开设备")
|
||
.font(.tjScaled(11))
|
||
.foregroundStyle(Tj.Palette.text3)
|
||
|
||
Button {
|
||
startRecognition()
|
||
} label: {
|
||
Text("开始识别")
|
||
.frame(maxWidth: .infinity)
|
||
}
|
||
.buttonStyle(TjPrimaryButton())
|
||
.disabled(images.isEmpty)
|
||
.opacity(images.isEmpty ? 0.4 : 1)
|
||
}
|
||
.padding(.horizontal, 18)
|
||
.padding(.bottom, 12)
|
||
}
|
||
.overlay(alignment: .topLeading) {
|
||
flowCancelButton { onClose() }
|
||
}
|
||
}
|
||
|
||
private var recognizingView: some View {
|
||
VStack(spacing: 18) {
|
||
if images.indices.contains(recognizeIndex) {
|
||
Image(uiImage: images[recognizeIndex])
|
||
.resizable()
|
||
.scaledToFit()
|
||
.frame(maxHeight: 320)
|
||
.clipShape(RoundedRectangle(cornerRadius: Tj.Radius.md, style: .continuous))
|
||
.padding(.horizontal, 24)
|
||
}
|
||
ProgressView().tint(Tj.Palette.ink)
|
||
Text("正在本地识别药品…")
|
||
.font(.tjScaled(14))
|
||
.foregroundStyle(Tj.Palette.text2)
|
||
Text("照片与文字均不离开设备")
|
||
.font(.tjScaled(12))
|
||
.foregroundStyle(Tj.Palette.text3)
|
||
}
|
||
.frame(maxWidth: .infinity, maxHeight: .infinity)
|
||
// 识别中也要能退出,不能让用户干等(§3.2 不卡死)
|
||
.overlay(alignment: .topLeading) {
|
||
flowCancelButton {
|
||
recognitionTask?.cancel()
|
||
onClose()
|
||
}
|
||
}
|
||
}
|
||
|
||
private func flowCancelButton(_ action: @escaping () -> Void) -> some View {
|
||
Button(action: action) {
|
||
Text("取消")
|
||
.font(.tjScaled(16, weight: .medium))
|
||
.foregroundStyle(Tj.Palette.text)
|
||
.padding(.horizontal, 18)
|
||
.frame(minHeight: 44)
|
||
.background(Capsule().fill(Tj.Palette.paper))
|
||
.overlay(Capsule().strokeBorder(Tj.Palette.line, lineWidth: 1))
|
||
.contentShape(Capsule())
|
||
}
|
||
.buttonStyle(.plain)
|
||
.padding(.leading, 16)
|
||
.padding(.top, 8)
|
||
}
|
||
|
||
// MARK: - 识别(选中单张 OCR → LLM 结构化)
|
||
|
||
private func startRecognition() {
|
||
guard images.indices.contains(recognizeIndex) else { return }
|
||
phase = .recognizing
|
||
let target = images[recognizeIndex]
|
||
recognitionTask = Task {
|
||
let (items, warning) = await recognize(target)
|
||
guard !Task.isCancelled else { return } // 识别中点了取消:不再回写 phase
|
||
await MainActor.run {
|
||
// 全失败也不卡死:给一条空行让用户手填(§3.2 失败回退红线)。
|
||
if items.isEmpty {
|
||
phase = .confirm(items: [EditableMedication(name: "", strength: "", usage: "")],
|
||
warning: warning ?? String(appLoc: "没读出药品,可以手动填写"))
|
||
} else {
|
||
phase = .confirm(items: items, warning: warning)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
private func recognize(_ image: UIImage) async -> (items: [EditableMedication], warning: String?) {
|
||
do {
|
||
// 一次只识别选中的这一张 → 一种药。
|
||
let text = (try? await OCRService.recognizeText(in: image))?
|
||
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||
if text.isEmpty {
|
||
return ([], String(appLoc: "没识别到文字,拍清楚一点再试"))
|
||
}
|
||
let parsed = try await MedicationScanService.shared.recognizeMedications(fromOCRText: text)
|
||
// 一次一种药:即使识别出多条,也只取第一条。
|
||
let items = parsed.prefix(1).map {
|
||
EditableMedication(name: $0.name, strength: $0.strength, usage: $0.usage)
|
||
}
|
||
return (items, items.isEmpty ? String(appLoc: "没读出药品,可以手动填写") : nil)
|
||
} catch CaptureError.modelNotReady {
|
||
return ([], String(appLoc: "AI 模型未就绪,可以手动填写"))
|
||
} catch let CaptureError.parseFailed(msg) {
|
||
return ([], String(appLoc: "解析失败:\(msg)"))
|
||
} catch let CaptureError.inferenceFailed(msg) {
|
||
return ([], String(appLoc: "识别失败:\(msg)"))
|
||
} catch {
|
||
return ([], String(appLoc: "未知错误:\(error.localizedDescription)"))
|
||
}
|
||
}
|
||
|
||
// MARK: - 保存
|
||
|
||
private func saveItems(_ items: [EditableMedication]) {
|
||
let meds = items
|
||
.filter { $0.include && !$0.name.trimmingCharacters(in: .whitespaces).isEmpty }
|
||
.map {
|
||
ParsedMedication(name: $0.name.trimmingCharacters(in: .whitespaces),
|
||
strength: $0.strength.trimmingCharacters(in: .whitespaces),
|
||
usage: $0.usage.trimmingCharacters(in: .whitespaces))
|
||
}
|
||
// 确认后入药品库(连同原图)。空条目则不入库,由调用方据数组是否为空决定。
|
||
onSave(meds, images)
|
||
onClose()
|
||
}
|
||
}
|
||
|
||
// MARK: - 入药品库(MainActor,SwiftData 写主上下文必须由 View 侧持有的 ctx 来做,§3.1)
|
||
|
||
/// 拍药盒确认后入药品库,两个入口(药品库页、写日记 · 拍药盒)共用:
|
||
/// 每条药建一个 `Medication`(挂原图),按 name+strength 软去重;**不写日记、不写 currentMedications**。
|
||
/// 服用流水改由「写日记 · 用药」生成带 `DiaryEntry.medicationTag` 的日记。
|
||
@MainActor
|
||
enum MedicationArchiver {
|
||
static func archive(medications: [ParsedMedication], images: [UIImage] = [], in ctx: ModelContext) {
|
||
guard !medications.isEmpty else { return }
|
||
|
||
// 原图写加密 Vault(§5/§6:落 Application Support/Vault,目录级硬件加密)。
|
||
// 多药共享同批原图时只挂「第一条新建的药」,避免同一 JPEG 被多个 Asset 引用、
|
||
// 删一条 cascade 误删另一条还在用的文件。
|
||
let savedAssets = images
|
||
.prefix(MedicationScanFlow.maxImages)
|
||
.compactMap { try? FileVault.shared.writeJPEG($0) }
|
||
|
||
let existing = (try? ctx.fetch(FetchDescriptor<Medication>())) ?? []
|
||
var attachedImages = false
|
||
for m in medications {
|
||
// 软去重:同 name+strength 已在库则只补用法 / 刷新时间,不重复建。
|
||
if let dup = existing.first(where: { $0.name == m.name && $0.strength == m.strength }) {
|
||
if dup.usage.isEmpty, !m.usage.isEmpty { dup.usage = m.usage }
|
||
dup.updatedAt = .now
|
||
continue
|
||
}
|
||
let med = Medication(name: m.name, strength: m.strength, usage: m.usage)
|
||
if !attachedImages {
|
||
for s in savedAssets {
|
||
let asset = Asset(relativePath: s.relativePath, bytes: s.bytes)
|
||
ctx.insert(asset)
|
||
med.assets.append(asset)
|
||
}
|
||
attachedImages = true
|
||
}
|
||
ctx.insert(med)
|
||
}
|
||
try? ctx.save()
|
||
}
|
||
}
|
||
|
||
// MARK: - 核对页
|
||
|
||
private struct MedicationConfirmView: View {
|
||
@State var items: [MedicationScanFlow.EditableMedication]
|
||
let warning: String?
|
||
let onSave: ([MedicationScanFlow.EditableMedication]) -> Void
|
||
let onRetake: () -> Void
|
||
|
||
private var canSave: Bool {
|
||
items.contains {
|
||
$0.include && !$0.name.trimmingCharacters(in: .whitespaces).isEmpty
|
||
}
|
||
}
|
||
|
||
var body: some View {
|
||
VStack(spacing: 0) {
|
||
Form {
|
||
if let warning {
|
||
Section {
|
||
Label(warning, systemImage: "exclamationmark.triangle")
|
||
.font(.tjScaled(13))
|
||
.foregroundStyle(Tj.Palette.amber)
|
||
}
|
||
}
|
||
|
||
ForEach($items) { $item in
|
||
Section {
|
||
TextField(String(appLoc: "药品名,如:缬沙坦胶囊"), text: $item.name)
|
||
.foregroundStyle(Tj.Palette.text)
|
||
TextField(String(appLoc: "规格,如:80mg×7粒"), text: $item.strength)
|
||
.foregroundStyle(Tj.Palette.text2)
|
||
TextField(String(appLoc: "用法,如:一日一次,一次一粒"), text: $item.usage)
|
||
.foregroundStyle(Tj.Palette.text2)
|
||
}
|
||
}
|
||
|
||
Section {
|
||
Button {
|
||
onRetake()
|
||
} label: {
|
||
Label("重拍", systemImage: "camera")
|
||
.foregroundStyle(Tj.Palette.ink)
|
||
}
|
||
} footer: {
|
||
Text("一次记一种药,多张照片都会作为这种药的原图存入药品库,供查看与 AI 解读参考。不提供任何用药建议。")
|
||
}
|
||
}
|
||
.scrollContentBackground(.hidden)
|
||
|
||
Button {
|
||
onSave(items)
|
||
} label: {
|
||
Text("存入药品库")
|
||
.frame(maxWidth: .infinity)
|
||
}
|
||
.buttonStyle(TjPrimaryButton())
|
||
.disabled(!canSave)
|
||
.opacity(canSave ? 1 : 0.4)
|
||
.padding(.horizontal, 18)
|
||
.padding(.bottom, 12)
|
||
}
|
||
.background(Tj.Palette.sand.ignoresSafeArea())
|
||
}
|
||
}
|
||
|
||
#Preview {
|
||
MedicationScanFlow(onSave: { _, _ in }, onClose: {})
|
||
}
|