feat(capture): 统一报告捕获流程并集成视觉语言模型识别
- 替换 QuickCaptureFlow 和 ArchiveFlow 为 UnifiedCaptureFlow 统一流程 - 新增 VLSession 封装 Qwen2.5-VL 模型进行图像文本推理 - 实现 AIRuntime 中 VL 模型的准备和分析功能 - 添加 VLPrompts 定义体检化验单识别的 JSON 输出模板 - 创建 CaptureReviewForm 提供 VL 解析结果的可编辑表单界面 - 集成 VisionKit 文档扫描器支持真机多页文档扫描 - 为模拟器实现 PhotosPicker 回退方案选择已有照片 - 在 RootView 中统一使用 UnifiedCaptureFlow 处理快速和归档流程 - 添加 CustomMetricEditor 支持自定义监测指标的创建编辑删除 - 扩展 KangkangApp 模型配置以支持新数据类型 - 实现档案列表中症状结束功能通过时间线行点击触发
This commit is contained in:
link2026
218
康康/Services/CaptureService.swift
Normal file
218
康康/Services/CaptureService.swift
Normal file
@@ -0,0 +1,218 @@
|
||||
import Foundation
|
||||
import UIKit
|
||||
|
||||
/// VL 解析结果(已结构化,可直接喂 SwiftData 模型构造)。
|
||||
/// 与 Indicator/Report 字段近似但解耦 —— 这样 prompt schema 调整不污染数据层。
|
||||
struct ParsedReport: Sendable {
|
||||
var title: String
|
||||
var typeRaw: String
|
||||
var reportDate: Date
|
||||
var institution: String
|
||||
var summary: String
|
||||
var pageCount: Int
|
||||
var indicators: [ParsedIndicator]
|
||||
|
||||
struct ParsedIndicator: Sendable {
|
||||
var name: String
|
||||
var value: String
|
||||
var unit: String
|
||||
var range: String
|
||||
var status: IndicatorStatus
|
||||
}
|
||||
|
||||
/// 一项都没识别出来 = 视作失败,UI 走手动录入回退。
|
||||
var isEmpty: Bool { indicators.isEmpty }
|
||||
|
||||
/// 占位空结果,失败回退时给 UI。
|
||||
static func empty(date: Date = .now) -> ParsedReport {
|
||||
ParsedReport(
|
||||
title: "",
|
||||
typeRaw: ReportType.other.rawValue,
|
||||
reportDate: date,
|
||||
institution: "",
|
||||
summary: "",
|
||||
pageCount: 1,
|
||||
indicators: []
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// CaptureService 错误 — UI 决定怎么呈现(回退表单 vs 重试)。
|
||||
enum CaptureError: Error, LocalizedError {
|
||||
case modelNotReady
|
||||
case writeAssetFailed
|
||||
case inferenceFailed(String)
|
||||
case parseFailed(String)
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .modelNotReady: return "VL 模型尚未就绪"
|
||||
case .writeAssetFailed: return "图片保存失败"
|
||||
case .inferenceFailed(let m): return "识别失败:\(m)"
|
||||
case .parseFailed(let m): return "结构化失败:\(m)"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `CaptureService` 是 actor 是因为它的方法会等 AIRuntime(也是 actor),
|
||||
/// 但本身不持任何可变状态 —— 单例 stateless,纯粹是 §3.1 模块边界的"门面"。
|
||||
actor CaptureService {
|
||||
static let shared = CaptureService()
|
||||
private init() {}
|
||||
|
||||
/// 写图 + VL 推理 + 解析 → ParsedReport。
|
||||
/// 任何阶段失败,都抛 CaptureError;UI 接住后切到「手动录入」表单。
|
||||
/// - Returns: (ParsedReport, [FileVault.SavedAsset]) 元组,
|
||||
/// SavedAsset 列表用于后续构造 Asset @Model。
|
||||
func analyze(images: [UIImage]) async throws
|
||||
-> (parsed: ParsedReport, assets: [FileVault.SavedAsset]) {
|
||||
|
||||
// 1. 写图到 Vault(全程加密目录)
|
||||
let assets: [FileVault.SavedAsset]
|
||||
do {
|
||||
assets = try images.map { try FileVault.shared.writeJPEG($0) }
|
||||
} catch {
|
||||
throw CaptureError.writeAssetFailed
|
||||
}
|
||||
|
||||
// 2. VL 推理
|
||||
try await AIRuntime.shared.prepareVL()
|
||||
let urls = assets.map { FileVault.shared.rootURL.appendingPathComponent($0.relativePath) }
|
||||
let raw: String
|
||||
do {
|
||||
raw = try await AIRuntime.shared.analyzeReport(
|
||||
imageURLs: urls,
|
||||
prompt: VLPrompts.reportExtraction
|
||||
)
|
||||
} catch {
|
||||
throw CaptureError.inferenceFailed("\(error)")
|
||||
}
|
||||
|
||||
// 3. JSON 解析(带容错:可能包含围栏 / 前后文字)
|
||||
do {
|
||||
let parsed = try CaptureService.parseReportJSON(raw, pageCount: assets.count)
|
||||
return (parsed, assets)
|
||||
} catch let CaptureError.parseFailed(msg) {
|
||||
throw CaptureError.parseFailed(msg)
|
||||
} catch {
|
||||
throw CaptureError.parseFailed("\(error)")
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - JSON parse(static + 纯函数 → 方便单测)
|
||||
|
||||
/// 从 VL 输出里抠出第一段合法 JSON 对象并解析。
|
||||
/// 容错:
|
||||
/// - 去掉 ```json``` markdown 围栏
|
||||
/// - 去掉首尾非 JSON 文字
|
||||
/// - 缺字段填默认值
|
||||
/// 解析不到任何 indicator 也算成功,但 ParsedReport.isEmpty = true,
|
||||
/// UI 走「手动录入」分支。
|
||||
static func parseReportJSON(_ raw: String, pageCount: Int = 1) throws -> ParsedReport {
|
||||
let jsonString = extractJSONObject(from: raw)
|
||||
guard let data = jsonString.data(using: .utf8) else {
|
||||
throw CaptureError.parseFailed("非 UTF-8 输出")
|
||||
}
|
||||
let obj: Any
|
||||
do {
|
||||
obj = try JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed])
|
||||
} catch {
|
||||
throw CaptureError.parseFailed("JSON 不合法:\(error.localizedDescription)")
|
||||
}
|
||||
guard let dict = obj as? [String: Any] else {
|
||||
throw CaptureError.parseFailed("根节点不是对象")
|
||||
}
|
||||
|
||||
let title = (dict["title"] as? String)?.trimmingCharacters(in: .whitespaces) ?? ""
|
||||
let typeRaw = parseReportType(dict["type"] as? String)
|
||||
let reportDate = parseDate(dict["report_date"] as? String) ?? .now
|
||||
let institution = (dict["institution"] as? String) ?? ""
|
||||
let summary = (dict["summary"] as? String) ?? ""
|
||||
let pages = (dict["page_count"] as? Int) ?? pageCount
|
||||
|
||||
let indicatorsRaw = (dict["indicators"] as? [[String: Any]]) ?? []
|
||||
let indicators: [ParsedReport.ParsedIndicator] = indicatorsRaw.compactMap {
|
||||
parseIndicator($0)
|
||||
}
|
||||
|
||||
return ParsedReport(
|
||||
title: title.isEmpty ? "拍摄识别" : title,
|
||||
typeRaw: typeRaw,
|
||||
reportDate: reportDate,
|
||||
institution: institution,
|
||||
summary: summary,
|
||||
pageCount: max(pages, pageCount),
|
||||
indicators: indicators
|
||||
)
|
||||
}
|
||||
|
||||
/// 从字符串里抠出第一段平衡的 {...}。处理 markdown 围栏、前后乱码。
|
||||
/// 失败返回原字符串(后续 JSONSerialization 报错)。
|
||||
static func extractJSONObject(from raw: String) -> String {
|
||||
var s = raw.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
|
||||
// 去 markdown 围栏
|
||||
if s.hasPrefix("```") {
|
||||
// 砍掉首行 ```json 或 ```
|
||||
if let firstNewline = s.firstIndex(of: "\n") {
|
||||
s = String(s[s.index(after: firstNewline)...])
|
||||
}
|
||||
// 砍掉末尾 ```
|
||||
if let endRange = s.range(of: "```", options: .backwards) {
|
||||
s = String(s[..<endRange.lowerBound])
|
||||
}
|
||||
s = s.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
}
|
||||
|
||||
// 找第一个 {,然后括号配对到匹配的 }
|
||||
guard let start = s.firstIndex(of: "{") else { return s }
|
||||
var depth = 0
|
||||
var inString = false
|
||||
var escape = false
|
||||
var idx = start
|
||||
while idx < s.endIndex {
|
||||
let ch = s[idx]
|
||||
if escape { escape = false }
|
||||
else if ch == "\\" { escape = true }
|
||||
else if ch == "\"" { inString.toggle() }
|
||||
else if !inString {
|
||||
if ch == "{" { depth += 1 }
|
||||
else if ch == "}" {
|
||||
depth -= 1
|
||||
if depth == 0 {
|
||||
return String(s[start...idx])
|
||||
}
|
||||
}
|
||||
}
|
||||
idx = s.index(after: idx)
|
||||
}
|
||||
return String(s[start...])
|
||||
}
|
||||
|
||||
private static func parseReportType(_ raw: String?) -> String {
|
||||
guard let raw = raw?.lowercased() else { return ReportType.other.rawValue }
|
||||
return ReportType(rawValue: raw)?.rawValue ?? ReportType.other.rawValue
|
||||
}
|
||||
|
||||
private static func parseDate(_ raw: String?) -> Date? {
|
||||
guard let s = raw?.trimmingCharacters(in: .whitespaces), !s.isEmpty else { return nil }
|
||||
let f = DateFormatter()
|
||||
f.locale = Locale(identifier: "en_US_POSIX")
|
||||
f.dateFormat = "yyyy-MM-dd"
|
||||
return f.date(from: s)
|
||||
}
|
||||
|
||||
private static func parseIndicator(_ d: [String: Any]) -> ParsedReport.ParsedIndicator? {
|
||||
guard let name = (d["name"] as? String)?.trimmingCharacters(in: .whitespaces),
|
||||
!name.isEmpty else { return nil }
|
||||
let value: String
|
||||
if let v = d["value"] as? String { value = v }
|
||||
else if let v = d["value"] as? NSNumber { value = v.stringValue }
|
||||
else { value = "" }
|
||||
let unit = (d["unit"] as? String) ?? ""
|
||||
let range = (d["range"] as? String) ?? ""
|
||||
let statusRaw = (d["status"] as? String)?.lowercased() ?? "normal"
|
||||
let status = IndicatorStatus(rawValue: statusRaw) ?? .normal
|
||||
return .init(name: name, value: value, unit: unit, range: range, status: status)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user