```
feat(AI): 统一多模态模型架构,整合文本和视觉推理路径 - 将文本生成和VL(图→文)功能合并到单一的Qwen3.5-4B多模态MNN模型 - 移除独立的Qwen3-VL-4B模型依赖,MLX VL改为使用.llm的多模态模型 - 更新ModelKind枚举,新增userFacing集合用于面向用户展示 - MNN后端现在同时支持文本和视觉任务,模拟器回退到MLX refactor(models): 模型管理和界面调整以适应新的多模态架构 - 更新模型管理界面,只显示统一的Qwen3.5-4B(MNN)模型给用户 - 修改就绪状态检查逻辑,使用ModelKind.userFacing替代allCases - 更新模型文件清单,从Qwen3.5-2B升级到Qwen3.5-4B-4bit - 调整模型管理页面UI,突出MNN+SME2端侧加速功能 feat(camera): 添加拍照识别引擎切换功能 - 实现双路径拍照识别:Apple Vision OCR + 文本模型 和 Qwen3-VL直接识别 - 添加预处理逻辑,优化Qwen3-VL对窄长区域图片的识别效果 - 在模型管理页面添加拍照识别引擎选择组件 - 提供用户界面选项,在两种识别方式间切换 style(ui): 优化输入框样式和颜色主题一致性 - 为指标快速表单添加浅色主题偏好 - 统一所有文本输入框的颜色样式(theme) - 创建EntryInputField组件,替换原有的单行输入+按钮模式 - 实现聊天框风格的条目输入,支持多行自适应和圆形发送按钮 fix(build): 修正Xcode项目配置中的重复框架搜索路径 - 清理project.pbxproj中重复的FRAMEWORK_SEARCH_PATHS配置 - 重新排列Swift桥接头文件配置确保正确引用 - 修复因路径配置重复导致的编译警告问题 test: 增加区域图片预处理和模型清单测试覆盖 - 添加RegionImageCropper.prepareForQwenVL的单元测试 - 验证宽而矮图片的放大和填充逻辑 - 更新ModelManifestTests中的字节数预期值以匹配新模型 - 修正OCRService中VNRecognizedTextObservation类型的处理 ```
This commit is contained in:
@@ -15,6 +15,8 @@ struct QuickRegionCaptureFlow: View {
|
||||
@Environment(\.modelContext) private var ctx
|
||||
let onClose: () -> Void
|
||||
|
||||
@AppStorage(QuickRegionRecognitionEngine.storageKey)
|
||||
private var recognitionEngineRaw = QuickRegionRecognitionEngine.defaultValue.rawValue
|
||||
@State private var phase: Phase = .idle
|
||||
|
||||
enum Phase {
|
||||
@@ -98,8 +100,20 @@ struct QuickRegionCaptureFlow: View {
|
||||
// MARK: - 识别(框内子图 → OCR → LLM)
|
||||
|
||||
/// 对已裁好的框内子图跑识别。失败/超时返回提示文案,绝不抛出(由 RegionAdjustView 展示)。
|
||||
/// 链路:Vision 端侧 OCR 取文本 → Qwen3-1.7B 结构化抽指标(对齐 indicator-capture-ocr-llm)。
|
||||
/// 链路由「我的 → 模型管理 → 拍照识别引擎」决定:
|
||||
/// - Apple Vision:Vision 端侧 OCR → Qwen3-1.7B 结构化抽指标
|
||||
/// - Qwen3-VL:局部图片 → Qwen3-VL 直接结构化抽指标
|
||||
private func recognizeRegion(_ image: UIImage) async -> (items: [QuickRegionItem], warning: String?) {
|
||||
let engine = QuickRegionRecognitionEngine(storedValue: recognitionEngineRaw)
|
||||
switch engine {
|
||||
case .appleVision:
|
||||
return await recognizeWithAppleVision(image)
|
||||
case .qwenVL:
|
||||
return await recognizeWithQwenVL(image)
|
||||
}
|
||||
}
|
||||
|
||||
private func recognizeWithAppleVision(_ image: UIImage) async -> (items: [QuickRegionItem], warning: String?) {
|
||||
do {
|
||||
let text = try await OCRService.recognizeText(in: image)
|
||||
if Task.isCancelled { return ([], nil) } // 超时:文案由调用方给
|
||||
@@ -125,6 +139,30 @@ struct QuickRegionCaptureFlow: View {
|
||||
}
|
||||
}
|
||||
|
||||
private func recognizeWithQwenVL(_ image: UIImage) async -> (items: [QuickRegionItem], warning: String?) {
|
||||
let prepared = RegionImageCropper.prepareForQwenVL(image)
|
||||
guard let data = prepared.jpegData(compressionQuality: 0.95) else {
|
||||
return ([], String(appLoc: "图片编码失败,手动补充"))
|
||||
}
|
||||
#if DEBUG
|
||||
print("🖼️ [Qwen3-VL region] prepared image=\(Int(prepared.size.width))x\(Int(prepared.size.height)), bytes=\(data.count)")
|
||||
#endif
|
||||
do {
|
||||
let parsed = try await CaptureService.shared.recognizeRegion(imageData: data)
|
||||
if Task.isCancelled { return ([], nil) }
|
||||
let items = Self.buildItems(from: parsed)
|
||||
return (items, items.isEmpty ? String(appLoc: "没读出指标,挪一下框再试") : nil)
|
||||
} catch CaptureError.modelNotReady {
|
||||
return ([], String(appLoc: "模型未就绪,请在模型管理下载或切回 Apple Vision"))
|
||||
} catch let CaptureError.parseFailed(msg) {
|
||||
return ([], String(appLoc: "解析失败:\(msg)"))
|
||||
} catch let CaptureError.inferenceFailed(msg) {
|
||||
return ([], Task.isCancelled ? nil : String(appLoc: "识别失败:\(msg)"))
|
||||
} catch {
|
||||
return ([], Task.isCancelled ? nil : String(appLoc: "未知错误:\(error.localizedDescription)"))
|
||||
}
|
||||
}
|
||||
|
||||
/// LLM 结果 → 可编辑行,异常项(high/low)置顶、默认勾选。
|
||||
private static func buildItems(from parsed: [ParsedReport.ParsedIndicator]) -> [QuickRegionItem] {
|
||||
let mapped = parsed.map {
|
||||
|
||||
31
康康/Features/Quick/QuickRegionRecognitionEngine.swift
Normal file
31
康康/Features/Quick/QuickRegionRecognitionEngine.swift
Normal file
@@ -0,0 +1,31 @@
|
||||
import Foundation
|
||||
|
||||
enum QuickRegionRecognitionEngine: String, CaseIterable, Identifiable, Sendable {
|
||||
case appleVision
|
||||
case qwenVL
|
||||
|
||||
static let storageKey = "quickRegionRecognitionEngine"
|
||||
static let defaultValue: QuickRegionRecognitionEngine = .appleVision
|
||||
|
||||
var id: String { rawValue }
|
||||
|
||||
init(storedValue: String) {
|
||||
self = QuickRegionRecognitionEngine(rawValue: storedValue) ?? Self.defaultValue
|
||||
}
|
||||
|
||||
var title: String {
|
||||
switch self {
|
||||
case .appleVision: return String(appLoc: "Apple Vision")
|
||||
case .qwenVL: return String(appLoc: "大模型直读")
|
||||
}
|
||||
}
|
||||
|
||||
var detail: String {
|
||||
switch self {
|
||||
case .appleVision:
|
||||
return String(appLoc: "系统 OCR + 文本模型解析")
|
||||
case .qwenVL:
|
||||
return String(appLoc: "Qwen3.5-4B 多模态直接看图(MNN/MLX)")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -352,6 +352,49 @@ enum RegionImageCropper {
|
||||
guard rect.width >= 1, rect.height >= 1, let cropped = cg.cropping(to: rect) else { return up }
|
||||
return UIImage(cgImage: cropped, scale: up.scale, orientation: .up)
|
||||
}
|
||||
|
||||
/// Qwen3-VL 局部图预处理:宽而矮的小框直接喂 VL 时,processor 再缩放容易把小字压没。
|
||||
/// 这里只用于 Qwen3-VL 分支,Apple Vision OCR 保持吃原始裁剪图。
|
||||
static func prepareForQwenVL(_ image: UIImage,
|
||||
minimumShortEdge: CGFloat = 448,
|
||||
maximumLongEdge: CGFloat = 2400,
|
||||
padding: CGFloat = 64) -> UIImage {
|
||||
let up = image.normalizedUp()
|
||||
guard let cg = up.cgImage else { return up }
|
||||
|
||||
let sourceSize = CGSize(width: cg.width, height: cg.height)
|
||||
guard sourceSize.width > 0, sourceSize.height > 0 else { return up }
|
||||
|
||||
let short = min(sourceSize.width, sourceSize.height)
|
||||
let long = max(sourceSize.width, sourceSize.height)
|
||||
var scale = max(1, minimumShortEdge / short)
|
||||
if long * scale > maximumLongEdge {
|
||||
scale = maximumLongEdge / long
|
||||
}
|
||||
|
||||
let contentSize = CGSize(
|
||||
width: max(1, (sourceSize.width * scale).rounded()),
|
||||
height: max(1, (sourceSize.height * scale).rounded())
|
||||
)
|
||||
let canvasSize = CGSize(
|
||||
width: contentSize.width + padding * 2,
|
||||
height: contentSize.height + padding * 2
|
||||
)
|
||||
|
||||
let format = UIGraphicsImageRendererFormat.default()
|
||||
format.scale = 1
|
||||
format.opaque = true
|
||||
let renderer = UIGraphicsImageRenderer(size: canvasSize, format: format)
|
||||
return renderer.image { ctx in
|
||||
UIColor.white.setFill()
|
||||
ctx.fill(CGRect(origin: .zero, size: canvasSize))
|
||||
|
||||
UIImage(cgImage: cg, scale: 1, orientation: .up).draw(
|
||||
in: CGRect(x: padding, y: padding,
|
||||
width: contentSize.width, height: contentSize.height)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
extension UIImage {
|
||||
|
||||
Reference in New Issue
Block a user