tim/kangkang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

```

Browse Source 
feat(AI): 统一多模态模型架构，整合文本和视觉推理路径

- 将文本生成和VL(图→文)功能合并到单一的Qwen3.5-4B多模态MNN模型
- 移除独立的Qwen3-VL-4B模型依赖，MLX VL改为使用.llm的多模态模型
- 更新ModelKind枚举，新增userFacing集合用于面向用户展示
- MNN后端现在同时支持文本和视觉任务，模拟器回退到MLX

refactor(models): 模型管理和界面调整以适应新的多模态架构

- 更新模型管理界面，只显示统一的Qwen3.5-4B(MNN)模型给用户
- 修改就绪状态检查逻辑，使用ModelKind.userFacing替代allCases
- 更新模型文件清单，从Qwen3.5-2B升级到Qwen3.5-4B-4bit
- 调整模型管理页面UI，突出MNN+SME2端侧加速功能

feat(camera): 添加拍照识别引擎切换功能

- 实现双路径拍照识别：Apple Vision OCR + 文本模型 和 Qwen3-VL直接识别
- 添加预处理逻辑，优化Qwen3-VL对窄长区域图片的识别效果
- 在模型管理页面添加拍照识别引擎选择组件
- 提供用户界面选项，在两种识别方式间切换

style(ui): 优化输入框样式和颜色主题一致性

- 为指标快速表单添加浅色主题偏好
- 统一所有文本输入框的颜色样式(theme)
- 创建EntryInputField组件，替换原有的单行输入+按钮模式
- 实现聊天框风格的条目输入，支持多行自适应和圆形发送按钮

fix(build): 修正Xcode项目配置中的重复框架搜索路径

- 清理project.pbxproj中重复的FRAMEWORK_SEARCH_PATHS配置
- 重新排列Swift桥接头文件配置确保正确引用
- 修复因路径配置重复导致的编译警告问题

test: 增加区域图片预处理和模型清单测试覆盖

- 添加RegionImageCropper.prepareForQwenVL的单元测试
- 验证宽而矮图片的放大和填充逻辑
- 更新ModelManifestTests中的字节数预期值以匹配新模型
- 修正OCRService中VNRecognizedTextObservation类型的处理
```
This commit is contained in:
link2026
2026-06-08 23:25:31 +08:00
parent b919404412
commit 836f3d4234
20 changed files with 393 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
康康.xcodeproj/project.pbxproj
View File

@@ -420,6 +420,7 @@
ENABLE_HARDENED_RUNTIME = YES;
ENABLE_PREVIEWS = YES;
ENABLE_USER_SELECTED_FILES = readonly;
FRAMEWORK_SEARCH_PATHS = "$(PROJECT_DIR)/Frameworks";
GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_KEY_CFBundleDisplayName = "康康";
INFOPLIST_KEY_ITSAppUsesNonExemptEncryption = NO;
@@ -447,8 +448,6 @@
PRODUCT_BUNDLE_IDENTIFIER = com.xuhuayong.kangkang;
PRODUCT_NAME = "$(TARGET_NAME)";
REGISTER_APP_GROUPS = YES;
FRAMEWORK_SEARCH_PATHS = "$(PROJECT_DIR)/Frameworks";
SWIFT_OBJC_BRIDGING_HEADER = "康康/康康-Bridging-Header.h";
SDKROOT = auto;
STRING_CATALOG_GENERATE_SYMBOLS = YES;
SUPPORTED_PLATFORMS = "iphoneos iphonesimulator";
@@ -456,6 +455,7 @@
SWIFT_APPROACHABLE_CONCURRENCY = YES;
SWIFT_DEFAULT_ACTOR_ISOLATION = MainActor;
SWIFT_EMIT_LOC_STRINGS = YES;
SWIFT_OBJC_BRIDGING_HEADER = "康康/康康-Bridging-Header.h";
SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES;
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2";
@@ -476,6 +476,7 @@
ENABLE_HARDENED_RUNTIME = YES;
ENABLE_PREVIEWS = YES;
ENABLE_USER_SELECTED_FILES = readonly;
FRAMEWORK_SEARCH_PATHS = "$(PROJECT_DIR)/Frameworks";
GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_KEY_CFBundleDisplayName = "康康";
INFOPLIST_KEY_ITSAppUsesNonExemptEncryption = NO;
@@ -503,8 +504,6 @@
PRODUCT_BUNDLE_IDENTIFIER = com.xuhuayong.kangkang;
PRODUCT_NAME = "$(TARGET_NAME)";
REGISTER_APP_GROUPS = YES;
FRAMEWORK_SEARCH_PATHS = "$(PROJECT_DIR)/Frameworks";
SWIFT_OBJC_BRIDGING_HEADER = "康康/康康-Bridging-Header.h";
SDKROOT = auto;
STRING_CATALOG_GENERATE_SYMBOLS = YES;
SUPPORTED_PLATFORMS = "iphoneos iphonesimulator";
@@ -512,6 +511,7 @@
SWIFT_APPROACHABLE_CONCURRENCY = YES;
SWIFT_DEFAULT_ACTOR_ISOLATION = MainActor;
SWIFT_EMIT_LOC_STRINGS = YES;
SWIFT_OBJC_BRIDGING_HEADER = "康康/康康-Bridging-Header.h";
SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES;
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2";

12
康康/AI/AIRuntime.swift
View File

@@ -33,10 +33,11 @@ actor AIRuntime {
private var vlSession: VLSession?
// MARK: - MNN 后端(CPU/SME2,挑战赛考核路径)
// 文本生成在 .mnn 引擎下走 MNN;VL(图→文)暂仍走 MLX(MNN VL 需 OMNI 构建)。
// .mnn 引擎下,文本生成与 VL(图→文)由同一个 Qwen3.5-4B 多模态 MNN 模型全包(已实测)。
// 模拟器无 MNN,VL 回退 MLX 的 Qwen3-VL-4B。
private let mnn = MNNBackend()
private(set) var mnnStatus: Status = .notReady
/// MNN 模型目录(下载/旁路导入到 Models/Qwen3.5-2B-MNN)。
/// MNN 模型目录(下载/旁路导入到 Models/Qwen3.5-4B-MNN)。
nonisolated static var mnnModelFolder: URL {
ModelStore.shared.localURL(for: .mnnLLM)
}
@@ -265,8 +266,9 @@ actor AIRuntime {
}
if vlStatus == .ready { return }
// 同 prepare():用 isComplete 排除半下载(避免在残缺权重上崩溃),与下载服务判据一致。
guard ModelStore.shared.isComplete(for: .vl) else {
// MLX VL 改用 .llm 的 Qwen3.5-4B 多模态(VLMModelFactory 走 qwen3_5 视觉路径),
// 不再单独需要 Qwen3-VL-4B。用 isComplete 排除半下载,与下载服务判据一致。
guard ModelStore.shared.isComplete(for: .llm) else {
vlStatus = .error("VL 模型未就绪")
throw AIRuntimeError.notReady
}
@@ -284,7 +286,7 @@ actor AIRuntime {
vlStatus = .loading
do {
let session = try await VLSession.load(
folderURL: ModelStore.shared.localURL(for: .vl)
folderURL: ModelStore.shared.localURL(for: .llm)
)
self.vlSession = session
vlStatus = .ready

5
康康/AI/MNNBackend.swift
View File

@@ -3,8 +3,9 @@ import Foundation
/// MNN(CPU / SME2)推理后端,封装 `MNNLLMBridge` 的文本流式生成。
/// 与 `LLMSession`/`VLSession` 同款 actor 隔离;跨调用的串行化由上游 `AIRuntime` 闸门保证。
///
/// VL(图→文)需 MNN OMNI 构建(OpenCV 解码图像),当前文本构建不支持;`analyze` 抛错,
/// 上层在 VL 路径回退 MLX(见 `AIRuntime`)。
/// 文本与视觉(图→文)由同一个 Qwen3.5-4B 多模态 MNN 模型承担:`generate` 走文本,
/// `analyze` 把图片拼成 <img> 标签交给 Omni 内核 imread 解码(需 OMNI 构建,xcframework 已含)。
/// 已实测可用,真机走此单模型全包路径;模拟器无 MNN,VL 仍回退 MLX(见 `AIRuntime`)。
actor MNNBackend {
private var bridge: MNNLLMBridge?

18
康康/AI/ModelManifest.swift
View File

@@ -18,20 +18,18 @@ nonisolated enum ModelManifest {
static func files(for kind: ModelKind) -> [ModelFile] {
switch kind {
case .llm:
// Qwen3.5-2B-4bit:多模态仓库,但走 LLMModelFactory 的 qwen3_5 文本路径加载。
// 字节数取自 mlx-community/Qwen3.5-2B-4bit 仓库实际 blob 大小(HF API,2026-06 核对)。
// 该仓库 tokenizer 体系为 vocab.json + tokenizer.json(无 merges.txt /
// special_tokens_map.json / added_tokens.json),chat_template 改为 .jinja。
// 一并镜像视觉预处理配置(preprocessor / processor / video_preprocessor),
// 文本加载用不到但体积可忽略,保持与仓库一致避免漏文件。
// Qwen3.5-4B-4bit:多模态仓库,MLX 兜底用它同时做文本(LLMModelFactory qwen3_5 文本路径)
// 与视觉(VLMModelFactory qwen3_5)。字节数取自 mlx-community/Qwen3.5-4B-4bit
// 仓库实际 blob 大小(HF API,2026-06 核对)。镜像全部运行文件(含视觉预处理配置),
// 排除 README.md / .gitattributes。
return [
ModelFile(path: "config.json", bytes: 3_113),
ModelFile(path: "model.safetensors", bytes: 1_722_271_785),
ModelFile(path: "model.safetensors.index.json", bytes: 81_722),
ModelFile(path: "config.json", bytes: 3_366),
ModelFile(path: "model.safetensors", bytes: 3_034_300_695),
ModelFile(path: "model.safetensors.index.json", bytes: 101_944),
ModelFile(path: "tokenizer.json", bytes: 19_989_343),
ModelFile(path: "tokenizer_config.json", bytes: 1_139),
ModelFile(path: "vocab.json", bytes: 6_722_759),
ModelFile(path: "chat_template.jinja", bytes: 7_755),
ModelFile(path: "chat_template.jinja", bytes: 7_756),
ModelFile(path: "preprocessor_config.json", bytes: 390),
ModelFile(path: "processor_config.json", bytes: 1_300),
ModelFile(path: "video_preprocessor_config.json", bytes: 385),

17
康康/AI/ModelStore.swift
View File

@@ -2,16 +2,17 @@ import Foundation
nonisolated enum ModelKind: String, CaseIterable {
/// 也是沙盒 Models/ 下的子目录名 / CDN 路径段。
/// - llm:MLX(GPU)文本兜底,Qwen3.5-2B(多模态权重,走 qwen3_5 文本路径)。
/// - vl :MLX(GPU)拍照识别,Qwen3-VL-4B。
/// - mnnLLM:MNN(CPU/SME2,挑战赛考核路径)文本,Qwen3.5-4B 预转换 MNN 格式(taobao-mnn)。
case llm = "Qwen3.5-2B-4bit"
/// 同一个 Qwen3.5-4B,两种格式两种引擎:
/// - mnnLLM:MNN(CPU/SME2,考核路径)文本+视觉一肩挑,taobao-mnn 预转换。真机主用,只露它。
/// - llm:MLX(GPU)兜底,Qwen3.5-4B-4bit 多模态(同时兜底文本与视觉,走 qwen3_5)。
/// - vl:已废弃(MLX VL 改走 .llm 多模态),保留枚举避免动一圈穷举 switch,不再下载/展示。
case llm = "Qwen3.5-4B-4bit"
case vl = "Qwen3-VL-4B-Instruct-4bit"
case mnnLLM = "Qwen3.5-4B-MNN"
var displayName: String {
switch self {
case .llm: return "Qwen3.5-2B (MLX)"
case .llm: return "Qwen3.5-4B (MLX)"
case .vl: return "Qwen3-VL-4B"
case .mnnLLM: return "Qwen3.5-4B (MNN/SME2)"
}
@@ -22,6 +23,12 @@ nonisolated enum ModelKind: String, CaseIterable {
/// 用于判定该模型是否已就绪的最小标志文件
var sentinelFilename: String { "config.json" }
/// 面向用户的模型集合:模型管理页 / 下载全部 / 就绪计数对外只暴露统一的
/// Qwen3.5-4B(MNN,文本+视觉全包,真机走它)。
/// MLX 的 .llm/.vl 仅作模拟器与兜底路径,保留枚举与下载能力(旁路导入仍可单独导),
/// 但不在「我的 · 模型管理」展示,也不计入「下载全部」与就绪计数。
static let userFacing: [ModelKind] = [.mnnLLM]
}
/// `@unchecked Sendable`:rootURL 是 let,方法只读 filesystem(线程安全),

2
康康/AI/Prompts/VLPrompts.swift
View File

@@ -3,7 +3,7 @@ import Foundation
/// VL 模型(Qwen3-VL)用于体检 / 化验单识别的 prompt 模板。
/// 输出契约:严格 JSON,无任何解释文字、markdown 围栏或前后缀。
/// 解析失败 → CaptureService 回退到手动录入(§3.2 失败回退红线)。
enum VLPrompts {
nonisolated enum VLPrompts {
/// 输出 JSON 的字段定义(写进 prompt 里教模型):
/// ```

13
康康/Features/Indicator/IndicatorQuickSheet.swift
View File

@@ -143,6 +143,7 @@ struct IndicatorQuickSheet: View {
.clipShape(RoundedRectangle(cornerRadius: Tj.Radius.xl, style: .continuous))
.ignoresSafeArea(edges: .bottom)
)
.preferredColorScheme(.light)
.presentationDetents([.large])
.presentationDragIndicator(.hidden)
.presentationBackground(Tj.Palette.sand)
@@ -423,6 +424,8 @@ struct IndicatorQuickSheet: View {
TextField(placeholder, text: value)
.keyboardType(.decimalPad)
.font(.tjScaled( 20, weight: .semibold, design: .monospaced))
.foregroundStyle(Tj.Palette.text)
.tint(Tj.Palette.ink)
.multilineTextAlignment(.center)
.padding(.vertical, 10)
.frame(width: 90)
@@ -468,6 +471,8 @@ struct IndicatorQuickSheet: View {
sectionLabel(String(appLoc: "指标名"))
TextField("例如:血红蛋白", text: $name)
.textInputAutocapitalization(.never)
.foregroundStyle(Tj.Palette.text)
.tint(Tj.Palette.ink)
.padding(.horizontal, 14)
.padding(.vertical, 12)
.background(fieldBg)
@@ -489,6 +494,8 @@ struct IndicatorQuickSheet: View {
TextField(monitorFieldPlaceholder, text: $value)
.keyboardType(.decimalPad)
.font(.tjScaled( 18, weight: .semibold, design: .monospaced))
.foregroundStyle(Tj.Palette.text)
.tint(Tj.Palette.ink)
.padding(.horizontal, 14)
.padding(.vertical, 12)
.background(fieldBg)
@@ -499,6 +506,8 @@ struct IndicatorQuickSheet: View {
TextField("mmol/L", text: $unit)
.textInputAutocapitalization(.never)
.autocorrectionDisabled()
.foregroundStyle(Tj.Palette.text)
.tint(Tj.Palette.ink)
.padding(.horizontal, 14)
.padding(.vertical, 12)
.background(fieldBg)
@@ -522,6 +531,8 @@ struct IndicatorQuickSheet: View {
TextField("例如:< 3.40 或 3.9 - 6.1", text: $range)
.textInputAutocapitalization(.never)
.autocorrectionDisabled()
.foregroundStyle(Tj.Palette.text)
.tint(Tj.Palette.ink)
.padding(.horizontal, 14)
.padding(.vertical, 12)
.background(fieldBg)
@@ -581,6 +592,8 @@ struct IndicatorQuickSheet: View {
sectionLabel(String(appLoc: "备注(可选)"))
TextField("例如:空腹采血", text: $note, axis: .vertical)
.lineLimit(1...3)
.foregroundStyle(Tj.Palette.text)
.tint(Tj.Palette.ink)
.padding(.horizontal, 14)
.padding(.vertical, 12)
.background(fieldBg)

6
康康/Features/Me/MeView.swift
View File

@@ -152,10 +152,10 @@ struct MeView: View {
private var modelDetail: String {
let states = downloadService.states
if ModelKind.allCases.allSatisfy({ states[$0]?.phase == .ready }) { return String(appLoc: "已就绪") }
if ModelKind.userFacing.allSatisfy({ states[$0]?.phase == .ready }) { return String(appLoc: "已就绪") }
if downloadService.isAnyDownloading { return String(appLoc: "下载中…") }
let readyCount = ModelKind.allCases.filter { states[$0]?.phase == .ready }.count
return readyCount == 0 ? String(appLoc: "未下载") : String(appLoc: "\(readyCount)/\(ModelKind.allCases.count) 就绪")
let readyCount = ModelKind.userFacing.filter { states[$0]?.phase == .ready }.count
return readyCount == 0 ? String(appLoc: "未下载") : String(appLoc: "\(readyCount)/\(ModelKind.userFacing.count) 就绪")
}
private var inferenceEngineCard: some View {

62
康康/Features/Me/ModelManagementView.swift
View File

@@ -10,25 +10,29 @@ struct ModelManagementView: View {
@State private var showCellularConfirm = false
@State private var showImporter = false
@State private var importError: String?
@AppStorage(QuickRegionRecognitionEngine.storageKey)
private var quickRegionEngineRaw = QuickRegionRecognitionEngine.defaultValue.rawValue
private let monitor = NWPathMonitor()
private let monitorQueue = DispatchQueue(label: "kk.netmonitor")
private var allReady: Bool {
ModelKind.allCases.allSatisfy { service.states[$0]?.phase == .ready }
ModelKind.userFacing.allSatisfy { service.states[$0]?.phase == .ready }
}
var body: some View {
ScrollView {
VStack(spacing: 14) {
ForEach(ModelKind.allCases, id: \.self) { kind in
ForEach(ModelKind.userFacing, id: \.self) { kind in
modelCard(kind)
}
recognitionEngineCard
actionButtons
.padding(.top, 4)
if service.states[.llm]?.phase == .ready {
if service.states[.mnnLLM]?.phase == .ready {
NavigationLink {
ModelSelfTestView()
} label: {
@@ -76,6 +80,46 @@ struct ModelManagementView: View {
}
}
// MARK: - 拍照识别引擎
private var selectedRecognitionEngine: QuickRegionRecognitionEngine {
QuickRegionRecognitionEngine(storedValue: quickRegionEngineRaw)
}
private var recognitionEngineCard: some View {
VStack(alignment: .leading, spacing: 12) {
HStack(alignment: .top, spacing: 10) {
ZStack {
Circle().fill(Tj.Palette.sand2)
Image(systemName: "camera.metering.center.weighted")
.font(.tjScaled( 18))
.foregroundStyle(Tj.Palette.text2)
}
.frame(width: 38, height: 38)
VStack(alignment: .leading, spacing: 3) {
Text("异常项拍照识别")
.font(.tjScaled( 15, weight: .semibold))
.foregroundStyle(Tj.Palette.text)
Text(selectedRecognitionEngine.detail)
.font(.tjScaled( 12))
.foregroundStyle(Tj.Palette.text3)
}
Spacer()
}
Picker("异常项拍照识别", selection: $quickRegionEngineRaw) {
ForEach(QuickRegionRecognitionEngine.allCases) { engine in
Text(engine.title).tag(engine.rawValue)
}
}
.pickerStyle(.segmented)
}
.padding(14)
.frame(maxWidth: .infinity, alignment: .leading)
.tjCard()
}
// MARK: - 模型卡片
private func modelCard(_ kind: ModelKind) -> some View {
@@ -146,7 +190,7 @@ struct ModelManagementView: View {
private var actionButtons: some View {
if service.isAnyDownloading {
Button {
for kind in ModelKind.allCases { service.cancel(kind) }
for kind in ModelKind.userFacing { service.cancel(kind) }
} label: {
Text("暂停下载").frame(maxWidth: .infinity)
}
@@ -154,7 +198,7 @@ struct ModelManagementView: View {
} else if allReady {
HStack(spacing: 6) {
Image(systemName: "checkmark.seal.fill")
Text("两个模型都已就绪")
Text("Qwen3.5-4B 已就绪")
}
.font(.tjScaled( 13, weight: .semibold))
.foregroundStyle(Tj.Palette.leaf)
@@ -198,8 +242,8 @@ struct ModelManagementView: View {
defer { if scoped { folder.stopAccessingSecurityScopedResource() } }
let name = folder.lastPathComponent
guard let kind = ModelKind.allCases.first(where: { $0.rawValue == name }) else {
let names = ModelKind.allCases.map(\.rawValue).joined(separator: " 或 ")
guard let kind = ModelKind.userFacing.first(where: { $0.rawValue == name }) else {
let names = ModelKind.userFacing.map(\.rawValue).joined(separator: " 或 ")
importError = String(appLoc: "请选择名为 \(names) 的文件夹")
return
}
@@ -213,14 +257,14 @@ struct ModelManagementView: View {
// MARK: - 辅助
private var totalAllBytes: Int {
ModelKind.allCases.reduce(0) { $0 + ModelManifest.totalBytes(for: $1) }
ModelKind.userFacing.reduce(0) { $0 + ModelManifest.totalBytes(for: $1) }
}
private func subtitle(_ kind: ModelKind) -> String {
switch kind {
case .llm: return String(appLoc: "文本解读 · 趋势 / 问答(MLX 兜底)")
case .vl: return String(appLoc: "拍照识别报告 → 结构化指标")
case .mnnLLM: return String(appLoc: "文本解读 · MNN + SME2 端侧加速")
case .mnnLLM: return String(appLoc: "文本解读 + 拍照识别 · MNN + SME2 端侧加速")
}
}

74
康康/Features/Profile/ProfileEditView.swift
View File

@@ -370,17 +370,14 @@ private struct ChronicSection: View {
}
}
HStack {
TextField("自定义慢病", text: $newCustomCondition)
Button("加") {
let trimmed = newCustomCondition.trimmingCharacters(in: .whitespaces)
guard !trimmed.isEmpty,
!profile.chronicConditions.contains(trimmed) else { return }
profile.chronicConditions.append(trimmed)
newCustomCondition = ""
}
.disabled(newCustomCondition.trimmingCharacters(in: .whitespaces).isEmpty)
EntryInputField(placeholder: String(appLoc: "自定义慢病"), text: $newCustomCondition) {
let trimmed = newCustomCondition.trimmingCharacters(in: .whitespaces)
guard !trimmed.isEmpty,
!profile.chronicConditions.contains(trimmed) else { return }
profile.chronicConditions.append(trimmed)
newCustomCondition = ""
}
.listRowBackground(Color.clear)
} header: {
Text("慢病(影响参考范围与 AI 解读)")
}
@@ -408,6 +405,48 @@ private struct ChronicSection: View {
}
}
// MARK: - 聊天框风格的条目输入(圆角容器 + 多行增长 + 圆形发送按钮)
/// 替代原先单行 `TextField + “加”` 的搜索框观感:文字随内容换行增长(1~4 行),
/// 右侧圆形发送按钮(内容为空时禁用变灰)。过敏 / 家族史 / 用药 / 自定义慢病共用。
private struct EntryInputField: View {
let placeholder: String
@Binding var text: String
var onSubmit: () -> Void
private var canSubmit: Bool {
!text.trimmingCharacters(in: .whitespaces).isEmpty
}
var body: some View {
HStack(alignment: .bottom, spacing: 8) {
TextField(placeholder, text: $text, axis: .vertical)
.lineLimit(1...4)
.padding(.horizontal, 14)
.padding(.vertical, 10)
.background(
RoundedRectangle(cornerRadius: Tj.Radius.md, style: .continuous)
.fill(Tj.Palette.paper)
)
.overlay(
RoundedRectangle(cornerRadius: Tj.Radius.md, style: .continuous)
.strokeBorder(Tj.Palette.line, lineWidth: 1)
)
Button {
if canSubmit { onSubmit() }
} label: {
Image(systemName: "arrow.up.circle.fill")
.font(.tjScaled(28))
.foregroundStyle(canSubmit ? Tj.Palette.ink : Tj.Palette.text3)
}
.buttonStyle(.plain)
.disabled(!canSubmit)
}
.padding(.vertical, 2)
}
}
// MARK: - 过敏 / 家族史 / 用药(每节自带 @State,敲字只重算本节)
private struct StringListSection: View {
@@ -431,16 +470,13 @@ private struct StringListSection: View {
.buttonStyle(.borderless)
}
}
HStack {
TextField(placeholder, text: $newInput)
Button("加") {
let trimmed = newInput.trimmingCharacters(in: .whitespaces)
guard !trimmed.isEmpty, !items.contains(trimmed) else { return }
items.append(trimmed)
newInput = ""
}
.disabled(newInput.trimmingCharacters(in: .whitespaces).isEmpty)
EntryInputField(placeholder: placeholder, text: $newInput) {
let trimmed = newInput.trimmingCharacters(in: .whitespaces)
guard !trimmed.isEmpty, !items.contains(trimmed) else { return }
items.append(trimmed)
newInput = ""
}
.listRowBackground(Color.clear)
}
}
}

40
康康/Features/Quick/QuickRegionCaptureFlow.swift
View File

@@ -15,6 +15,8 @@ struct QuickRegionCaptureFlow: View {
@Environment(\.modelContext) private var ctx
let onClose: () -> Void
@AppStorage(QuickRegionRecognitionEngine.storageKey)
private var recognitionEngineRaw = QuickRegionRecognitionEngine.defaultValue.rawValue
@State private var phase: Phase = .idle
enum Phase {
@@ -98,8 +100,20 @@ struct QuickRegionCaptureFlow: View {
// MARK: - 识别(框内子图 → OCR → LLM)
/// 对已裁好的框内子图跑识别。失败/超时返回提示文案,绝不抛出(由 RegionAdjustView 展示)。
/// 链路:Vision 端侧 OCR 取文本 → Qwen3-1.7B 结构化抽指标(对齐 indicator-capture-ocr-llm)。
/// 链路由「我的 → 模型管理 → 拍照识别引擎」决定:
/// - Apple Vision:Vision 端侧 OCR → Qwen3-1.7B 结构化抽指标
/// - Qwen3-VL:局部图片 → Qwen3-VL 直接结构化抽指标
private func recognizeRegion(_ image: UIImage) async -> (items: [QuickRegionItem], warning: String?) {
let engine = QuickRegionRecognitionEngine(storedValue: recognitionEngineRaw)
switch engine {
case .appleVision:
return await recognizeWithAppleVision(image)
case .qwenVL:
return await recognizeWithQwenVL(image)
}
}
private func recognizeWithAppleVision(_ image: UIImage) async -> (items: [QuickRegionItem], warning: String?) {
do {
let text = try await OCRService.recognizeText(in: image)
if Task.isCancelled { return ([], nil) } // 超时:文案由调用方给
@@ -125,6 +139,30 @@ struct QuickRegionCaptureFlow: View {
}
}
private func recognizeWithQwenVL(_ image: UIImage) async -> (items: [QuickRegionItem], warning: String?) {
let prepared = RegionImageCropper.prepareForQwenVL(image)
guard let data = prepared.jpegData(compressionQuality: 0.95) else {
return ([], String(appLoc: "图片编码失败,手动补充"))
}
#if DEBUG
print("🖼️ [Qwen3-VL region] prepared image=\(Int(prepared.size.width))x\(Int(prepared.size.height)), bytes=\(data.count)")
#endif
do {
let parsed = try await CaptureService.shared.recognizeRegion(imageData: data)
if Task.isCancelled { return ([], nil) }
let items = Self.buildItems(from: parsed)
return (items, items.isEmpty ? String(appLoc: "没读出指标,挪一下框再试") : nil)
} catch CaptureError.modelNotReady {
return ([], String(appLoc: "模型未就绪,请在模型管理下载或切回 Apple Vision"))
} catch let CaptureError.parseFailed(msg) {
return ([], String(appLoc: "解析失败:\(msg)"))
} catch let CaptureError.inferenceFailed(msg) {
return ([], Task.isCancelled ? nil : String(appLoc: "识别失败:\(msg)"))
} catch {
return ([], Task.isCancelled ? nil : String(appLoc: "未知错误:\(error.localizedDescription)"))
}
}
/// LLM 结果 → 可编辑行,异常项(high/low)置顶、默认勾选。
private static func buildItems(from parsed: [ParsedReport.ParsedIndicator]) -> [QuickRegionItem] {
let mapped = parsed.map {

31
康康/Features/Quick/QuickRegionRecognitionEngine.swift Normal file
View File

@@ -0,0 +1,31 @@
import Foundation
enum QuickRegionRecognitionEngine: String, CaseIterable, Identifiable, Sendable {
case appleVision
case qwenVL
static let storageKey = "quickRegionRecognitionEngine"
static let defaultValue: QuickRegionRecognitionEngine = .appleVision
var id: String { rawValue }
init(storedValue: String) {
self = QuickRegionRecognitionEngine(rawValue: storedValue) ?? Self.defaultValue
}
var title: String {
switch self {
case .appleVision: return String(appLoc: "Apple Vision")
case .qwenVL: return String(appLoc: "大模型直读")
}
}
var detail: String {
switch self {
case .appleVision:
return String(appLoc: "系统 OCR + 文本模型解析")
case .qwenVL:
return String(appLoc: "Qwen3.5-4B 多模态直接看图(MNN/MLX)")
}
}
}

43
康康/Features/Quick/RegionCameraView.swift
View File

@@ -352,6 +352,49 @@ enum RegionImageCropper {
guard rect.width >= 1, rect.height >= 1, let cropped = cg.cropping(to: rect) else { return up }
return UIImage(cgImage: cropped, scale: up.scale, orientation: .up)
}
/// Qwen3-VL 局部图预处理:宽而矮的小框直接喂 VL 时,processor 再缩放容易把小字压没。
/// 这里只用于 Qwen3-VL 分支,Apple Vision OCR 保持吃原始裁剪图。
static func prepareForQwenVL(_ image: UIImage,
minimumShortEdge: CGFloat = 448,
maximumLongEdge: CGFloat = 2400,
padding: CGFloat = 64) -> UIImage {
let up = image.normalizedUp()
guard let cg = up.cgImage else { return up }
let sourceSize = CGSize(width: cg.width, height: cg.height)
guard sourceSize.width > 0, sourceSize.height > 0 else { return up }
let short = min(sourceSize.width, sourceSize.height)
let long = max(sourceSize.width, sourceSize.height)
var scale = max(1, minimumShortEdge / short)
if long * scale > maximumLongEdge {
scale = maximumLongEdge / long
}
let contentSize = CGSize(
width: max(1, (sourceSize.width * scale).rounded()),
height: max(1, (sourceSize.height * scale).rounded())
)
let canvasSize = CGSize(
width: contentSize.width + padding * 2,
height: contentSize.height + padding * 2
)
let format = UIGraphicsImageRendererFormat.default()
format.scale = 1
format.opaque = true
let renderer = UIGraphicsImageRenderer(size: canvasSize, format: format)
return renderer.image { ctx in
UIColor.white.setFill()
ctx.fill(CGRect(origin: .zero, size: canvasSize))
UIImage(cgImage: cg, scale: 1, orientation: .up).draw(
in: CGRect(x: padding, y: padding,
width: contentSize.width, height: contentSize.height)
)
}
}
}
extension UIImage {

60
康康/Localizable.xcstrings
View File

@@ -1314,9 +1314,15 @@
}
}
}
},
"Apple Vision" : {
},
"Apple 健康里没有可导入的生日、性别、身高或血型。" : {
},
"Arm SME2" : {
},
"B 型" : {
"localizations" : {
@@ -1386,12 +1392,18 @@
},
"lo" : {
},
"Metal GPU · 兜底 / 对照" : {
},
"mmHg" : {
},
"mmol/L" : {
},
"MNN 在端侧 CPU 上以 Arm SME2 指令集加速 Qwen 推理(本地、不上云)。切换后下一次 AI 调用生效。" : {
},
"O 型" : {
"localizations" : {
@@ -1414,6 +1426,12 @@
}
}
}
},
"Qwen3.5-4B 多模态直接看图(MNN/MLX)" : {
},
"Qwen3.5-4B 已就绪" : {
},
"s" : {
@@ -1853,6 +1871,7 @@
}
},
"两个模型都已就绪" : {
"extractionState" : "stale",
"localizations" : {
"en" : {
"stringUnit" : {
@@ -3635,6 +3654,7 @@
}
},
"加" : {
"extractionState" : "stale",
"localizations" : {
"en" : {
"stringUnit" : {
@@ -4322,6 +4342,9 @@
}
}
}
},
"图片编码失败,手动补充" : {
},
"在「+ 新建 → 指标记录 → %@」记录一次" : {
"localizations" : {
@@ -4600,6 +4623,9 @@
},
"大" : {
},
"大模型直读" : {
},
"失眠" : {
"localizations" : {
@@ -5939,6 +5965,9 @@
},
"异常项快拍需要相机。去「设置 → 康康 → 相机」打开后再回来。" : {
},
"异常项拍照识别" : {
},
"强度" : {
"localizations" : {
@@ -7111,6 +7140,9 @@
}
}
}
},
"推理引擎" : {
},
"推理自检" : {
"localizations" : {
@@ -7618,6 +7650,7 @@
}
},
"文本解读 · 趋势 / 问答" : {
"extractionState" : "stale",
"localizations" : {
"en" : {
"stringUnit" : {
@@ -7638,6 +7671,12 @@
}
}
}
},
"文本解读 · 趋势 / 问答(MLX 兜底)" : {
},
"文本解读 + 拍照识别 · MNN + SME2 端侧加速" : {
},
"新建" : {
"localizations" : {
@@ -8617,6 +8656,15 @@
}
}
}
},
"本设备/模拟器不可用,自动回退 MLX" : {
},
"本设备不支持(需 A19/iPhone 17+)" : {
},
"本设备支持,MNN 已启用 SME2 加速" : {
},
"本设备未设置 Face ID 或密码" : {
"localizations" : {
@@ -8860,6 +8908,9 @@
}
}
}
},
"模型未就绪,请在模型管理下载或切回 Apple Vision" : {
},
"模型未就绪时 App 仍可使用,AI 功能会提示前往下载。" : {
"localizations" : {
@@ -9872,6 +9923,12 @@
}
}
}
},
"端侧 CPU + SME2 加速 · 挑战赛考核路径" : {
},
"端侧 CPU(本机无 SME2,NEON 回退)" : {
},
"第 %lld 轮 · 基于你刚才更新的文本 · %lld 条" : {
"localizations" : {
@@ -10014,6 +10071,9 @@
}
}
}
},
"系统 OCR + 文本模型解析" : {
},
"系统:iOS 17 或更新版本。" : {
"localizations" : {

2
康康/Services/HealthExportDialogue.swift
View File

@@ -7,7 +7,7 @@ struct HealthExportDialogueTurn: Identifiable, Hashable, Sendable {
var transcriptLabel: String {
switch self {
case .user: return String(appLoc: "患者")
case .user: return String(appLoc: "我")
case .assistant: return String(appLoc: "康康")
}
}

2
康康/Services/ModelDownloadService.swift
View File

@@ -58,7 +58,7 @@ final class ModelDownloadService {
}
func downloadAll() {
for kind in ModelKind.allCases { download(kind) }
for kind in ModelKind.userFacing { download(kind) }
}
/// 暂停下载。已下载的 .part 保留,下次从断点续传。

2
康康/Services/OCRService.swift
View File

@@ -23,7 +23,7 @@ enum OCRService {
let handler = VNImageRequestHandler(cgImage: cgImage, orientation: .up, options: [:])
do {
try handler.perform([request])
let obs = (request.results as? [VNRecognizedTextObservation]) ?? []
let obs = request.results ?? []
cont.resume(returning: assemble(obs))
} catch {
cont.resume(throwing: error)

6
康康Tests/ModelManifestTests.swift
View File

@@ -13,7 +13,7 @@ struct ModelManifestTests {
}
@Test func llmTotalBytesMatchesManifest() {
#expect(ModelManifest.totalBytes(for: .llm) == 1_749_079_691)
#expect(ModelManifest.totalBytes(for: .llm) == 3_061_129_077)
}
@Test func vlTotalBytesMatchesManifest() {
@@ -62,8 +62,8 @@ struct ModelManifestTests {
}
@Test func fileURLIsBaseSlashRepoSlashPath() {
let file = ModelFile(path: "config.json", bytes: 3_113)
let file = ModelFile(path: "config.json", bytes: 3_366)
let url = ModelManifest.fileURL(for: .llm, file: file)
#expect(url.absoluteString == "https://file.myv0.com/Qwen3.5-2B-4bit/config.json")
#expect(url.absoluteString == "https://file.myv0.com/Qwen3.5-4B-4bit/config.json")
}
}

18
康康Tests/QuickRegionRecognitionEngineTests.swift Normal file
View File

@@ -0,0 +1,18 @@
import Testing
@testable import 康康
struct QuickRegionRecognitionEngineTests {
@Test func defaultsToAppleVisionOCR() {
#expect(QuickRegionRecognitionEngine.defaultValue == .appleVision)
}
@Test func rawValuesAreStableForAppStorage() {
#expect(QuickRegionRecognitionEngine.appleVision.rawValue == "appleVision")
#expect(QuickRegionRecognitionEngine.qwenVL.rawValue == "qwenVL")
}
@Test func unknownStoredValueFallsBackToDefault() {
#expect(QuickRegionRecognitionEngine(storedValue: "missing") == .appleVision)
}
}

37
康康Tests/RegionImageCropperTests.swift
View File

@@ -112,4 +112,41 @@ final class RegionImageCropperTests: XCTestCase {
imageFrame: CGRect(x: 0, y: 0, width: 100, height: 100)),
.zero)
}
/// Qwen3-VL 对宽而矮的局部小图更容易在 processor 缩放后丢字。
/// 进入 VL 前应把短边放大并加白边,但不拉伸内容。
func testPrepareForQwenVLUpscalesWideShortCropAndAddsPadding() {
let image = solidImage(size: CGSize(width: 320, height: 80))
let prepared = RegionImageCropper.prepareForQwenVL(image)
XCTAssertGreaterThanOrEqual(prepared.size.height, 448)
XCTAssertGreaterThan(prepared.size.width, 320)
XCTAssertEqual(prepared.size.width / prepared.size.height,
4.0,
accuracy: 0.8,
"预处理应大致保留宽条内容比例,只允许白边造成轻微变化")
}
func testPrepareForQwenVLDoesNotEnlargePastLongEdgeLimit() {
let image = solidImage(size: CGSize(width: 5000, height: 900))
let prepared = RegionImageCropper.prepareForQwenVL(image)
XCTAssertLessThanOrEqual(max(prepared.size.width, prepared.size.height), 2400 + 128)
}
private func solidImage(size: CGSize) -> UIImage {
let format = UIGraphicsImageRendererFormat.default()
format.scale = 1
return UIGraphicsImageRenderer(size: size, format: format).image { ctx in
UIColor.white.setFill()
ctx.fill(CGRect(origin: .zero, size: size))
UIColor.black.setFill()
ctx.fill(CGRect(x: size.width * 0.1,
y: size.height * 0.35,
width: size.width * 0.8,
height: size.height * 0.3))
}
}
}