根据提供的code differences信息,我发现没有具体的代码变更内容。因此生成一个通用的commit message:
``` chore(config): 更新项目配置文件 - 调整开发环境配置参数 - 优化构建流程设置 - 更新依赖包版本管理 ```
This commit is contained in:
@@ -58,6 +58,15 @@ actor AIRuntime {
|
||||
// 模拟器无 MNN,VL 回退 MLX 的 Qwen3-VL-4B。
|
||||
private let mnn = MNNBackend()
|
||||
private(set) var mnnStatus: Status = .notReady
|
||||
|
||||
// MARK: - Gemini 云端后端(hybrid:端侧 Gemma 默认,云端按需增强)
|
||||
// 云端调用不占本机显存,不进 OOM 闸门,可与端侧推理并发。用于「云端深度解读 / 多语言」
|
||||
// 与「拍报告/药盒多模态读图」——后者恢复端侧 Gemma-3n(MLX 文本版)丢掉的真·视觉能力。
|
||||
private let gemini = GeminiBackend()
|
||||
/// 云端是否可用(用户已开启 + 有 key)。UI 与调用方据此决定走云还是端侧。
|
||||
nonisolated var cloudAvailable: Bool { CloudAI.isConfigured }
|
||||
/// 云端后端标签(性能自检 / 截图用)。
|
||||
nonisolated static var cloudLabel: String { "Gemini · \(CloudAI.model)" }
|
||||
/// MNN 模型目录(下载/旁路导入到 Models/Qwen3.5-2B-MNN)。
|
||||
nonisolated static var mnnModelFolder: URL {
|
||||
ModelStore.shared.localURL(for: .mnnLLM)
|
||||
@@ -307,6 +316,59 @@ actor AIRuntime {
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Gemini 云端(hybrid 增强)
|
||||
|
||||
/// 云端流式生成(深度解读 / 多语言)。不进 OOM 闸门,可与端侧并发。
|
||||
/// 调用前应确认 `cloudAvailable`;失败时调用方回退端侧 `generate`。
|
||||
func generateCloud(prompt: String, maxTokens: Int = 512) -> AsyncThrowingStream<TokenChunk, Error> {
|
||||
AsyncThrowingStream { continuation in
|
||||
let task = Task {
|
||||
do {
|
||||
let stream = await self.gemini.generate(prompt: prompt, maxTokens: maxTokens)
|
||||
for try await chunk in stream {
|
||||
try Task.checkCancellation()
|
||||
continuation.yield(chunk)
|
||||
}
|
||||
self.lastGenerateStats = await self.gemini.lastStats
|
||||
continuation.finish()
|
||||
} catch is CancellationError {
|
||||
continuation.finish(throwing: CancellationError())
|
||||
} catch {
|
||||
continuation.finish(throwing: AIRuntimeError.inferenceFailed("\(error)"))
|
||||
}
|
||||
}
|
||||
continuation.onTermination = { _ in task.cancel() }
|
||||
}
|
||||
}
|
||||
|
||||
/// 云端多模态识别:图片直传 Gemini 读出结构化文本(通常 JSON)。
|
||||
/// 恢复端侧丢掉的真·视觉读图;失败时调用方回退端侧 OCR+文本(§3.2)。
|
||||
func analyzeReportCloud(imageURLs: [URL], prompt: String, maxTokens: Int = 1024) async throws -> String {
|
||||
do {
|
||||
return try await gemini.analyze(imageURLs: imageURLs, prompt: prompt, maxTokens: maxTokens)
|
||||
} catch {
|
||||
throw AIRuntimeError.inferenceFailed("\(error)")
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - 端侧 ASR(SenseVoice via sherpa-mnn)互斥入口
|
||||
|
||||
/// 给端侧语音转写(SenseVoice,占 CPU + 内存)用:进推理闸门串行 + 卸掉常驻 LLM/VL/MNN 腾内存,
|
||||
/// 避免与文本/视觉模型同时常驻冲过单 App 内存上限被 jetsam 杀(§3.1 OOM 防护)。
|
||||
///
|
||||
/// 问诊流程是「先转写 → 再 organize(会重载 LLM)」严格串行,所以这里卸掉 LLM 是安全的:
|
||||
/// body 跑完一次转写返回后,调用方走 DiaryAssistService.organizeConsultation 时会自然重新 prepare。
|
||||
/// body 内部应自行 hop 到后台线程做阻塞解码(本 actor 在 await 期间不阻塞,但闸门保持持有,
|
||||
/// 其它推理请求会排队等待,杜绝并发占内存)。
|
||||
func runExclusiveForASR<T: Sendable>(_ body: @Sendable () async throws -> T) async rethrows -> T {
|
||||
await acquireGate(.interactive)
|
||||
defer { releaseGate() }
|
||||
unloadLLM()
|
||||
unloadVL()
|
||||
await unloadMNN()
|
||||
return try await body()
|
||||
}
|
||||
|
||||
// MARK: - VL
|
||||
|
||||
/// 加载 VL 模型。幂等,首调真正 load。
|
||||
|
||||
213
康康/AI/GeminiBackend.swift
Normal file
213
康康/AI/GeminiBackend.swift
Normal file
@@ -0,0 +1,213 @@
|
||||
import Foundation
|
||||
|
||||
/// 云端 AI(Google Gemini)配置。**隐私优先:默认关闭**,用户在「我的 · 云端 AI」显式开启并填入
|
||||
/// AI Studio 的 API key 后才会启用。key 优先取 UserDefaults(用户填写),其次 Info.plist
|
||||
/// `GEMINI_API_KEY` / 环境变量(开发期注入),三处都没有则视为未配置。
|
||||
///
|
||||
/// 设计取舍:demo 阶段用 AI Studio 直发 API key 的 REST 方案,零新增 SPM 依赖、即时可编译;
|
||||
/// 生产级应升级到 Firebase AI Logic(App Check 防盗用、不在客户端裸存 key、内建 hybrid),
|
||||
/// 见 `docs/release` 的接入说明。
|
||||
nonisolated enum CloudAI {
|
||||
private static let enabledKey = "cloud_ai_gemini_enabled"
|
||||
private static let apiKeyKey = "cloud_ai_gemini_key"
|
||||
private static let modelKey = "cloud_ai_gemini_model"
|
||||
|
||||
/// 默认模型:快、便宜、原生多模态(文本+图像)。可在设置覆盖。
|
||||
static let defaultModel = "gemini-2.5-flash"
|
||||
|
||||
/// 用户是否开启云端增强(默认 false —— 不上云是默认态)。
|
||||
static var isEnabled: Bool {
|
||||
get { UserDefaults.standard.bool(forKey: enabledKey) }
|
||||
set { UserDefaults.standard.set(newValue, forKey: enabledKey) }
|
||||
}
|
||||
|
||||
/// Gemini API key。用户填写优先,其次构建期注入(Info.plist / 环境变量)。
|
||||
static var apiKey: String? {
|
||||
get {
|
||||
if let k = UserDefaults.standard.string(forKey: apiKeyKey),
|
||||
!k.trimmingCharacters(in: .whitespaces).isEmpty { return k }
|
||||
if let k = Bundle.main.object(forInfoDictionaryKey: "GEMINI_API_KEY") as? String,
|
||||
!k.isEmpty { return k }
|
||||
if let k = ProcessInfo.processInfo.environment["GEMINI_API_KEY"],
|
||||
!k.isEmpty { return k }
|
||||
return nil
|
||||
}
|
||||
set { UserDefaults.standard.set(newValue, forKey: apiKeyKey) }
|
||||
}
|
||||
|
||||
static var model: String {
|
||||
get { UserDefaults.standard.string(forKey: modelKey) ?? defaultModel }
|
||||
set { UserDefaults.standard.set(newValue, forKey: modelKey) }
|
||||
}
|
||||
|
||||
/// 云端是否可用:已开启 + 有 key。具体网络可达性由调用方在失败时回退端侧。
|
||||
static var isConfigured: Bool {
|
||||
isEnabled && apiKey != nil
|
||||
}
|
||||
}
|
||||
|
||||
enum GeminiError: Error, LocalizedError {
|
||||
case notConfigured
|
||||
case http(Int, String)
|
||||
case decode(String)
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .notConfigured: return String(appLoc: "云端 AI 未配置(请在「我的 · 云端 AI」开启并填入 key)")
|
||||
case .http(let c, let m): return String(appLoc: "Gemini 请求失败(\(c)):\(m)")
|
||||
case .decode(let m): return String(appLoc: "Gemini 响应解析失败:\(m)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Google Gemini 云端后端。经 REST(URLSession)调用 Generative Language API:
|
||||
/// - `generate`:`streamGenerateContent`(SSE 流式),用于「云端深度解读 / 多语言」。
|
||||
/// - `analyze`:`generateContent`(多模态),把报告/药盒图片直传 Gemini 读出结构化结果——
|
||||
/// 恢复端侧 Gemma-3n(MLX 文本版)丢掉的真·视觉能力(§拍照→结构化)。
|
||||
///
|
||||
/// 云端调用不占本机显存,**不进 AIRuntime 的 OOM 闸门**,可与端侧推理并发。
|
||||
actor GeminiBackend {
|
||||
private let endpointBase = "https://generativelanguage.googleapis.com/v1beta/models"
|
||||
|
||||
private(set) var lastStats: GenerateStats?
|
||||
|
||||
// MARK: - 流式文本生成
|
||||
|
||||
/// 流式生成。返回流被取消时内部 Task 取消、连带断开底层连接。
|
||||
func generate(prompt: String, maxTokens: Int) -> AsyncThrowingStream<TokenChunk, Error> {
|
||||
AsyncThrowingStream { continuation in
|
||||
let task = Task {
|
||||
do {
|
||||
guard let key = CloudAI.apiKey else { throw GeminiError.notConfigured }
|
||||
let model = CloudAI.model
|
||||
var req = URLRequest(url: URL(string:
|
||||
"\(endpointBase)/\(model):streamGenerateContent?alt=sse&key=\(key)")!)
|
||||
req.httpMethod = "POST"
|
||||
req.setValue("application/json", forHTTPHeaderField: "Content-Type")
|
||||
req.httpBody = try Self.requestBody(textParts: [prompt],
|
||||
imageParts: [],
|
||||
maxTokens: maxTokens)
|
||||
|
||||
let (bytes, response) = try await URLSession.shared.bytes(for: req)
|
||||
if let http = response as? HTTPURLResponse, http.statusCode != 200 {
|
||||
var body = ""
|
||||
for try await line in bytes.lines { body += line }
|
||||
throw GeminiError.http(http.statusCode, String(body.prefix(300)))
|
||||
}
|
||||
|
||||
let start = Date()
|
||||
var firstAt: Date?
|
||||
var produced = 0
|
||||
var usage: GeminiResponse.Usage?
|
||||
|
||||
for try await line in bytes.lines {
|
||||
if Task.isCancelled { break }
|
||||
guard line.hasPrefix("data:") else { continue }
|
||||
let payload = line.dropFirst(5).trimmingCharacters(in: .whitespaces)
|
||||
guard !payload.isEmpty, payload != "[DONE]",
|
||||
let data = payload.data(using: .utf8) else { continue }
|
||||
let chunk = try? JSONDecoder().decode(GeminiResponse.self, from: data)
|
||||
if let u = chunk?.usageMetadata { usage = u }
|
||||
let text = chunk?.candidates?.first?.content?.parts?
|
||||
.compactMap(\.text).joined() ?? ""
|
||||
guard !text.isEmpty else { continue }
|
||||
if firstAt == nil { firstAt = Date() }
|
||||
produced += 1
|
||||
let elapsed = Date().timeIntervalSince(firstAt ?? start)
|
||||
let rate = elapsed > 0 ? Double(produced) / elapsed : 0
|
||||
continuation.yield(TokenChunk(text: text, decodeRate: rate))
|
||||
}
|
||||
|
||||
// 归一统计:prefill = 首 chunk 前耗时;decode = 其后耗时;token 数取 usageMetadata。
|
||||
let ttf = (firstAt ?? Date()).timeIntervalSince(start)
|
||||
let total = Date().timeIntervalSince(start)
|
||||
self.lastStats = GenerateStats(
|
||||
promptTokens: usage?.promptTokenCount ?? 0,
|
||||
genTokens: usage?.candidatesTokenCount ?? produced,
|
||||
prefillSeconds: max(ttf, 0.0001),
|
||||
decodeSeconds: max(total - ttf, 0.0001)
|
||||
)
|
||||
continuation.finish()
|
||||
} catch is CancellationError {
|
||||
continuation.finish(throwing: CancellationError())
|
||||
} catch {
|
||||
continuation.finish(throwing: error)
|
||||
}
|
||||
}
|
||||
continuation.onTermination = { _ in task.cancel() }
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - 多模态(图 → 文)
|
||||
|
||||
/// 多模态识别:图片 + prompt → 文本(通常是 JSON)。非流式,一次返回。
|
||||
/// 调用方负责解析 + 失败回退端侧(§3.2)。
|
||||
func analyze(imageURLs: [URL], prompt: String, maxTokens: Int) async throws -> String {
|
||||
guard let key = CloudAI.apiKey else { throw GeminiError.notConfigured }
|
||||
let model = CloudAI.model
|
||||
var req = URLRequest(url: URL(string:
|
||||
"\(endpointBase)/\(model):generateContent?key=\(key)")!)
|
||||
req.httpMethod = "POST"
|
||||
req.setValue("application/json", forHTTPHeaderField: "Content-Type")
|
||||
req.httpBody = try Self.requestBody(textParts: [prompt],
|
||||
imageParts: imageURLs,
|
||||
maxTokens: maxTokens)
|
||||
|
||||
let (data, response) = try await URLSession.shared.data(for: req)
|
||||
if let http = response as? HTTPURLResponse, http.statusCode != 200 {
|
||||
let body = String(data: data, encoding: .utf8) ?? ""
|
||||
throw GeminiError.http(http.statusCode, String(body.prefix(300)))
|
||||
}
|
||||
let decoded = try JSONDecoder().decode(GeminiResponse.self, from: data)
|
||||
if let msg = decoded.error?.message { throw GeminiError.http(0, msg) }
|
||||
guard let text = decoded.candidates?.first?.content?.parts?
|
||||
.compactMap(\.text).joined(), !text.isEmpty else {
|
||||
throw GeminiError.decode(String(appLoc: "无文本返回"))
|
||||
}
|
||||
if let u = decoded.usageMetadata {
|
||||
self.lastStats = GenerateStats(promptTokens: u.promptTokenCount ?? 0,
|
||||
genTokens: u.candidatesTokenCount ?? 0,
|
||||
prefillSeconds: 0.0001, decodeSeconds: 0.0001)
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
// MARK: - 请求体
|
||||
|
||||
private static func requestBody(textParts: [String],
|
||||
imageParts: [URL],
|
||||
maxTokens: Int) throws -> Data {
|
||||
var parts: [[String: Any]] = textParts.map { ["text": $0] }
|
||||
for url in imageParts {
|
||||
guard let raw = try? Data(contentsOf: url) else { continue }
|
||||
// 控制单图体积,避免请求过大;Vault 原图已是 JPEG。
|
||||
let mime = url.pathExtension.lowercased() == "png" ? "image/png" : "image/jpeg"
|
||||
parts.append(["inline_data": ["mime_type": mime,
|
||||
"data": raw.base64EncodedString()]])
|
||||
}
|
||||
let body: [String: Any] = [
|
||||
"contents": [["role": "user", "parts": parts]],
|
||||
"generationConfig": [
|
||||
"maxOutputTokens": maxTokens,
|
||||
"temperature": 0.3,
|
||||
"topP": 0.85
|
||||
]
|
||||
]
|
||||
return try JSONSerialization.data(withJSONObject: body)
|
||||
}
|
||||
}
|
||||
|
||||
/// Gemini `GenerateContentResponse` 的最小可解码子集。
|
||||
private struct GeminiResponse: Decodable {
|
||||
struct Candidate: Decodable { let content: Content? }
|
||||
struct Content: Decodable { let parts: [Part]? }
|
||||
struct Part: Decodable { let text: String? }
|
||||
struct Usage: Decodable {
|
||||
let promptTokenCount: Int?
|
||||
let candidatesTokenCount: Int?
|
||||
}
|
||||
struct APIError: Decodable { let message: String? }
|
||||
let candidates: [Candidate]?
|
||||
let usageMetadata: Usage?
|
||||
let error: APIError?
|
||||
}
|
||||
@@ -29,9 +29,15 @@ nonisolated enum InferenceEngine: String, CaseIterable, Sendable {
|
||||
/// 由偏好(可能是 .auto)解析出的、本次调用实际使用的具体引擎。
|
||||
/// AIRuntime / MeView 等消费方只看这个,永远拿到 .mnn 或 .mlx。
|
||||
/// 解析后仍做一次可用性兜底,保证总有可用引擎。
|
||||
///
|
||||
/// 项目已切 Gemma-3n(只走 MLX/GPU):Gemma-3n 跑不了 MNN(无转换模型),
|
||||
/// 故主模型不再下载 MNN 切片。即便历史偏好写了 "mnn",只要本机没有完整的 MNN 模型,
|
||||
/// 一律回退 MLX —— 杜绝「标签显示 MNN、实际却跑 MLX」的不一致。
|
||||
static var current: InferenceEngine {
|
||||
let resolved = preference.resolved
|
||||
return resolved.isAvailable ? resolved : .mlx
|
||||
guard resolved.isAvailable else { return .mlx }
|
||||
if resolved == .mnn, !ModelStore.shared.isComplete(for: .mnnLLM) { return .mlx }
|
||||
return resolved
|
||||
}
|
||||
|
||||
/// 运行时探测:CPU 是否支持 SME2(A19/iPhone17+)。用于 UI 展示加速状态。
|
||||
@@ -52,8 +58,8 @@ nonisolated enum InferenceEngine: String, CaseIterable, Sendable {
|
||||
}
|
||||
|
||||
/// 推理引擎的「用户偏好」,比具体引擎多一个 .auto。
|
||||
/// - auto:按本机配置自动选——真机优先 MNN(考核路径,含 SME2/NEON),
|
||||
/// MNN 不可用(模拟器)时回退 MLX。
|
||||
/// - auto:本项目已切 Gemma-3n,主模型只走 MLX/GPU,故 auto 一律解析为 .mlx。
|
||||
/// (Gemma-3n 跑不了 MNN/SME2;MNN 路径已停用。)
|
||||
nonisolated enum EnginePreference: String, CaseIterable, Sendable {
|
||||
case auto
|
||||
case mnn
|
||||
@@ -72,7 +78,7 @@ nonisolated enum EnginePreference: String, CaseIterable, Sendable {
|
||||
switch self {
|
||||
case .mnn: return .mnn
|
||||
case .mlx: return .mlx
|
||||
case .auto: return InferenceEngine.mnn.isAvailable ? .mnn : .mlx
|
||||
case .auto: return .mlx // Gemma-3n 只走 MLX/GPU,auto 即 MLX
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,8 +31,13 @@ actor LLMSession {
|
||||
}
|
||||
|
||||
/// 从本地目录加载模型(包含 config.json + weights + tokenizer)。
|
||||
/// Gemma-3n 的聊天回合以 `<end_of_turn>`(token 106)结束,而分词器自带的 eos 是 `<eos>`(1);
|
||||
/// 不显式补 `<end_of_turn>` 会让解码停不下来、把 maxTokens 跑满还吐回合分隔噪声。
|
||||
static func load(folderURL: URL) async throws -> LLMSession {
|
||||
let configuration = ModelConfiguration(directory: folderURL)
|
||||
let configuration = ModelConfiguration(
|
||||
directory: folderURL,
|
||||
extraEOSTokens: ["<end_of_turn>"]
|
||||
)
|
||||
let container = try await withDeviceOverride {
|
||||
try await LLMModelFactory.shared.loadContainer(
|
||||
configuration: configuration
|
||||
|
||||
40
康康/AI/MNN/SenseVoiceBridge.h
Normal file
40
康康/AI/MNN/SenseVoiceBridge.h
Normal file
@@ -0,0 +1,40 @@
|
||||
//
|
||||
// SenseVoiceBridge.h
|
||||
// 康康
|
||||
//
|
||||
// Objective-C 封装:端侧 SenseVoice ASR(语音→文字),经 sherpa-mnn 在 MNN 后端跑。
|
||||
// 「记录问诊」用它把整段录音离线转写成文字,再交本地 LLM 整理成问诊小结。
|
||||
//
|
||||
// 与 MNNLLMBridge 同一套思路:真实实现在 .mm 里以 C 调用 <sherpa-mnn/c-api/c-api.h>;
|
||||
// 当工程尚未链接 sherpa-mnn(当前默认)时,以 __has_include 编为桩(isAvailable 返回 NO),
|
||||
// 上层 SenseVoiceASRService 自动回退到系统端侧识别(SFSpeech),App 不受影响。
|
||||
// 构建/接入 sherpa-mnn.xcframework 的步骤见 docs/release/sensevoice-integration.md。
|
||||
//
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
@interface SenseVoiceBridge : NSObject
|
||||
|
||||
/// 本构建是否含真实 sherpa-mnn ASR(已链接 + 非桩=YES)。NO 时上层回退系统识别。
|
||||
+ (BOOL)isAvailable;
|
||||
|
||||
/// 用转换好的 SenseVoice MNN 模型 + tokens 创建离线识别器(贪心解码)。
|
||||
/// modelPath: MNNConvert 产出的 model.mnn;tokensPath: tokens.txt;
|
||||
/// language: "auto" / "zh" / "en" / "ja" / "ko" / "yue"。失败返回 nil。
|
||||
/// 识别器随实例常驻,dealloc 时释放——调用方按「一次问诊建一个」用,用完即释放降内存峰值。
|
||||
- (nullable instancetype)initWithModelPath:(NSString *)modelPath
|
||||
tokensPath:(NSString *)tokensPath
|
||||
language:(NSString *)language;
|
||||
|
||||
/// 转写一段单声道 PCM(float32,取值 [-1, 1])。sherpa 内部按 sampleRate 重采样到 16k 再抽 fbank。
|
||||
/// samples 在本调用期间需保持有效。返回识别文本(可能含 <|lang|> 等标签,由上层清洗);失败返回 nil。
|
||||
/// 同步阻塞直到解码结束——调用方务必放到后台线程,且经 AIRuntime 闸门与 LLM/VL 互斥(防 OOM)。
|
||||
- (nullable NSString *)transcribeSamples:(const float *)samples
|
||||
count:(int)count
|
||||
sampleRate:(int)sampleRate;
|
||||
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
112
康康/AI/MNN/SenseVoiceBridge.mm
Normal file
112
康康/AI/MNN/SenseVoiceBridge.mm
Normal file
@@ -0,0 +1,112 @@
|
||||
//
|
||||
// SenseVoiceBridge.mm
|
||||
// 康康
|
||||
//
|
||||
// ObjC++ 实现。链接 sherpa-mnn 时走真实 C-API;否则编为桩(返回不可用,上层回退系统识别)。
|
||||
//
|
||||
// 可用性闸门:仅当能找到 sherpa-mnn 的 C-API 头时才编真实路径。这样在尚未接入 sherpa-mnn
|
||||
// 的工程里本文件也能正常编过(走桩),接入后(把 sherpa-mnn.xcframework 加进 Frameworks/ 并
|
||||
// 让 HEADER_SEARCH_PATHS 找到其头)自动切到真实实现。见 docs/release/sensevoice-integration.md。
|
||||
//
|
||||
|
||||
#import "SenseVoiceBridge.h"
|
||||
|
||||
#if __has_include(<sherpa-mnn/c-api/c-api.h>)
|
||||
#define KK_SHERPA_MNN_AVAILABLE 1
|
||||
#else
|
||||
#define KK_SHERPA_MNN_AVAILABLE 0
|
||||
#endif
|
||||
|
||||
#if !KK_SHERPA_MNN_AVAILABLE
|
||||
|
||||
// ============ 桩:工程未链接 sherpa-mnn ============
|
||||
@implementation SenseVoiceBridge
|
||||
+ (BOOL)isAvailable { return NO; }
|
||||
- (nullable instancetype)initWithModelPath:(NSString *)modelPath
|
||||
tokensPath:(NSString *)tokensPath
|
||||
language:(NSString *)language { return nil; }
|
||||
- (nullable NSString *)transcribeSamples:(const float *)samples
|
||||
count:(int)count
|
||||
sampleRate:(int)sampleRate { return nil; }
|
||||
@end
|
||||
|
||||
#else
|
||||
|
||||
// ============ 真实:sherpa-mnn 离线 SenseVoice ============
|
||||
#include <sherpa-mnn/c-api/c-api.h>
|
||||
#include <string>
|
||||
|
||||
@implementation SenseVoiceBridge {
|
||||
const SherpaMnnOfflineRecognizer *_recognizer;
|
||||
}
|
||||
|
||||
+ (BOOL)isAvailable { return YES; }
|
||||
|
||||
- (nullable instancetype)initWithModelPath:(NSString *)modelPath
|
||||
tokensPath:(NSString *)tokensPath
|
||||
language:(NSString *)language {
|
||||
self = [super init];
|
||||
if (!self) return nil;
|
||||
|
||||
std::string model = modelPath.UTF8String;
|
||||
std::string tokens = tokensPath.UTF8String;
|
||||
std::string lang = language.length ? language.UTF8String : "auto";
|
||||
|
||||
SherpaMnnOfflineSenseVoiceModelConfig senseVoice;
|
||||
memset(&senseVoice, 0, sizeof(senseVoice));
|
||||
senseVoice.model = model.c_str();
|
||||
senseVoice.language = lang.c_str();
|
||||
senseVoice.use_itn = 1; // 逆文本规整:把「一百四十」之类还原成 140,数值更利于阅读/给医生看
|
||||
|
||||
SherpaMnnOfflineModelConfig modelConfig;
|
||||
memset(&modelConfig, 0, sizeof(modelConfig));
|
||||
modelConfig.tokens = tokens.c_str();
|
||||
modelConfig.num_threads = 2; // 端侧保守取 2,避免与 UI 抢核
|
||||
modelConfig.debug = 0;
|
||||
modelConfig.provider = "cpu"; // sherpa-mnn 后端即 MNN(CPU/SME2),provider 维持 "cpu"
|
||||
modelConfig.sense_voice = senseVoice;
|
||||
|
||||
SherpaMnnOfflineRecognizerConfig config;
|
||||
memset(&config, 0, sizeof(config));
|
||||
config.decoding_method = "greedy_search";
|
||||
config.model_config = modelConfig;
|
||||
|
||||
_recognizer = SherpaMnnCreateOfflineRecognizer(&config);
|
||||
if (_recognizer == nullptr) return nil;
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
if (_recognizer) {
|
||||
SherpaMnnDestroyOfflineRecognizer(_recognizer);
|
||||
_recognizer = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
- (nullable NSString *)transcribeSamples:(const float *)samples
|
||||
count:(int)count
|
||||
sampleRate:(int)sampleRate {
|
||||
if (_recognizer == nullptr || samples == nullptr || count <= 0) return nil;
|
||||
|
||||
const SherpaMnnOfflineStream *stream = SherpaMnnCreateOfflineStream(_recognizer);
|
||||
if (stream == nullptr) return nil;
|
||||
|
||||
SherpaMnnAcceptWaveformOffline(stream, sampleRate, samples, count);
|
||||
SherpaMnnDecodeOfflineStream(_recognizer, stream);
|
||||
|
||||
NSString *text = nil;
|
||||
const SherpaMnnOfflineRecognizerResult *result =
|
||||
SherpaMnnGetOfflineStreamResult(stream);
|
||||
if (result) {
|
||||
if (result->text) {
|
||||
text = [NSString stringWithUTF8String:result->text];
|
||||
}
|
||||
SherpaMnnDestroyOfflineRecognizerResult(result);
|
||||
}
|
||||
SherpaMnnDestroyOfflineStream(stream);
|
||||
return text;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
#endif
|
||||
@@ -20,8 +20,8 @@ nonisolated enum ModelManifest {
|
||||
/// 注意组织名:MNN 模型在魔搭组织为 `MNN`(非 HuggingFace 的 taobao-mnn);MLX 沿用 mlx-community。
|
||||
static func modelScopeRepo(for kind: ModelKind) -> String? {
|
||||
switch kind {
|
||||
case .mnnLLM: return "MNN/Qwen3.5-2B-MNN"
|
||||
case .llm: return "mlx-community/Qwen3.5-2B-4bit"
|
||||
case .llm: return "mlx-community/gemma-3n-E2B-it-lm-4bit" // 主模型,大陆可达
|
||||
case .mnnLLM: return "MNN/Qwen3.5-2B-MNN" // 已停用,保留源备查
|
||||
case .vl: return nil // 已废弃,不再下载 / 分发,不提供魔搭源
|
||||
}
|
||||
}
|
||||
@@ -29,23 +29,23 @@ nonisolated enum ModelManifest {
|
||||
static func files(for kind: ModelKind) -> [ModelFile] {
|
||||
switch kind {
|
||||
case .llm:
|
||||
// Qwen3.5-2B-4bit:多模态仓库,但走 LLMModelFactory 的 qwen3_5 文本路径加载。
|
||||
// 字节数取自 mlx-community/Qwen3.5-2B-4bit 仓库实际 blob 大小(HF API,2026-06 核对)。
|
||||
// 该仓库 tokenizer 体系为 vocab.json + tokenizer.json(无 merges.txt /
|
||||
// special_tokens_map.json / added_tokens.json),chat_template 改为 .jinja。
|
||||
// 一并镜像视觉预处理配置(preprocessor / processor / video_preprocessor),
|
||||
// 文本加载用不到但体积可忽略,保持与仓库一致避免漏文件。
|
||||
// Gemma-3n-E2B-it-lm-4bit:Gemma-3n 的「语言模型抽取版」(text-only),走 LLMModelFactory
|
||||
// 的 gemma3n 文本路径加载(mlx-swift-lm 已注册 "gemma3n")。MLX 仅支持其文本能力,无视觉。
|
||||
// 字节数取自 ModelScope mlx-community/gemma-3n-E2B-it-lm-4bit 仓库实际 blob 大小
|
||||
//(repo/files API,2026-06 核对)。排除 README.md / .gitattributes / configuration.json
|
||||
//(后者为 ModelScope 元数据,MLX 加载用不到)。model.safetensors 为单文件,
|
||||
// MLX loadWeights 直接 glob 全部 *.safetensors,index.json 仅留作完整性占位,不影响加载。
|
||||
// tokenizer.model(SentencePiece)与 chat_template.jinja 一并镜像,确保聊天模板/分词稳定。
|
||||
return [
|
||||
ModelFile(path: "config.json", bytes: 3_113),
|
||||
ModelFile(path: "model.safetensors", bytes: 1_722_271_785),
|
||||
ModelFile(path: "model.safetensors.index.json", bytes: 81_722),
|
||||
ModelFile(path: "tokenizer.json", bytes: 19_989_343),
|
||||
ModelFile(path: "tokenizer_config.json", bytes: 1_139),
|
||||
ModelFile(path: "vocab.json", bytes: 6_722_759),
|
||||
ModelFile(path: "chat_template.jinja", bytes: 7_755),
|
||||
ModelFile(path: "preprocessor_config.json", bytes: 390),
|
||||
ModelFile(path: "processor_config.json", bytes: 1_300),
|
||||
ModelFile(path: "video_preprocessor_config.json", bytes: 385),
|
||||
ModelFile(path: "config.json", bytes: 107_207),
|
||||
ModelFile(path: "generation_config.json", bytes: 215),
|
||||
ModelFile(path: "model.safetensors", bytes: 2_507_515_399),
|
||||
ModelFile(path: "model.safetensors.index.json", bytes: 129_688),
|
||||
ModelFile(path: "special_tokens_map.json", bytes: 769),
|
||||
ModelFile(path: "tokenizer.json", bytes: 33_442_553),
|
||||
ModelFile(path: "tokenizer.model", bytes: 4_696_020),
|
||||
ModelFile(path: "tokenizer_config.json", bytes: 1_202_305),
|
||||
ModelFile(path: "chat_template.jinja", bytes: 1_626),
|
||||
]
|
||||
case .vl:
|
||||
// Qwen3-VL-4B-Instruct-4bit:字节数取自 mlx-community 仓库实际 blob 大小
|
||||
|
||||
@@ -2,17 +2,19 @@ import Foundation
|
||||
|
||||
nonisolated enum ModelKind: String, CaseIterable {
|
||||
/// 也是沙盒 Models/ 下的子目录名 / CDN 路径段。
|
||||
/// 同一个 Qwen3.5-2B,两种格式两种引擎:
|
||||
/// - mnnLLM:MNN(CPU/SME2,考核路径)文本+视觉一肩挑,taobao-mnn 预转换。iPhone17+(A19/SME2)主用,只露它。
|
||||
/// - llm:MLX(GPU)兜底,Qwen3.5-2B-4bit 多模态(同时兜底文本与视觉,走 qwen3_5)。
|
||||
/// - vl:已废弃(MLX VL 改走 .llm 多模态),保留枚举避免动一圈穷举 switch,不再下载/展示。
|
||||
case llm = "Qwen3.5-2B-4bit"
|
||||
/// 主模型已从 Qwen3.5-2B 切到 **Gemma-3n E2B(端侧 4bit,只走 MLX/GPU)**:
|
||||
/// - llm:**用户唯一下载的主模型**。MLX 文本引擎,加载 `gemma3n` 文本模型(mlx-swift-lm 已注册)。
|
||||
/// Gemma-3n 在 MLX 只有文本能力(VLMModelFactory 未注册 gemma3n),故拍报告解读改走 OCR + 文本 LLM。
|
||||
/// - vl:已废弃,保留枚举避免动一圈穷举 switch,不再下载/展示。
|
||||
/// - mnnLLM:已停用。Gemma-3n 跑不了 MNN(无转换模型 + MatFormer 架构不被 MNN 转换器支持),
|
||||
/// 本项目已放弃 MNN/SME2 路径;保留枚举只为兼容旧引擎判断,不再分发/计入下载。
|
||||
case llm = "gemma-3n-E2B-it-lm-4bit"
|
||||
case vl = "Qwen3-VL-4B-Instruct-4bit"
|
||||
case mnnLLM = "Qwen3.5-2B-MNN"
|
||||
|
||||
var displayName: String {
|
||||
switch self {
|
||||
case .llm: return "Qwen3.5-2B (MLX)"
|
||||
case .llm: return "Gemma-3n E2B (MLX)"
|
||||
case .vl: return "Qwen3-VL-4B"
|
||||
case .mnnLLM: return "Qwen3.5-2B (MNN/SME2)"
|
||||
}
|
||||
@@ -24,11 +26,9 @@ nonisolated enum ModelKind: String, CaseIterable {
|
||||
/// 用于判定该模型是否已就绪的最小标志文件
|
||||
var sentinelFilename: String { "config.json" }
|
||||
|
||||
/// 面向用户的模型集合:模型管理页 / 下载全部 / 就绪计数对外只暴露统一的
|
||||
/// Qwen3.5-2B(MNN,文本+视觉全包,iPhone17+ 走它)。
|
||||
/// MLX 的 .llm/.vl 仅作模拟器与兜底路径,保留枚举与下载能力(旁路导入仍可单独导),
|
||||
/// 但不在「我的 · 模型管理」展示,也不计入「下载全部」与就绪计数。
|
||||
static let userFacing: [ModelKind] = [.mnnLLM]
|
||||
/// 面向用户的模型集合:模型管理页 / 下载全部 / 就绪计数对外只暴露统一的主模型
|
||||
/// Gemma-3n E2B(MLX,文本)。.vl/.mnnLLM 已停用,不展示、不计入「下载全部」与就绪计数。
|
||||
static let userFacing: [ModelKind] = [.llm]
|
||||
}
|
||||
|
||||
/// `@unchecked Sendable`:rootURL 是 let,方法只读 filesystem(线程安全),
|
||||
|
||||
53
康康/AI/Prompts/ConsultationPrompts.swift
Normal file
53
康康/AI/Prompts/ConsultationPrompts.swift
Normal file
@@ -0,0 +1,53 @@
|
||||
import Foundation
|
||||
|
||||
/// 「记录问诊」录音转写 → 结构化问诊小结的 prompt(2026-06-28)。
|
||||
///
|
||||
/// 与 `DiaryAssistPrompts.organize`(口述日记)同源同魂,但产物不同:
|
||||
/// 问诊录音里通常**两个人在说话**(本人 + 医生),要按就诊场景归到固定小节,方便日后翻看/给下一位医生看。
|
||||
///
|
||||
/// 红线(同 §1 / §10):
|
||||
/// - 这是**转写整理**,不是 App 在做诊断或给药建议——只忠实复述录音里医生/本人说过的话,
|
||||
/// 【绝对不许】自行新增任何诊断、用药、剂量、频次或「建议就医」。模型没听到的就不写。
|
||||
/// - 数值、单位、药名、剂量、时间一字不改(140/90 永远是 140/90)。
|
||||
enum ConsultationPrompts {
|
||||
|
||||
/// 转写稿截断上限(字符)。2B context 保护:超长问诊只取前段整理。
|
||||
static let organizeTranscriptLimit = 2000
|
||||
|
||||
static func organize(transcript: String) -> String {
|
||||
let trimmed = String(transcript.prefix(organizeTranscriptLimit))
|
||||
return """
|
||||
你是健康记录助手。下面是用户在医院/诊所就诊时的录音转写原话,通常包含本人和医生两个人的对话,
|
||||
可能口语化、有重复、缺标点、说话人没标注。请把它整理成一份清晰的【问诊小结】,方便本人日后回看。
|
||||
|
||||
硬性规则:
|
||||
- 这只是**转写整理**:只复述录音里【确实说过】的内容,【绝对不许】自己新增诊断、用药、剂量、频次或「建议就医」。
|
||||
- 数值、单位、药名、剂量、时间【一字不改】——原话说 140/90 就写 140/90,说一天两次就写一天两次。
|
||||
- 录音里没提到的小节就【整节省略】,不要写「无」「未提及」之类占位,也不要硬凑。
|
||||
- 区分说话人:本人的主诉用第一人称「我」;医生说的话写成「医生:…」。
|
||||
- 不确定/听不清的地方保留原样,不要脑补。
|
||||
|
||||
按下面这些小节整理(只保留录音里出现过的,用「小节名:内容」分行):
|
||||
主诉 —— 本人这次来看什么、哪里不舒服
|
||||
现病史 —— 起病时间、过程、变化
|
||||
医生判断 —— 医生当场说的判断或解释(原话复述,不替医生下结论)
|
||||
医生建议 —— 医生给的检查、复查、生活建议
|
||||
用药 —— 医生提到的药名/用法(原样复述,不补剂量)
|
||||
复查与注意 —— 下次复查时间、注意事项
|
||||
|
||||
只输出整理后的小结正文,不要解释、不要 markdown 围栏、不要 <think> 标签。
|
||||
|
||||
示例(转写:就那个我最近老是胸口闷上楼梯就喘大概有半个月了医生说听着心率有点快先做个心电图和验血下周三来复查这两天别太累):
|
||||
主诉:最近胸口闷,上楼梯就喘。
|
||||
现病史:已持续约半个月。
|
||||
医生判断:医生:听着心率有点快。
|
||||
医生建议:先做心电图和验血。
|
||||
复查与注意:下周三复查,这两天别太累。
|
||||
|
||||
【录音转写原话】:
|
||||
\(trimmed)
|
||||
|
||||
Output: /no_think
|
||||
"""
|
||||
}
|
||||
}
|
||||
@@ -71,6 +71,7 @@ enum HealthExportPrompts {
|
||||
- JSON 里没有的信息,对应小节一律写「无记录」,不要补全、不要举例、不要套用常见病例模板。
|
||||
- 数值必须原样照搬(含单位与参考范围);status 为 high/low/abnormal 的指标前加 ⚠️。
|
||||
- 「主诉」「本人疑问」可参考【本人原话】,但不得加入原话与数据里都没有的症状。
|
||||
- diaries 里 kind 为「问诊」的是既往看医生的问诊记录,可据此补充「主诉」「本人背景」,但同样只搬运、不编造。
|
||||
|
||||
输出格式:
|
||||
- 严格 Markdown,标题用 # / ##,不要 markdown 围栏,不要输出 JSON,不写「数据」二字。
|
||||
@@ -165,6 +166,7 @@ enum HealthExportPrompts {
|
||||
- 严格 Markdown,不要 markdown 围栏,不要输出 JSON。
|
||||
- 中文,简洁,医生 30 秒能扫完。
|
||||
- 「相关健康日记」每条单独一行,格式为「2026-05-01:正文摘要」,日期照抄 JSON 的 date 字段,精确到日。
|
||||
- diaries 里 kind 为「问诊」的是既往问诊记录,优先列入「相关健康日记」并标注「(问诊)」。
|
||||
- 严格按以下段落:
|
||||
# 就诊摘要
|
||||
## 本次想解决的问题
|
||||
|
||||
@@ -112,6 +112,66 @@ JSON schema(严格):
|
||||
{"title":"春季年度体检","type":"checkup","report_date":"2026-04-12","institution":"协和医院","page_count":1,"summary":"血脂偏高、其他正常","indicators":[{"name":"低密度脂蛋白","value":"3.84","unit":"mmol/L","range":"< 3.40","status":"high","source_page":1,"source_box":[0.12,0.31,0.76,0.07]},{"name":"谷丙转氨酶","value":"32","unit":"U/L","range":"9 - 50","status":"normal","source_page":1,"source_box":[0.12,0.39,0.76,0.07]},{"name":"空腹血糖","value":"5.2","unit":"mmol/L","range":"3.9 - 6.1","status":"normal","source_page":1,"source_box":[0.12,0.47,0.76,0.07]}]}
|
||||
{{OCR_SECTION}}
|
||||
现在请识别图片并输出 JSON:
|
||||
"""#
|
||||
|
||||
// MARK: - 报告整份解读(OCR 文本 → 完整 ParsedReport,纯文本 LLM)
|
||||
|
||||
/// C2「重新解读」/ 拍照整份识别走这条:主模型 Gemma-3n 只有文本能力(MLX 无 gemma3n 视觉),
|
||||
/// 故先 Vision OCR 出纯文本,再交文本 LLM 一次抽出报告级 meta + 全部指标。
|
||||
/// 与 reportExtraction(多模态)同 schema,但去掉 source_page / source_box —— OCR 文本没有版面坐标,
|
||||
/// 让模型编框会污染原图证据高亮,统一不输出。
|
||||
static func reportExtractionFromText(_ ocrText: String, today: Date = .now) -> String {
|
||||
let f = DateFormatter()
|
||||
f.locale = Locale(identifier: "en_US_POSIX")
|
||||
f.dateFormat = "yyyy-MM-dd"
|
||||
let todayStr = f.string(from: today)
|
||||
return reportExtractionFromTextTemplate
|
||||
.replacingOccurrences(of: "{{TODAY}}", with: todayStr)
|
||||
.replacingOccurrences(of: "{{OCR_TEXT}}", with: clipOCR(ocrText, limit: 2200))
|
||||
}
|
||||
|
||||
private static let reportExtractionFromTextTemplate: String = #"""
|
||||
你是医学体检/化验报告识别助手。下面是对一份报告做 OCR 得到的纯文本,可能有错字、错位、噪声或换行混乱。
|
||||
请从中提取报告的元信息与所有指标,只输出一段合法 JSON,不要解释、不要 markdown 围栏、不要任何前后缀文字。
|
||||
|
||||
今天的日期是 {{TODAY}}。
|
||||
|
||||
JSON schema(严格):
|
||||
{
|
||||
"title": string,
|
||||
"type": "checkup" | "lab" | "imaging" | "prescription" | "other",
|
||||
"report_date": "YYYY-MM-DD",
|
||||
"institution": string,
|
||||
"page_count": number,
|
||||
"summary": string,
|
||||
"indicators": [
|
||||
{
|
||||
"name": string,
|
||||
"value": string,
|
||||
"unit": string,
|
||||
"range": string,
|
||||
"status": "high" | "low" | "normal"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
规则:
|
||||
- status 优先看箭头/标记(↑/H/偏高 → "high",↓/L/偏低 → "low");没有标记时用 value 与 range 比较;都没有 → "normal"。
|
||||
- range 保留原文(如 "< 3.40"、"3.9 - 6.1"、"0 - 5");OCR 把破折号写成 "--" / "~" 都归一成 " - ";没有参考范围就填 ""。
|
||||
- 凡是「指标名 + 明确数值」可读的都要提取——**没有参考范围不是跳过的理由**,结论页叙述式文字(如「总胆红素: 23.0(μmol/L)↑」)同样提取。只有数值本身是乱码无法判断才跳过,绝不发明指标,同一指标只输出一次。
|
||||
- title / institution / summary 读不出就填 "";report_date 挑「报告/检查/采样日期」其一统一成 YYYY-MM-DD,只有年月就补 -01,实在读不出填上面给出的「今天」({{TODAY}})。
|
||||
- type:化验单→"lab";体检套餐→"checkup";影像(B超/CT/X光/MRI)→"imaging";处方→"prescription";拿不准→"other"。
|
||||
- summary 用一句话概括整体(如「血脂偏高,其余正常」);页眉、医生签名、栏目标题、OCR 噪声一律忽略。
|
||||
|
||||
示例 OCR 文本:
|
||||
协和医院体检中心 健康体检报告 体检日期:2026-04-12 低密度脂蛋白 3.84 mmol/L <3.40 ↑ 空腹血糖 5.2 mmol/L 3.9-6.1
|
||||
对应输出:
|
||||
{"title":"健康体检报告","type":"checkup","report_date":"2026-04-12","institution":"协和医院体检中心","page_count":1,"summary":"血脂偏高,血糖正常","indicators":[{"name":"低密度脂蛋白","value":"3.84","unit":"mmol/L","range":"< 3.40","status":"high"},{"name":"空腹血糖","value":"5.2","unit":"mmol/L","range":"3.9 - 6.1","status":"normal"}]}
|
||||
|
||||
现在请解析下面这段 OCR 文本,只输出 JSON。
|
||||
|
||||
OCR 文本:
|
||||
{{OCR_TEXT}}
|
||||
"""#
|
||||
|
||||
// MARK: - 报告归档 · 轻量 meta(只抽日期/机构/类型/标题,不识别指标)
|
||||
|
||||
@@ -366,13 +366,21 @@ struct ArchiveListView: View {
|
||||
.buttonStyle(.plain)
|
||||
}
|
||||
|
||||
/// 分类过滤条(2026-06-28 重组):把原先扁平的一排标签按语义归三组——
|
||||
/// 自述(日记/症状/问诊)· 检查(指标/报告)· 用药——组间加竖线分隔、每枚标签带图标 + 类别色,
|
||||
/// 让「一堆标签」读成有结构的三块。过滤逻辑仍是单选某个 TimelineKind(选中再点取消)。
|
||||
private var filterChips: some View {
|
||||
ScrollView(.horizontal, showsIndicators: false) {
|
||||
HStack(spacing: 8) {
|
||||
chip(label: String(appLoc: "全部"), selected: filter == nil) { filter = nil }
|
||||
ForEach(TimelineKind.allCases) { kind in
|
||||
chip(label: kind.label, selected: filter == kind) {
|
||||
filter = filter == kind ? nil : kind
|
||||
allChip
|
||||
ForEach(TimelineKind.Group.allCases) { group in
|
||||
groupSeparator
|
||||
// 单成员组(用药)不另标组名:组名与唯一标签重复,直接出标签即可。
|
||||
if group.kinds.count > 1 {
|
||||
groupCaption(group.caption)
|
||||
}
|
||||
ForEach(group.kinds) { kind in
|
||||
kindChip(kind)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -380,23 +388,53 @@ struct ArchiveListView: View {
|
||||
}
|
||||
}
|
||||
|
||||
private func chip(label: String, selected: Bool, action: @escaping () -> Void) -> some View {
|
||||
Button(action: action) {
|
||||
Text(label)
|
||||
private var allChip: some View {
|
||||
let selected = filter == nil
|
||||
return Button { filter = nil } label: {
|
||||
Text(String(appLoc: "全部"))
|
||||
.font(.tjScaled( 13, weight: selected ? .semibold : .regular))
|
||||
.foregroundStyle(selected ? Tj.Palette.paper : Tj.Palette.text)
|
||||
.padding(.horizontal, 14)
|
||||
.padding(.vertical, 8)
|
||||
.background(
|
||||
Capsule().fill(selected ? Tj.Palette.ink : Tj.Palette.paper)
|
||||
)
|
||||
.overlay(
|
||||
Capsule().strokeBorder(Tj.Palette.line, lineWidth: selected ? 0 : 1)
|
||||
)
|
||||
.background(Capsule().fill(selected ? Tj.Palette.ink : Tj.Palette.paper))
|
||||
.overlay(Capsule().strokeBorder(selected ? Color.clear : Tj.Palette.line, lineWidth: 1))
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
}
|
||||
|
||||
/// 单枚分类标签:图标 + 名称,用该类别的 accent 着色;选中填充 accent。
|
||||
private func kindChip(_ kind: TimelineKind) -> some View {
|
||||
let selected = filter == kind
|
||||
return Button { filter = selected ? nil : kind } label: {
|
||||
HStack(spacing: 5) {
|
||||
Image(systemName: kind.icon)
|
||||
.font(.tjScaled( 11, weight: .semibold))
|
||||
.foregroundStyle(selected ? Tj.Palette.paper : kind.accent)
|
||||
Text(kind.label)
|
||||
.font(.tjScaled( 13, weight: selected ? .semibold : .regular))
|
||||
.foregroundStyle(selected ? Tj.Palette.paper : Tj.Palette.text)
|
||||
}
|
||||
.padding(.horizontal, 12)
|
||||
.padding(.vertical, 8)
|
||||
.background(Capsule().fill(selected ? kind.accent : Tj.Palette.paper))
|
||||
.overlay(Capsule().strokeBorder(selected ? Color.clear : Tj.Palette.line, lineWidth: 1))
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
}
|
||||
|
||||
private var groupSeparator: some View {
|
||||
RoundedRectangle(cornerRadius: 1, style: .continuous)
|
||||
.fill(Tj.Palette.lineSoft)
|
||||
.frame(width: 1, height: 18)
|
||||
.padding(.horizontal, 2)
|
||||
}
|
||||
|
||||
private func groupCaption(_ text: String) -> some View {
|
||||
Text(text)
|
||||
.font(.tjScaled( 11, weight: .semibold))
|
||||
.foregroundStyle(Tj.Palette.text3)
|
||||
}
|
||||
|
||||
private func sectionHeader(_ section: DateSection, count: Int) -> some View {
|
||||
HStack {
|
||||
Text(section.label)
|
||||
|
||||
130
康康/Features/Consultation/ConsultationAudioPlayer.swift
Normal file
130
康康/Features/Consultation/ConsultationAudioPlayer.swift
Normal file
@@ -0,0 +1,130 @@
|
||||
import SwiftUI
|
||||
import AVFoundation
|
||||
import Combine
|
||||
|
||||
/// Vault 内一段问诊录音的回放控件(2026-06-28)。播放/暂停 + 进度条 + 时间。
|
||||
/// 录音存在加密 Vault 里,App 前台即解锁可读;读不到(损坏/异常)显示占位,不崩。
|
||||
/// 不做拖拽 seek(降复杂度):进度条只读,demo 够用。
|
||||
struct ConsultationAudioPlayer: View {
|
||||
let asset: Asset
|
||||
|
||||
@State private var player: AVAudioPlayer?
|
||||
@State private var isPlaying = false
|
||||
@State private var currentTime: Double = 0
|
||||
@State private var duration: Double = 0
|
||||
@State private var loadFailed = false
|
||||
|
||||
/// 播放时刷新进度/时间;不播放时空转(开销可忽略)。AVAudioPlayer 播完 isPlaying 自然转 false。
|
||||
private let ticker = Timer.publish(every: 0.2, on: .main, in: .common).autoconnect()
|
||||
|
||||
var body: some View {
|
||||
VStack(alignment: .leading, spacing: 10) {
|
||||
HStack(spacing: 6) {
|
||||
Image(systemName: "waveform")
|
||||
.font(.tjScaled( 12, weight: .semibold))
|
||||
.foregroundStyle(Tj.Palette.ink2)
|
||||
Text(String(appLoc: "问诊录音"))
|
||||
.font(.tjScaled( 12, weight: .semibold))
|
||||
.foregroundStyle(Tj.Palette.text2)
|
||||
Spacer()
|
||||
Text(String(appLoc: "本机存储"))
|
||||
.font(.tjScaled( 11)).foregroundStyle(Tj.Palette.text3)
|
||||
}
|
||||
|
||||
if loadFailed {
|
||||
Text("录音无法读取")
|
||||
.font(.tjScaled( 13)).foregroundStyle(Tj.Palette.text3)
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
} else {
|
||||
HStack(spacing: 12) {
|
||||
Button(action: togglePlay) {
|
||||
Image(systemName: isPlaying ? "pause.fill" : "play.fill")
|
||||
.font(.tjScaled( 16, weight: .bold))
|
||||
.foregroundStyle(Tj.Palette.paper)
|
||||
.frame(width: 40, height: 40)
|
||||
.background(Circle().fill(Tj.Palette.ink2))
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
|
||||
VStack(alignment: .leading, spacing: 6) {
|
||||
GeometryReader { geo in
|
||||
ZStack(alignment: .leading) {
|
||||
Capsule().fill(Tj.Palette.sand2)
|
||||
Capsule().fill(Tj.Palette.ink2)
|
||||
.frame(width: geo.size.width * progressFraction)
|
||||
}
|
||||
}
|
||||
.frame(height: 5)
|
||||
HStack {
|
||||
Text(Self.timeText(currentTime))
|
||||
Spacer()
|
||||
Text(Self.timeText(duration))
|
||||
}
|
||||
.font(.tjScaled( 11, design: .monospaced))
|
||||
.foregroundStyle(Tj.Palette.text3)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.padding(14)
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
.background(
|
||||
RoundedRectangle(cornerRadius: Tj.Radius.md, style: .continuous)
|
||||
.fill(Tj.Palette.paper)
|
||||
)
|
||||
.overlay(
|
||||
RoundedRectangle(cornerRadius: Tj.Radius.md, style: .continuous)
|
||||
.strokeBorder(Tj.Palette.lineSoft, lineWidth: 1)
|
||||
)
|
||||
.onAppear(perform: load)
|
||||
.onDisappear {
|
||||
player?.stop()
|
||||
isPlaying = false
|
||||
}
|
||||
.onReceive(ticker) { _ in
|
||||
guard let p = player, isPlaying else { return }
|
||||
currentTime = p.currentTime
|
||||
if !p.isPlaying { // 播完:复位
|
||||
isPlaying = false
|
||||
currentTime = 0
|
||||
p.currentTime = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private var progressFraction: CGFloat {
|
||||
guard duration > 0 else { return 0 }
|
||||
return CGFloat(min(max(currentTime / duration, 0), 1))
|
||||
}
|
||||
|
||||
private func load() {
|
||||
guard player == nil, !loadFailed else { return }
|
||||
do {
|
||||
let url = try FileVault.shared.absoluteURL(forReading: asset.relativePath)
|
||||
let p = try AVAudioPlayer(contentsOf: url)
|
||||
p.prepareToPlay()
|
||||
player = p
|
||||
duration = p.duration
|
||||
} catch {
|
||||
loadFailed = true
|
||||
}
|
||||
}
|
||||
|
||||
private func togglePlay() {
|
||||
guard let p = player else { return }
|
||||
if isPlaying {
|
||||
p.pause()
|
||||
isPlaying = false
|
||||
} else {
|
||||
try? AVAudioSession.sharedInstance().setCategory(.playback, options: [])
|
||||
try? AVAudioSession.sharedInstance().setActive(true)
|
||||
p.play()
|
||||
isPlaying = true
|
||||
}
|
||||
}
|
||||
|
||||
private static func timeText(_ seconds: Double) -> String {
|
||||
let s = Int(seconds.rounded())
|
||||
return String(format: "%d:%02d", s / 60, s % 60)
|
||||
}
|
||||
}
|
||||
657
康康/Features/Consultation/ConsultationSheet.swift
Normal file
657
康康/Features/Consultation/ConsultationSheet.swift
Normal file
@@ -0,0 +1,657 @@
|
||||
import SwiftUI
|
||||
import SwiftData
|
||||
|
||||
/// 「记录问诊」录入 sheet(2026-06-28)。
|
||||
///
|
||||
/// 流程:开始录音 → 端侧实时转写(`ConsultationRecorder`)→ 停止 → AI 整理成问诊小结
|
||||
/// (`DiaryAssistService.organizeConsultation`)→ 审阅可编辑 → 保存为带「问诊」tag 的 DiaryEntry
|
||||
/// (录音作为 audio/m4a Asset 落加密 Vault)。
|
||||
///
|
||||
/// 守红线:UI 不直接调 AIRuntime(经 DiaryAssistService);识别 + 录音全程本机;
|
||||
/// AI 整理失败回退录音原文(#5);录音落盘尽力而为,失败也照常存文字。
|
||||
/// 本机不支持端侧识别(模拟器/老机型)→ 退化为手动文字录入,功能仍可用。
|
||||
struct ConsultationSheet: View {
|
||||
@Environment(\.modelContext) private var ctx
|
||||
@Environment(\.dismiss) private var dismiss
|
||||
|
||||
enum Phase: Equatable { case idle, recording, transcribing, organizing, review }
|
||||
@State private var phase: Phase = .idle
|
||||
|
||||
@State private var liveTranscript = ""
|
||||
@State private var recordingSeconds = 0
|
||||
/// 审阅区可编辑的问诊小结(整理稿;整理失败时是录音原文)。最终落库的就是它。
|
||||
@State private var note = ""
|
||||
/// 最终转写原话:「改用原话」回退用,也作整理失败兜底。
|
||||
@State private var rawTranscript = ""
|
||||
/// 录音临时文件(tmp 里的 m4a)。保存时 importFile 搬进 Vault;未保存则 onDisappear 删掉。
|
||||
@State private var audioTempURL: URL?
|
||||
@State private var createdAt: Date = .now
|
||||
@State private var decodeRate: Double = 0
|
||||
@State private var voiceNote: String?
|
||||
@State private var deniedAlert = false
|
||||
@State private var saved = false
|
||||
|
||||
@State private var recordTask: Task<Void, Never>?
|
||||
@State private var organizeTask: Task<Void, Never>?
|
||||
@State private var watchdog: Task<Void, Never>?
|
||||
/// 必须 @State:struct View 重建时普通 let 会换新实例,导致 stop() 落在没在录音的新实例上、
|
||||
/// 而真正录音的老实例关不掉麦克风悬挂(同 DiaryQuickSheet.dictation 的注释)。
|
||||
@State private var recorder = ConsultationRecorder()
|
||||
@FocusState private var noteFocused: Bool
|
||||
|
||||
/// 录音上限 10 分钟(超时由看门狗触发停止并整理)。
|
||||
private let maxSeconds = 600
|
||||
|
||||
/// 本机不支持端侧识别 → 手动录入模式(进入即可打字,无录音)。
|
||||
private var manualMode: Bool { !ConsultationRecorder.isAvailable }
|
||||
|
||||
private var canSave: Bool {
|
||||
!note.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
VStack(spacing: 0) {
|
||||
grabber
|
||||
header
|
||||
ScrollView(showsIndicators: false) {
|
||||
VStack(alignment: .leading, spacing: 16) {
|
||||
if let note = voiceNote { noteBanner(note) }
|
||||
phaseContent
|
||||
}
|
||||
.padding(.horizontal, 20)
|
||||
.padding(.top, 4)
|
||||
.padding(.bottom, 8)
|
||||
}
|
||||
.scrollDismissesKeyboard(.interactively)
|
||||
bottomBar
|
||||
}
|
||||
.background(
|
||||
Tj.Palette.sand
|
||||
.clipShape(RoundedRectangle(cornerRadius: Tj.Radius.xl, style: .continuous))
|
||||
.ignoresSafeArea(edges: .bottom)
|
||||
)
|
||||
.presentationDetents([.large])
|
||||
.presentationDragIndicator(.hidden)
|
||||
.presentationBackground(Tj.Palette.sand)
|
||||
.presentationCornerRadius(Tj.Radius.xl)
|
||||
.onDisappear(perform: cleanup)
|
||||
.alert(String(appLoc: "需要麦克风与语音识别权限"), isPresented: $deniedAlert) {
|
||||
Button(String(appLoc: "前往设置")) {
|
||||
if let url = URL(string: UIApplication.openSettingsURLString) {
|
||||
UIApplication.shared.open(url)
|
||||
}
|
||||
}
|
||||
Button(String(appLoc: "取消"), role: .cancel) {}
|
||||
} message: {
|
||||
Text("问诊录音全程在本机完成,声音和文字都不会上传。请在设置中允许麦克风和语音识别。")
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Header
|
||||
|
||||
private var grabber: some View {
|
||||
Capsule()
|
||||
.fill(Tj.Palette.line)
|
||||
.frame(width: 40, height: 4)
|
||||
.padding(.top, 10)
|
||||
.padding(.bottom, 14)
|
||||
}
|
||||
|
||||
private var header: some View {
|
||||
HStack(alignment: .center, spacing: 12) {
|
||||
Button { dismiss() } label: {
|
||||
Image(systemName: "xmark")
|
||||
.font(.tjScaled( 15, weight: .semibold))
|
||||
.foregroundStyle(Tj.Palette.text)
|
||||
.frame(width: 32, height: 32)
|
||||
.background(Circle().fill(Tj.Palette.sand2))
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
VStack(alignment: .leading, spacing: 2) {
|
||||
Text("记录问诊")
|
||||
.font(.tjH2())
|
||||
.foregroundStyle(Tj.Palette.text)
|
||||
Text("录音 → 本机转写 → AI 整理成问诊小结")
|
||||
.font(.tjScaled( 11))
|
||||
.foregroundStyle(Tj.Palette.text3)
|
||||
}
|
||||
Spacer()
|
||||
TjLockChip()
|
||||
}
|
||||
.padding(.horizontal, 20)
|
||||
.padding(.bottom, 12)
|
||||
}
|
||||
|
||||
// MARK: - Phase content
|
||||
|
||||
@ViewBuilder
|
||||
private var phaseContent: some View {
|
||||
switch phase {
|
||||
case .idle:
|
||||
if manualMode { manualEntry } else { idleStart }
|
||||
case .recording:
|
||||
recordingCard
|
||||
case .transcribing:
|
||||
transcribingCard
|
||||
case .organizing:
|
||||
organizingCard
|
||||
case .review:
|
||||
reviewArea
|
||||
}
|
||||
}
|
||||
|
||||
/// 待开始:大录音按钮 + 三条说明(本机识别 / 可回放 / 不上传)。
|
||||
private var idleStart: some View {
|
||||
VStack(spacing: 18) {
|
||||
Button(action: startRecording) {
|
||||
VStack(spacing: 10) {
|
||||
ZStack {
|
||||
Circle().fill(Tj.Palette.ink2)
|
||||
.frame(width: 76, height: 76)
|
||||
.shadow(color: Tj.Palette.ink2.opacity(0.3), radius: 12, y: 4)
|
||||
Image(systemName: "mic.fill")
|
||||
.font(.tjScaled( 30, weight: .semibold))
|
||||
.foregroundStyle(Tj.Palette.paper)
|
||||
}
|
||||
Text("开始录音")
|
||||
.font(.tjScaled( 15, weight: .semibold))
|
||||
.foregroundStyle(Tj.Palette.ink2)
|
||||
}
|
||||
.frame(maxWidth: .infinity)
|
||||
.padding(.vertical, 22)
|
||||
.contentShape(Rectangle())
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
|
||||
VStack(alignment: .leading, spacing: 10) {
|
||||
bullet("stethoscope", String(appLoc: "看医生时录下来,事后不怕记不全"))
|
||||
bullet("waveform", String(appLoc: "录音结束后在本机转写,原声存进加密档案"))
|
||||
bullet("sparkles", String(appLoc: "康康自动整理成主诉/医生建议/复查小节"))
|
||||
}
|
||||
.padding(16)
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
.background(
|
||||
RoundedRectangle(cornerRadius: Tj.Radius.md, style: .continuous)
|
||||
.fill(Tj.Palette.paper)
|
||||
)
|
||||
.overlay(
|
||||
RoundedRectangle(cornerRadius: Tj.Radius.md, style: .continuous)
|
||||
.strokeBorder(Tj.Palette.lineSoft, lineWidth: 1)
|
||||
)
|
||||
|
||||
Text("仅供个人记录,不构成诊断或用药建议。")
|
||||
.font(.tjScaled( 11)).foregroundStyle(Tj.Palette.text3)
|
||||
}
|
||||
.padding(.top, 6)
|
||||
}
|
||||
|
||||
private func bullet(_ icon: String, _ text: String) -> some View {
|
||||
HStack(alignment: .top, spacing: 10) {
|
||||
Image(systemName: icon)
|
||||
.font(.tjScaled( 13, weight: .semibold))
|
||||
.foregroundStyle(Tj.Palette.ink2)
|
||||
.frame(width: 20)
|
||||
Text(text)
|
||||
.font(.tjScaled( 13))
|
||||
.foregroundStyle(Tj.Palette.text2)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
Spacer(minLength: 0)
|
||||
}
|
||||
}
|
||||
|
||||
/// 录音中:计时 + 实时字幕 + 结束按钮。
|
||||
private var recordingCard: some View {
|
||||
VStack(alignment: .leading, spacing: 12) {
|
||||
HStack(spacing: 8) {
|
||||
Image(systemName: "waveform")
|
||||
.font(.tjScaled( 13, weight: .semibold))
|
||||
.foregroundStyle(Tj.Palette.brick)
|
||||
.symbolEffect(.variableColor.iterative, options: .repeating)
|
||||
Text("正在录音 · 声音不上传")
|
||||
.font(.tjScaled( 13, weight: .medium))
|
||||
.foregroundStyle(Tj.Palette.text2)
|
||||
Spacer(minLength: 0)
|
||||
Text(Self.timeText(recordingSeconds))
|
||||
.font(.tjScaled( 13, design: .monospaced))
|
||||
.foregroundStyle(recordingSeconds >= maxSeconds - 30 ? Tj.Palette.brick : Tj.Palette.text3)
|
||||
}
|
||||
|
||||
// 离线转写(SenseVoice)无实时字幕:录音中只给声纹动效 + 说明,结束后整段转写。
|
||||
VStack(spacing: 10) {
|
||||
Image(systemName: "waveform")
|
||||
.font(.tjScaled( 44, weight: .regular))
|
||||
.foregroundStyle(Tj.Palette.ink2)
|
||||
.symbolEffect(.variableColor.iterative, options: .repeating)
|
||||
Text("正在录音,结束后会在本机把整段录音转成文字")
|
||||
.font(.tjScaled( 13))
|
||||
.foregroundStyle(Tj.Palette.text3)
|
||||
.multilineTextAlignment(.center)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
}
|
||||
.frame(maxWidth: .infinity, minHeight: 150)
|
||||
.padding(.vertical, 8)
|
||||
|
||||
Button(action: stopAndOrganize) {
|
||||
HStack(spacing: 8) {
|
||||
Image(systemName: "stop.circle.fill")
|
||||
Text("结束并整理")
|
||||
}
|
||||
.font(.tjScaled( 15, weight: .semibold))
|
||||
.foregroundStyle(Tj.Palette.paper)
|
||||
.frame(maxWidth: .infinity)
|
||||
.padding(.vertical, 13)
|
||||
.background(
|
||||
RoundedRectangle(cornerRadius: Tj.Radius.sm, style: .continuous)
|
||||
.fill(Tj.Palette.brick)
|
||||
)
|
||||
.contentShape(Rectangle())
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
}
|
||||
.padding(14)
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
.background(
|
||||
RoundedRectangle(cornerRadius: Tj.Radius.md, style: .continuous)
|
||||
.fill(Tj.Palette.paper)
|
||||
)
|
||||
.overlay(
|
||||
RoundedRectangle(cornerRadius: Tj.Radius.md, style: .continuous)
|
||||
.strokeBorder(Tj.Palette.lineSoft, lineWidth: 1)
|
||||
)
|
||||
}
|
||||
|
||||
/// 录音转写中:整段录音离线转文字(优先本地 SenseVoice,不可用回退本机识别)。无实时字幕,统一转写。
|
||||
private var transcribingCard: some View {
|
||||
VStack(alignment: .leading, spacing: 12) {
|
||||
HStack(spacing: 8) {
|
||||
Image(systemName: "waveform")
|
||||
.font(.tjScaled( 13, weight: .semibold))
|
||||
.foregroundStyle(Tj.Palette.brick)
|
||||
.symbolEffect(.variableColor.iterative, options: .repeating)
|
||||
Text(SenseVoiceASRService.isAvailable
|
||||
? String(appLoc: "正在转写录音 · 本地 SenseVoice")
|
||||
: String(appLoc: "正在转写录音 · 本机识别"))
|
||||
.font(.tjScaled( 13, weight: .medium))
|
||||
.foregroundStyle(Tj.Palette.text2)
|
||||
Spacer(minLength: 0)
|
||||
}
|
||||
Text("录音已结束,正在本机把整段录音转成文字,请稍候…")
|
||||
.font(.tjScaled( 13))
|
||||
.foregroundStyle(Tj.Palette.text3)
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
AIFlowBar()
|
||||
}
|
||||
.padding(14)
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
.background(
|
||||
RoundedRectangle(cornerRadius: Tj.Radius.md, style: .continuous)
|
||||
.fill(Tj.Palette.paper)
|
||||
)
|
||||
.overlay(
|
||||
RoundedRectangle(cornerRadius: Tj.Radius.md, style: .continuous)
|
||||
.strokeBorder(Tj.Palette.lineSoft, lineWidth: 1)
|
||||
)
|
||||
}
|
||||
|
||||
/// AI 整理中:呼吸流光 + 置灰原话预览 + 取消(取消即用原话)。
|
||||
private var organizingCard: some View {
|
||||
VStack(alignment: .leading, spacing: 12) {
|
||||
HStack(spacing: 8) {
|
||||
Image(systemName: "sparkles")
|
||||
.font(.tjScaled( 13, weight: .semibold))
|
||||
.foregroundStyle(Tj.Palette.brick)
|
||||
.symbolEffect(.pulse, options: .repeating)
|
||||
Text(decodeRate > 0
|
||||
? String(format: String(appLoc: "正在整理问诊小结 · %.1f tok/s"), decodeRate)
|
||||
: String(appLoc: "正在整理问诊小结 · 本地推理"))
|
||||
.font(.tjScaled( 13, weight: .medium))
|
||||
.foregroundStyle(Tj.Palette.text2)
|
||||
Spacer(minLength: 0)
|
||||
Button(String(appLoc: "用原话")) { cancelOrganize() }
|
||||
.font(.tjScaled( 12, weight: .semibold))
|
||||
.foregroundStyle(Tj.Palette.text3)
|
||||
.buttonStyle(.plain)
|
||||
}
|
||||
Text(rawTranscript)
|
||||
.font(.tjScaled( 14))
|
||||
.foregroundStyle(Tj.Palette.text3)
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
.lineLimit(6)
|
||||
AIFlowBar()
|
||||
}
|
||||
.padding(14)
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
.background(
|
||||
RoundedRectangle(cornerRadius: Tj.Radius.md, style: .continuous)
|
||||
.fill(Tj.Palette.paper)
|
||||
)
|
||||
.overlay(
|
||||
RoundedRectangle(cornerRadius: Tj.Radius.md, style: .continuous)
|
||||
.strokeBorder(Tj.Palette.lineSoft, lineWidth: 1)
|
||||
)
|
||||
}
|
||||
|
||||
/// 审阅:录音已附提示 + 可编辑小结 + (改用原话 / 重新整理) + 时间。
|
||||
private var reviewArea: some View {
|
||||
VStack(alignment: .leading, spacing: 14) {
|
||||
if audioTempURL != nil {
|
||||
HStack(spacing: 6) {
|
||||
Image(systemName: "waveform.circle.fill")
|
||||
.font(.tjScaled( 13))
|
||||
.foregroundStyle(Tj.Palette.ink2)
|
||||
Text("录音已保存,可在记录详情里回放")
|
||||
.font(.tjScaled( 12))
|
||||
.foregroundStyle(Tj.Palette.text2)
|
||||
Spacer(minLength: 0)
|
||||
}
|
||||
}
|
||||
|
||||
VStack(alignment: .leading, spacing: 8) {
|
||||
HStack {
|
||||
Text(String(appLoc: "问诊小结"))
|
||||
.font(.tjScaled( 12, weight: .semibold))
|
||||
.tracking(0.3)
|
||||
.foregroundStyle(Tj.Palette.text2)
|
||||
Spacer()
|
||||
if decodeRate > 0 {
|
||||
Text(String(format: "%.1f tok/s", decodeRate))
|
||||
.font(.tjScaled( 10, design: .monospaced))
|
||||
.foregroundStyle(Tj.Palette.leaf)
|
||||
}
|
||||
}
|
||||
TextField(String(appLoc: "整理后的问诊小结,可在这里修改…"),
|
||||
text: $note, axis: .vertical)
|
||||
.font(.tjScaled( 15))
|
||||
.lineLimit(6...16)
|
||||
.focused($noteFocused)
|
||||
.padding(.horizontal, 14)
|
||||
.padding(.vertical, 12)
|
||||
.background(
|
||||
RoundedRectangle(cornerRadius: Tj.Radius.sm, style: .continuous)
|
||||
.fill(Tj.Palette.paper)
|
||||
)
|
||||
.overlay(
|
||||
RoundedRectangle(cornerRadius: Tj.Radius.sm, style: .continuous)
|
||||
.strokeBorder(Tj.Palette.line, lineWidth: 1)
|
||||
)
|
||||
|
||||
HStack(spacing: 10) {
|
||||
if !rawTranscript.isEmpty && note != rawTranscript {
|
||||
reviewChip("arrow.uturn.backward", String(appLoc: "改用原话")) {
|
||||
withAnimation(.snappy(duration: 0.18)) { note = rawTranscript }
|
||||
}
|
||||
}
|
||||
if !rawTranscript.isEmpty {
|
||||
reviewChip("arrow.clockwise", String(appLoc: "重新整理")) { reorganize() }
|
||||
}
|
||||
Spacer(minLength: 0)
|
||||
}
|
||||
}
|
||||
|
||||
VStack(alignment: .leading, spacing: 8) {
|
||||
Text(String(appLoc: "时间"))
|
||||
.font(.tjScaled( 12, weight: .semibold))
|
||||
.tracking(0.3)
|
||||
.foregroundStyle(Tj.Palette.text2)
|
||||
DatePicker("", selection: $createdAt, in: ...Date.now)
|
||||
.datePickerStyle(.compact)
|
||||
.labelsHidden()
|
||||
}
|
||||
|
||||
AIDisclaimerFooter()
|
||||
}
|
||||
}
|
||||
|
||||
private func reviewChip(_ icon: String, _ label: String, action: @escaping () -> Void) -> some View {
|
||||
Button(action: action) {
|
||||
HStack(spacing: 4) {
|
||||
Image(systemName: icon).font(.tjScaled( 10, weight: .semibold))
|
||||
Text(label).font(.tjScaled( 11, weight: .semibold))
|
||||
}
|
||||
.foregroundStyle(Tj.Palette.ink)
|
||||
.padding(.horizontal, 10)
|
||||
.padding(.vertical, 5)
|
||||
.background(Capsule().strokeBorder(Tj.Palette.line, lineWidth: 1))
|
||||
.contentShape(Capsule())
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
}
|
||||
|
||||
/// 手动模式(本机不支持识别):直接打字录入问诊小结。
|
||||
private var manualEntry: some View {
|
||||
VStack(alignment: .leading, spacing: 12) {
|
||||
HStack(spacing: 6) {
|
||||
Image(systemName: "info.circle")
|
||||
.font(.tjScaled( 12))
|
||||
.foregroundStyle(Tj.Palette.text3)
|
||||
Text("本机暂不支持录音转写,可手动记录这次问诊")
|
||||
.font(.tjScaled( 12))
|
||||
.foregroundStyle(Tj.Palette.text3)
|
||||
Spacer(minLength: 0)
|
||||
}
|
||||
TextField(String(appLoc: "记录这次问诊:主诉、医生说了什么、用药、复查…"),
|
||||
text: $note, axis: .vertical)
|
||||
.font(.tjScaled( 15))
|
||||
.lineLimit(6...16)
|
||||
.focused($noteFocused)
|
||||
.padding(.horizontal, 14)
|
||||
.padding(.vertical, 12)
|
||||
.background(
|
||||
RoundedRectangle(cornerRadius: Tj.Radius.sm, style: .continuous)
|
||||
.fill(Tj.Palette.paper)
|
||||
)
|
||||
.overlay(
|
||||
RoundedRectangle(cornerRadius: Tj.Radius.sm, style: .continuous)
|
||||
.strokeBorder(Tj.Palette.line, lineWidth: 1)
|
||||
)
|
||||
DatePicker("", selection: $createdAt, in: ...Date.now)
|
||||
.datePickerStyle(.compact)
|
||||
.labelsHidden()
|
||||
}
|
||||
.padding(.top, 6)
|
||||
}
|
||||
|
||||
// MARK: - Bottom bar
|
||||
|
||||
@ViewBuilder
|
||||
private var bottomBar: some View {
|
||||
switch phase {
|
||||
case .idle where manualMode:
|
||||
HStack(spacing: 8) {
|
||||
Button(String(appLoc: "取消")) { dismiss() }
|
||||
.buttonStyle(TjGhostButton(height: 44, fontSize: 15, horizontalPadding: 14))
|
||||
Button(String(appLoc: "保存")) { save() }
|
||||
.buttonStyle(TjPrimaryButton(height: 44, fontSize: 15, horizontalPadding: 14))
|
||||
.disabled(!canSave)
|
||||
.opacity(canSave ? 1 : 0.4)
|
||||
}
|
||||
.padding(.horizontal, 20)
|
||||
.padding(.vertical, 14)
|
||||
case .idle:
|
||||
// 非手动模式的待开始页:开始录音是内容区大按钮,关闭走 header 的 ✕,底部不再占一行。
|
||||
EmptyView()
|
||||
case .review:
|
||||
HStack(spacing: 8) {
|
||||
Button(String(appLoc: "重新录音")) { restartRecording() }
|
||||
.buttonStyle(TjGhostButton(height: 44, fontSize: 15, horizontalPadding: 14))
|
||||
Button(String(appLoc: "保存")) { save() }
|
||||
.buttonStyle(TjPrimaryButton(height: 44, fontSize: 15, horizontalPadding: 14))
|
||||
.disabled(!canSave)
|
||||
.opacity(canSave ? 1 : 0.4)
|
||||
}
|
||||
.padding(.horizontal, 20)
|
||||
.padding(.vertical, 14)
|
||||
case .recording, .transcribing, .organizing:
|
||||
EmptyView()
|
||||
}
|
||||
}
|
||||
|
||||
private func noteBanner(_ text: String) -> some View {
|
||||
HStack(spacing: 6) {
|
||||
Image(systemName: "exclamationmark.circle.fill")
|
||||
.font(.tjScaled( 11)).foregroundStyle(Tj.Palette.amber)
|
||||
Text(text).font(.tjScaled( 12)).foregroundStyle(Tj.Palette.text2)
|
||||
Spacer(minLength: 0)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Actions
|
||||
|
||||
private func startRecording() {
|
||||
voiceNote = nil
|
||||
noteFocused = false
|
||||
recordTask = Task { @MainActor in
|
||||
guard await recorder.requestAuthorization() else {
|
||||
deniedAlert = true
|
||||
return
|
||||
}
|
||||
do {
|
||||
liveTranscript = ""
|
||||
recordingSeconds = 0
|
||||
try recorder.start { partial in liveTranscript = partial }
|
||||
withAnimation(.snappy(duration: 0.2)) { phase = .recording }
|
||||
watchdog = Task { @MainActor in
|
||||
while !Task.isCancelled {
|
||||
try? await Task.sleep(nanoseconds: 1_000_000_000)
|
||||
guard !Task.isCancelled, phase == .recording else { return }
|
||||
recordingSeconds += 1
|
||||
if recordingSeconds >= maxSeconds { stopAndOrganize(); return }
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
#if DEBUG
|
||||
print("[Consultation] recorder start failed: \(error)")
|
||||
#endif
|
||||
voiceNote = String(appLoc: "无法开始录音,请检查麦克风 / 语音识别权限")
|
||||
phase = .idle
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func stopAndOrganize() {
|
||||
guard phase == .recording else { return }
|
||||
watchdog?.cancel()
|
||||
organizeTask = Task { @MainActor in
|
||||
let result = await recorder.stop()
|
||||
audioTempURL = result.audioTempURL
|
||||
|
||||
// 系统端侧识别(SFSpeech,录音时顺带跑)作为 SenseVoice 不可用 / 失败时的自动回退稿。
|
||||
var sfFallback = result.transcript.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if sfFallback.isEmpty {
|
||||
sfFallback = liveTranscript.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
}
|
||||
|
||||
// 离线转写整段录音:优先本地 SenseVoice(MNN),不可用 / 失败 / 空 → 回退 SFSpeech(守红线 #5)。
|
||||
withAnimation(.snappy(duration: 0.2)) { phase = .transcribing }
|
||||
var transcript = ""
|
||||
if SenseVoiceASRService.isAvailable, let url = result.audioTempURL {
|
||||
transcript = (try? await SenseVoiceASRService.shared.transcribe(audioFileURL: url)) ?? ""
|
||||
}
|
||||
if Task.isCancelled { return }
|
||||
if transcript.isEmpty { transcript = sfFallback }
|
||||
rawTranscript = transcript
|
||||
|
||||
guard !transcript.isEmpty else {
|
||||
// 没听清:进审阅让用户手动补;录音(若有)仍可保存。
|
||||
voiceNote = String(appLoc: "没听清,可手动补充或重新录音")
|
||||
note = ""
|
||||
withAnimation(.snappy(duration: 0.2)) { phase = .review }
|
||||
return
|
||||
}
|
||||
|
||||
withAnimation(.snappy(duration: 0.2)) { phase = .organizing }
|
||||
await runOrganize(transcript: transcript)
|
||||
}
|
||||
}
|
||||
|
||||
/// 调用 LLM 整理;成功填小结,取消/失败回退原话(红线 #5)。供首次整理与「重新整理」复用。
|
||||
private func runOrganize(transcript: String) async {
|
||||
do {
|
||||
let organized = try await DiaryAssistService.shared.organizeConsultation(transcript: transcript)
|
||||
guard !Task.isCancelled else { return }
|
||||
note = organized.text
|
||||
decodeRate = organized.decodeRate
|
||||
} catch is CancellationError {
|
||||
return // cancelOrganize 已处理回退
|
||||
} catch {
|
||||
note = transcript
|
||||
voiceNote = String(appLoc: "AI 整理没成功,已填入录音原文")
|
||||
}
|
||||
withAnimation(.snappy(duration: 0.2)) { phase = .review }
|
||||
}
|
||||
|
||||
private func cancelOrganize() {
|
||||
guard phase == .organizing else { return }
|
||||
organizeTask?.cancel()
|
||||
note = rawTranscript
|
||||
withAnimation(.snappy(duration: 0.2)) { phase = .review }
|
||||
}
|
||||
|
||||
/// 审阅区「重新整理」:从原话再跑一遍 LLM(首次整理结果不满意时)。
|
||||
private func reorganize() {
|
||||
guard !rawTranscript.isEmpty else { return }
|
||||
organizeTask?.cancel()
|
||||
decodeRate = 0
|
||||
withAnimation(.snappy(duration: 0.2)) { phase = .organizing }
|
||||
organizeTask = Task { @MainActor in
|
||||
await runOrganize(transcript: rawTranscript)
|
||||
}
|
||||
}
|
||||
|
||||
/// 审阅区「重新录音」:丢弃当前录音/小结,回到待开始。
|
||||
private func restartRecording() {
|
||||
organizeTask?.cancel()
|
||||
if let url = audioTempURL { try? FileManager.default.removeItem(at: url) }
|
||||
audioTempURL = nil
|
||||
note = ""
|
||||
rawTranscript = ""
|
||||
liveTranscript = ""
|
||||
decodeRate = 0
|
||||
voiceNote = nil
|
||||
withAnimation(.snappy(duration: 0.2)) { phase = .idle }
|
||||
}
|
||||
|
||||
private func save() {
|
||||
let content = note.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !content.isEmpty else { return }
|
||||
let entry = DiaryEntry(content: content,
|
||||
createdAt: createdAt,
|
||||
tags: [DiaryEntry.consultationTag])
|
||||
ctx.insert(entry)
|
||||
// 录音搬进加密 Vault 并挂为 Asset(尽力而为:失败也照常存文字)。
|
||||
if let temp = audioTempURL,
|
||||
let savedAsset = try? FileVault.shared.importFile(at: temp, preferredExtension: "m4a") {
|
||||
let asset = Asset(relativePath: savedAsset.relativePath,
|
||||
mimeType: "audio/m4a",
|
||||
bytes: savedAsset.bytes)
|
||||
ctx.insert(asset)
|
||||
entry.assets.append(asset)
|
||||
audioTempURL = nil // 已搬走,cleanup 不再删
|
||||
}
|
||||
try? ctx.save()
|
||||
saved = true
|
||||
dismiss()
|
||||
}
|
||||
|
||||
/// 关闭时收尾:停录音、撤任务、删未保存的录音临时文件。
|
||||
private func cleanup() {
|
||||
recordTask?.cancel()
|
||||
organizeTask?.cancel()
|
||||
watchdog?.cancel()
|
||||
recorder.abort()
|
||||
if !saved, let url = audioTempURL {
|
||||
try? FileManager.default.removeItem(at: url)
|
||||
}
|
||||
}
|
||||
|
||||
private static func timeText(_ seconds: Int) -> String {
|
||||
String(format: "%d:%02d", seconds / 60, seconds % 60)
|
||||
}
|
||||
}
|
||||
|
||||
#Preview {
|
||||
ConsultationSheet()
|
||||
.modelContainer(for: [DiaryEntry.self, Asset.self], inMemory: true)
|
||||
}
|
||||
@@ -1,9 +1,13 @@
|
||||
import SwiftUI
|
||||
|
||||
/// 推理引擎设置:在 MNN(CPU/SME2,考核路径)与 MLX(GPU,兜底)间切换,并展示 SME2 探测状态。
|
||||
/// 推理引擎设置:本项目已切 Gemma-3n(端侧 4bit),主模型只走 MLX/GPU。
|
||||
/// MNN/SME2 路径已停用(Gemma-3n 无 MNN 转换模型),引擎行保留但 MNN 置灰。
|
||||
/// 切换只改持久化选择;下一次 AI 调用(prepare/generate)按新引擎加载。
|
||||
struct InferenceSettingsView: View {
|
||||
@AppStorage("kk.inferenceEngine") private var engineRaw = EnginePreference.auto.rawValue
|
||||
// 云端 AI(Gemini)开关与 key —— 键名与 CloudAI 对齐,@AppStorage 写入即被后端读到。
|
||||
@AppStorage("cloud_ai_gemini_enabled") private var cloudEnabled = false
|
||||
@AppStorage("cloud_ai_gemini_key") private var cloudKey = ""
|
||||
@State private var modelService = ModelDownloadService.shared
|
||||
/// 性能自检改为当前页就地展开,不再 push 新页面。
|
||||
@State private var showSelfTest = false
|
||||
@@ -12,10 +16,9 @@ struct InferenceSettingsView: View {
|
||||
EnginePreference(rawValue: engineRaw) ?? .auto
|
||||
}
|
||||
|
||||
/// 性能自检需要模型就绪(MNN 主或 MLX 兜底任一)。
|
||||
/// 性能自检需要主模型(Gemma-3n,MLX)就绪。
|
||||
private var modelReady: Bool {
|
||||
modelService.states[.mnnLLM]?.phase == .ready
|
||||
|| modelService.states[.llm]?.phase == .ready
|
||||
modelService.states[.llm]?.phase == .ready
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
@@ -34,8 +37,8 @@ struct InferenceSettingsView: View {
|
||||
engineRow(engine)
|
||||
}
|
||||
|
||||
sme2Card
|
||||
selfTestSection
|
||||
cloudSection
|
||||
noteCard
|
||||
}
|
||||
.padding(.horizontal, 16)
|
||||
@@ -134,11 +137,11 @@ struct InferenceSettingsView: View {
|
||||
.disabled(!available)
|
||||
}
|
||||
|
||||
/// .auto 永远可用;具体引擎看自身可用性。
|
||||
/// .auto 永远可用;MLX 看自身可用性。MNN 已停用(Gemma-3n 无 MNN 模型),恒置灰。
|
||||
private func isAvailable(_ engine: EnginePreference) -> Bool {
|
||||
switch engine {
|
||||
case .auto: return true
|
||||
case .mnn: return InferenceEngine.mnn.isAvailable
|
||||
case .mnn: return false
|
||||
case .mlx: return InferenceEngine.mlx.isAvailable
|
||||
}
|
||||
}
|
||||
@@ -154,52 +157,74 @@ struct InferenceSettingsView: View {
|
||||
private func subtitle(_ engine: EnginePreference, available: Bool) -> String {
|
||||
switch engine {
|
||||
case .auto:
|
||||
// 显示自动解析后实际命中的引擎,让用户看清「这台机选了什么」。
|
||||
let resolved = engine.resolved
|
||||
if resolved == .mnn {
|
||||
return InferenceEngine.cpuSupportsSME2
|
||||
? String(appLoc: "按本机配置选择 · 当前 MNN + SME2")
|
||||
: String(appLoc: "按本机配置选择 · 当前 MNN(NEON)")
|
||||
} else {
|
||||
return String(appLoc: "按本机配置选择 · 当前 MLX(MNN 不可用)")
|
||||
}
|
||||
// 已切 Gemma-3n,auto 恒解析为 MLX。
|
||||
return String(appLoc: "按本机配置选择 · 当前 MLX · GPU")
|
||||
case .mnn:
|
||||
if !available { return String(appLoc: "本设备/模拟器不可用,自动回退 MLX") }
|
||||
return InferenceEngine.cpuSupportsSME2
|
||||
? String(appLoc: "端侧 CPU + SME2 加速 · 挑战赛考核路径")
|
||||
: String(appLoc: "端侧 CPU(本机无 SME2,NEON 回退)")
|
||||
return String(appLoc: "已停用:Gemma-3n 无 MNN 转换模型,统一走 MLX")
|
||||
case .mlx:
|
||||
return String(appLoc: "Metal GPU · 兜底 / 对照")
|
||||
return String(appLoc: "Metal GPU · 端侧推理 Gemma-3n E2B")
|
||||
}
|
||||
}
|
||||
|
||||
private var sme2Card: some View {
|
||||
let sme2 = InferenceEngine.cpuSupportsSME2
|
||||
return HStack(spacing: 12) {
|
||||
ZStack {
|
||||
Circle().fill(sme2 ? Tj.Palette.leafSoft : Tj.Palette.sand2)
|
||||
Image(systemName: sme2 ? "checkmark.seal.fill" : "minus.circle")
|
||||
.font(.tjScaled(18))
|
||||
.foregroundStyle(sme2 ? Tj.Palette.ink : Tj.Palette.text2)
|
||||
}
|
||||
.frame(width: 44, height: 44)
|
||||
VStack(alignment: .leading, spacing: 2) {
|
||||
Text("Arm SME2")
|
||||
.font(.tjScaled(15, weight: .medium))
|
||||
private var cloudConfigured: Bool {
|
||||
cloudEnabled && !cloudKey.trimmingCharacters(in: .whitespaces).isEmpty
|
||||
}
|
||||
|
||||
/// 云端 AI(Gemini)开关区。hybrid 的「云端」一侧:默认关(隐私优先),
|
||||
/// 开启并填入 AI Studio 的 key 后,「读报告原图 / 深度解读 / 多语言」走 Google Gemini。
|
||||
private var cloudSection: some View {
|
||||
VStack(alignment: .leading, spacing: 12) {
|
||||
HStack {
|
||||
Text("云端 AI · Gemini")
|
||||
.font(.tjTitle())
|
||||
.foregroundStyle(Tj.Palette.text)
|
||||
Text(sme2 ? String(appLoc: "本设备支持,MNN 已启用 SME2 加速")
|
||||
: String(appLoc: "本设备不支持(需 A19/iPhone 17+)"))
|
||||
.font(.tjScaled(12))
|
||||
.foregroundStyle(Tj.Palette.text3)
|
||||
Spacer()
|
||||
}
|
||||
Spacer()
|
||||
.padding(.top, 10)
|
||||
|
||||
VStack(alignment: .leading, spacing: 12) {
|
||||
Toggle(isOn: $cloudEnabled) {
|
||||
VStack(alignment: .leading, spacing: 2) {
|
||||
Text("启用云端增强")
|
||||
.font(.tjScaled(15, weight: .semibold))
|
||||
.foregroundStyle(Tj.Palette.text)
|
||||
Text("默认端侧 Gemma-3n;开启后「读报告原图 / 深度解读 / 多语言」走 Google Gemini。")
|
||||
.font(.tjScaled(12))
|
||||
.foregroundStyle(Tj.Palette.text3)
|
||||
}
|
||||
}
|
||||
.tint(Tj.Palette.leaf)
|
||||
|
||||
if cloudEnabled {
|
||||
SecureField(String(appLoc: "粘贴 Google AI Studio 的 API Key"), text: $cloudKey)
|
||||
.font(.tjScaled(14))
|
||||
.textInputAutocapitalization(.never)
|
||||
.autocorrectionDisabled()
|
||||
.padding(12)
|
||||
.background(RoundedRectangle(cornerRadius: Tj.Radius.md).fill(Tj.Palette.sand2))
|
||||
|
||||
HStack(spacing: 6) {
|
||||
Image(systemName: cloudConfigured ? "checkmark.seal.fill" : "exclamationmark.circle")
|
||||
.font(.tjScaled(13))
|
||||
.foregroundStyle(cloudConfigured ? Tj.Palette.leaf : Tj.Palette.text3)
|
||||
Text(cloudConfigured
|
||||
? String(appLoc: "已就绪 · \(AIRuntime.cloudLabel)")
|
||||
: String(appLoc: "请填入 API Key(aistudio.google.com 免费获取)"))
|
||||
.font(.tjScaled(12))
|
||||
.foregroundStyle(Tj.Palette.text3)
|
||||
}
|
||||
Text("Key 仅存本机;断网或额度用尽时自动回退端侧 Gemma-3n,功能不中断。")
|
||||
.font(.tjScaled(11))
|
||||
.foregroundStyle(Tj.Palette.text3)
|
||||
}
|
||||
}
|
||||
.padding(14)
|
||||
.tjCard()
|
||||
}
|
||||
.padding(14)
|
||||
.tjCard()
|
||||
}
|
||||
|
||||
private var noteCard: some View {
|
||||
Text("MNN 在端侧 CPU 上以 Arm SME2 指令集加速 Qwen 推理(本地、不上云)。切换后下一次 AI 调用生效。")
|
||||
Text("隐私优先:默认端侧 Gemma-3n(MLX · Metal GPU)推理,数据不出设备;仅在你开启「云端 AI」后,深度任务才走 Google Gemini。切换后下一次 AI 调用生效。")
|
||||
.font(.tjScaled(12))
|
||||
.foregroundStyle(Tj.Palette.text3)
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
|
||||
@@ -141,7 +141,7 @@ struct ModelManagementView: View {
|
||||
} else if allReady {
|
||||
HStack(spacing: 6) {
|
||||
Image(systemName: "checkmark.seal.fill")
|
||||
Text("Qwen3.5-2B 已就绪")
|
||||
Text("Gemma-3n E2B 已就绪")
|
||||
}
|
||||
.font(.tjScaled( 13, weight: .semibold))
|
||||
.foregroundStyle(Tj.Palette.leaf)
|
||||
@@ -205,9 +205,9 @@ struct ModelManagementView: View {
|
||||
|
||||
private func subtitle(_ kind: ModelKind) -> String {
|
||||
switch kind {
|
||||
case .llm: return String(appLoc: "文本解读 · 趋势 / 问答(MLX 兜底)")
|
||||
case .llm: return String(appLoc: "文本解读 · 趋势 / 问答 / 拍照识别 · MLX 端侧推理")
|
||||
case .vl: return String(appLoc: "拍照识别报告 → 结构化指标")
|
||||
case .mnnLLM: return String(appLoc: "文本解读 + 拍照识别 · MNN + SME2 端侧加速")
|
||||
case .mnnLLM: return String(appLoc: "已停用 · 旧 MNN 路径")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -177,7 +177,7 @@ struct ModelSelfTestView: View {
|
||||
}
|
||||
}
|
||||
}
|
||||
Text("在「我的 · 推理引擎」切换引擎后再跑一次,即可对比 SME2 与 GPU。")
|
||||
Text("Gemma-3n E2B 在端侧 MLX(Metal GPU)推理,100% 本地、不上云。")
|
||||
.font(.tjScaled( 10))
|
||||
.foregroundStyle(Tj.Palette.text3)
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import SwiftUI
|
||||
|
||||
enum RecordKind: String, Identifiable, CaseIterable {
|
||||
case quick, indicator, healthExport, archive, diary, symptom, reminder, medicationLibrary
|
||||
case quick, indicator, healthExport, archive, diary, symptom, reminder, medicationLibrary, consultation
|
||||
var id: String { rawValue }
|
||||
|
||||
/// RecordSheet 列表的展示顺序(从上到下)。与 enum 声明序解耦,改顺序只动这里。
|
||||
@@ -9,7 +9,7 @@ enum RecordKind: String, Identifiable, CaseIterable {
|
||||
/// `.symptom`(记录症状)与拍药盒一起并入 `.diary`(健康日记)顶部三选一,不再单列;
|
||||
/// `.medicationLibrary`(药品库)是浏览/管理目的地,主入口在「记录」Tab 顶部板卡,
|
||||
/// 这里垫底保留一个快捷方式(创建动作在前,药品库作为「去管理」入口排最后)。
|
||||
static let displayOrder: [RecordKind] = [.diary, .reminder, .indicator, .healthExport, .archive, .medicationLibrary]
|
||||
static let displayOrder: [RecordKind] = [.diary, .consultation, .reminder, .indicator, .healthExport, .archive, .medicationLibrary]
|
||||
|
||||
/// 健康日记行的功能提示 pill(代替 subtitle,让"症状/药盒在日记里"一眼可见)。
|
||||
/// 计算属性:每次按当前语言解析,语言切换即时更新(同 ProfileEditView 的 presets 约定)。
|
||||
@@ -24,6 +24,7 @@ enum RecordKind: String, Identifiable, CaseIterable {
|
||||
case .healthExport: return String(appLoc: "身体档案")
|
||||
case .archive: return String(appLoc: "体检报告归档")
|
||||
case .diary: return String(appLoc: "健康日记")
|
||||
case .consultation: return String(appLoc: "记录问诊")
|
||||
case .symptom: return String(appLoc: "记录症状")
|
||||
case .reminder: return String(appLoc: "开启一个提醒")
|
||||
case .medicationLibrary: return String(appLoc: "药品库")
|
||||
@@ -36,6 +37,7 @@ enum RecordKind: String, Identifiable, CaseIterable {
|
||||
case .healthExport: return String(appLoc: "多轮问答后生成给医生看的整理报告")
|
||||
case .archive: return String(appLoc: "完整保存整份报告(可多页)")
|
||||
case .diary: return String(appLoc: "写日记或拍药盒记录用药 · 可让 AI 辅助")
|
||||
case .consultation: return String(appLoc: "看医生时录音,本机转写并整理成小结")
|
||||
case .symptom: return String(appLoc: "开始一个持续症状,结束时再点结束")
|
||||
case .reminder: return String(appLoc: "管理用药、复查、监测的周期提醒")
|
||||
case .medicationLibrary: return String(appLoc: "管理常用药清单 · 拍药盒或手动添加")
|
||||
@@ -48,6 +50,7 @@ enum RecordKind: String, Identifiable, CaseIterable {
|
||||
case .healthExport: return "doc.text.below.ecg"
|
||||
case .archive: return "doc.fill"
|
||||
case .diary: return "heart.text.square"
|
||||
case .consultation: return "stethoscope"
|
||||
case .symptom: return "waveform.path.ecg"
|
||||
case .reminder: return "bell.badge"
|
||||
case .medicationLibrary: return "pills.fill"
|
||||
@@ -60,6 +63,7 @@ enum RecordKind: String, Identifiable, CaseIterable {
|
||||
case .healthExport: return Tj.Palette.ink
|
||||
case .archive: return Tj.Palette.ink
|
||||
case .diary: return Tj.Palette.leaf
|
||||
case .consultation: return Tj.Palette.ink2
|
||||
case .symptom: return Tj.Palette.amber
|
||||
case .reminder: return Tj.Palette.leaf
|
||||
case .medicationLibrary: return Tj.Palette.ink
|
||||
|
||||
@@ -3,36 +3,69 @@ import SwiftData
|
||||
import Foundation
|
||||
|
||||
enum TimelineKind: String, CaseIterable, Identifiable {
|
||||
case diary, symptom, indicator, medication, report
|
||||
case diary, symptom, consultation, indicator, medication, report
|
||||
var id: String { rawValue }
|
||||
|
||||
var label: String {
|
||||
switch self {
|
||||
case .indicator: return String(appLoc: "指标")
|
||||
case .report: return String(appLoc: "报告")
|
||||
case .symptom: return String(appLoc: "症状")
|
||||
case .diary: return String(appLoc: "日记")
|
||||
case .medication: return String(appLoc: "用药")
|
||||
case .indicator: return String(appLoc: "指标")
|
||||
case .report: return String(appLoc: "报告")
|
||||
case .symptom: return String(appLoc: "症状")
|
||||
case .diary: return String(appLoc: "日记")
|
||||
case .consultation: return String(appLoc: "问诊")
|
||||
case .medication: return String(appLoc: "用药")
|
||||
}
|
||||
}
|
||||
|
||||
var icon: String {
|
||||
switch self {
|
||||
case .indicator: return "drop.fill"
|
||||
case .report: return "doc.fill"
|
||||
case .symptom: return "waveform.path.ecg"
|
||||
case .diary: return "pencil"
|
||||
case .medication: return "pills.fill"
|
||||
case .indicator: return "drop.fill"
|
||||
case .report: return "doc.fill"
|
||||
case .symptom: return "waveform.path.ecg"
|
||||
case .diary: return "pencil"
|
||||
case .consultation: return "stethoscope"
|
||||
case .medication: return "pills.fill"
|
||||
}
|
||||
}
|
||||
|
||||
var accent: Color {
|
||||
switch self {
|
||||
case .indicator: return Tj.Palette.brick
|
||||
case .report: return Tj.Palette.ink2
|
||||
case .symptom: return Tj.Palette.amber
|
||||
case .diary: return Tj.Palette.leaf
|
||||
case .medication: return Tj.Palette.ink
|
||||
case .indicator: return Tj.Palette.brick
|
||||
case .report: return Tj.Palette.ink2
|
||||
case .symptom: return Tj.Palette.amber
|
||||
case .diary: return Tj.Palette.leaf
|
||||
case .consultation: return Tj.Palette.ink2
|
||||
case .medication: return Tj.Palette.ink
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
extension TimelineKind {
|
||||
/// 记录页分类分组(2026-06-28):把扁平的一排标签按语义归三组,过滤条按组排序、组间加分隔,
|
||||
/// 让「一堆标签」读成「自述 / 检查 / 用药」三块。仅影响过滤条的排列与分隔——
|
||||
/// 过滤逻辑仍是单选某个 TimelineKind,不引入多选状态。
|
||||
enum Group: String, CaseIterable, Identifiable {
|
||||
case selfLog // 自述记录:日记、症状、问诊(本人写/说的)
|
||||
case clinical // 检查数据:指标、报告(测量来的客观数据)
|
||||
case medication // 用药
|
||||
|
||||
var id: String { rawValue }
|
||||
|
||||
/// 组标题。单成员组(用药)在过滤条里不另外标题(标题与唯一标签重复),见 ArchiveListView。
|
||||
var caption: String {
|
||||
switch self {
|
||||
case .selfLog: return String(appLoc: "自述")
|
||||
case .clinical: return String(appLoc: "检查")
|
||||
case .medication: return String(appLoc: "用药")
|
||||
}
|
||||
}
|
||||
|
||||
var kinds: [TimelineKind] {
|
||||
switch self {
|
||||
case .selfLog: return [.diary, .symptom, .consultation]
|
||||
case .clinical: return [.indicator, .report]
|
||||
case .medication: return [.medication]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -165,17 +198,28 @@ struct TimelineEntry: Identifiable, Hashable {
|
||||
}
|
||||
}
|
||||
|
||||
/// 带「用药」tag 的日记(拍药盒入档)归到 .medication 分类,其余是普通文字日记。
|
||||
/// id 统一用 "diary-" 前缀:TimelineDetail.resolve 两个分类都反查 diaries。
|
||||
/// DiaryEntry 据 tag 归类:问诊录音 → .consultation,拍药盒/用药 → .medication,其余普通文字日记。
|
||||
/// id 统一用 "diary-" 前缀:TimelineDetail.resolve 这几个分类都反查 diaries。
|
||||
static func from(diary d: DiaryEntry) -> TimelineEntry {
|
||||
let isMed = d.isMedicationLog
|
||||
let kind: TimelineKind
|
||||
let subtitle: String
|
||||
if d.isConsultation {
|
||||
kind = .consultation
|
||||
subtitle = String(appLoc: "问诊记录")
|
||||
} else if d.isMedicationLog {
|
||||
kind = .medication
|
||||
subtitle = String(appLoc: "用药记录")
|
||||
} else {
|
||||
kind = .diary
|
||||
subtitle = String(appLoc: "文字日记")
|
||||
}
|
||||
return TimelineEntry(
|
||||
id: "diary-\(d.persistentModelID)",
|
||||
kind: isMed ? .medication : .diary,
|
||||
kind: kind,
|
||||
date: d.createdAt,
|
||||
title: d.content.firstLine(),
|
||||
subtitle: isMed ? String(appLoc: "用药记录") : String(appLoc: "文字日记"),
|
||||
trailing: nil,
|
||||
subtitle: subtitle,
|
||||
trailing: d.isConsultation && d.audioAsset != nil ? String(appLoc: "录音") : nil,
|
||||
trailingIsAlert: false,
|
||||
isOngoing: false
|
||||
)
|
||||
|
||||
@@ -22,8 +22,8 @@ enum TimelineDetail {
|
||||
case .report:
|
||||
return reports.first { "report-\($0.persistentModelID)" == entry.id }
|
||||
.map(TimelineDetail.report)
|
||||
case .diary, .medication:
|
||||
// 用药记录本质是带「用药」tag 的 DiaryEntry,详情同日记。
|
||||
case .diary, .medication, .consultation:
|
||||
// 用药记录 / 问诊记录本质都是带特定 tag 的 DiaryEntry,详情统一反查 diaries。
|
||||
return diaries.first { "diary-\($0.persistentModelID)" == entry.id }
|
||||
.map(TimelineDetail.diary)
|
||||
case .symptom:
|
||||
@@ -201,7 +201,9 @@ struct TimelineEntryDetailView: View {
|
||||
case .indicator: return String(appLoc: "指标详情")
|
||||
case .bloodPressure: return String(appLoc: "血压详情")
|
||||
case .report: return String(appLoc: "报告详情")
|
||||
case .diary(let d): return d.isMedicationLog ? String(appLoc: "用药详情") : String(appLoc: "日记详情")
|
||||
case .diary(let d):
|
||||
if d.isConsultation { return String(appLoc: "问诊详情") }
|
||||
return d.isMedicationLog ? String(appLoc: "用药详情") : String(appLoc: "日记详情")
|
||||
case .symptom: return String(appLoc: "症状详情")
|
||||
}
|
||||
}
|
||||
@@ -377,7 +379,9 @@ struct TimelineEntryDetailView: View {
|
||||
|
||||
@ViewBuilder
|
||||
private func diaryBody(_ d: DiaryEntry) -> some View {
|
||||
if d.isMedicationLog {
|
||||
if d.isConsultation {
|
||||
consultationBody(d)
|
||||
} else if d.isMedicationLog {
|
||||
medicationBody(d)
|
||||
} else {
|
||||
VStack(alignment: .leading, spacing: 16) {
|
||||
@@ -398,6 +402,42 @@ struct TimelineEntryDetailView: View {
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - 问诊记录(录音回放 + 结构化小结)
|
||||
|
||||
/// 问诊记录(带「问诊」tag 的日记):顶部录音回放(若有),中间结构化问诊小结。
|
||||
/// 录音 + 文字都在本机(header 的 TjLockChip 已示「本地」);AI 整理稿附免责声明,不构成诊断/用药建议。
|
||||
private func consultationBody(_ d: DiaryEntry) -> some View {
|
||||
VStack(alignment: .leading, spacing: 16) {
|
||||
if let audio = d.audioAsset {
|
||||
ConsultationAudioPlayer(asset: audio)
|
||||
}
|
||||
card {
|
||||
HStack(spacing: 6) {
|
||||
Image(systemName: "stethoscope")
|
||||
.font(.tjScaled( 12, weight: .semibold))
|
||||
.foregroundStyle(Tj.Palette.ink2)
|
||||
Text(String(appLoc: "问诊小结"))
|
||||
.font(.tjScaled( 12, weight: .semibold))
|
||||
.foregroundStyle(Tj.Palette.text2)
|
||||
Spacer()
|
||||
Text(Self.dateTimeText(d.createdAt))
|
||||
.font(.tjScaled( 11)).foregroundStyle(Tj.Palette.text3)
|
||||
}
|
||||
divider
|
||||
Text(d.content)
|
||||
.font(.tjScaled( 15))
|
||||
.foregroundStyle(Tj.Palette.text)
|
||||
.textSelection(.enabled)
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
}
|
||||
Text("内容由本机语音识别 + AI 整理,仅供个人回看,可能与原话有出入,不构成诊断或用药建议。")
|
||||
.font(.tjScaled( 11)).foregroundStyle(Tj.Palette.text3)
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - 用药使用记录(展示药名/剂量/时间 + 设置提醒)
|
||||
|
||||
/// 用药使用记录(带「用药」tag 的日记):展示「药名 [规格] · 剂量」+ 时间,下方「设置提醒」。
|
||||
|
||||
@@ -28,7 +28,7 @@ final class HealthExport {
|
||||
var inferredLabelCN: String?
|
||||
|
||||
// demo 卖点凭证
|
||||
/// 模型 tag,如 "Qwen3.5-2B-MNN"(iPhone17+ 主路径)或 "Qwen3.5-2B-4bit"(MLX 兜底)。截图能证明本地推理。
|
||||
/// 模型 tag,如 "gemma-3n-E2B-it-lm-4bit"(MLX 端侧主模型)。截图能证明本地推理。
|
||||
var modelTag: String
|
||||
/// 末次 tok/s,对应 demo 卖点 #6 Live Activity 数据。
|
||||
var decodeRate: Double
|
||||
@@ -44,7 +44,7 @@ final class HealthExport {
|
||||
inferredTimeToDate: Date? = nil,
|
||||
inferredIntent: String? = nil,
|
||||
inferredLabelCN: String? = nil,
|
||||
modelTag: String = "Qwen3.5-2B-MNN",
|
||||
modelTag: String = "gemma-3n-E2B-it-lm-4bit",
|
||||
decodeRate: Double = 0) {
|
||||
self.prompt = prompt
|
||||
self.content = content
|
||||
|
||||
@@ -189,7 +189,16 @@ extension DiaryEntry {
|
||||
/// (语言切换后旧数据要还能被识别)。时间线据此把该日记归到「用药」分类。
|
||||
static let medicationTag = "用药"
|
||||
|
||||
/// 「记录问诊」(录音转写)落库时打的 tag。同上:是数据标识不是 UI 文案,不走本地化。
|
||||
/// 时间线据此把该日记归到「问诊」分类;详情页据此渲染录音播放 + 转写原文。
|
||||
static let consultationTag = "问诊"
|
||||
|
||||
var isMedicationLog: Bool { tags.contains(Self.medicationTag) }
|
||||
|
||||
var isConsultation: Bool { tags.contains(Self.consultationTag) }
|
||||
|
||||
/// 关联的录音原件(问诊记录可能挂一段 m4a)。按 mimeType 前缀识别,与拍药盒的图片 Asset 区分。
|
||||
var audioAsset: Asset? { assets.first { $0.mimeType.hasPrefix("audio") } }
|
||||
}
|
||||
|
||||
@Model
|
||||
|
||||
@@ -87,6 +87,32 @@ nonisolated final class FileVault: @unchecked Sendable {
|
||||
return SavedAsset(relativePath: filename, bytes: data.count)
|
||||
}
|
||||
|
||||
/// 把已录制好的临时文件(如问诊录音 m4a)搬进 Vault,并打上硬件级文件保护。
|
||||
/// 录音过程写在 `tmp`(不加密、可边录边写),录完调用本方法落库——与 writeJPEG 一样以
|
||||
/// `.complete` 保护落盘(§6 隐私:Vault 全目录硬件加密)。move 失败(源不存在/占用)抛 writeFailed。
|
||||
/// 返回相对路径 + 字节数,供调用方建 `Asset`。源临时文件搬走后即被移除。
|
||||
nonisolated func importFile(at sourceURL: URL, preferredExtension ext: String) throws -> SavedAsset {
|
||||
let fm = FileManager.default
|
||||
let filename = "\(UUID().uuidString).\(ext)"
|
||||
let dest = rootURL.appendingPathComponent(filename)
|
||||
do {
|
||||
if fm.fileExists(atPath: dest.path) { try fm.removeItem(at: dest) }
|
||||
try fm.moveItem(at: sourceURL, to: dest)
|
||||
try fm.setAttributes([.protectionKey: FileProtectionType.complete],
|
||||
ofItemAtPath: dest.path)
|
||||
} catch {
|
||||
throw FileVaultError.writeFailed
|
||||
}
|
||||
let bytes = (try? fm.attributesOfItem(atPath: dest.path))?[.size] as? Int ?? 0
|
||||
return SavedAsset(relativePath: filename, bytes: bytes)
|
||||
}
|
||||
|
||||
/// 取 Vault 内文件的绝对 URL(只读用,如 AVAudioPlayer 播放录音)。
|
||||
/// 走与读写一致的路径安全校验(禁子目录 / `..` 越界)。App 在前台即解锁,可正常读取受保护文件。
|
||||
nonisolated func absoluteURL(forReading relativePath: String) throws -> URL {
|
||||
try resolveSafePath(relativePath)
|
||||
}
|
||||
|
||||
nonisolated func loadImage(relativePath: String) throws -> UIImage {
|
||||
let url = try resolveSafePath(relativePath)
|
||||
let data: Data
|
||||
|
||||
@@ -51,6 +51,8 @@ struct RootView: View {
|
||||
/// 语音「写日记」直达:跳过日记 sheet 顶部入口选择,光标直接落到正文。
|
||||
@State private var diaryDirectWrite = false
|
||||
@State private var showIndicator = false
|
||||
/// 「记录问诊」:录音 → 本机转写 → AI 整理成问诊小结。
|
||||
@State private var showConsultation = false
|
||||
@State private var showReminders = false
|
||||
@State private var showHealthExport = false
|
||||
/// 长按 + :语音直达(说一句话 → LLM 意图分类 → 打开对应入口)。
|
||||
@@ -122,6 +124,7 @@ struct RootView: View {
|
||||
case .archive: activeFlow = .archive
|
||||
case .symptom: showSymptomStart = true
|
||||
case .diary: diaryDirectWrite = false; showDiary = true
|
||||
case .consultation: showConsultation = true
|
||||
case .indicator: showIndicator = true
|
||||
case .reminder: showReminders = true
|
||||
case .healthExport: showHealthExport = true
|
||||
@@ -136,6 +139,9 @@ struct RootView: View {
|
||||
.sheet(isPresented: $showDiary) {
|
||||
DiaryQuickSheet(directWrite: diaryDirectWrite)
|
||||
}
|
||||
.sheet(isPresented: $showConsultation) {
|
||||
ConsultationSheet()
|
||||
}
|
||||
.sheet(isPresented: $showIndicator) {
|
||||
// 「拍照识别」入口:关闭手输表单 → 打开指标速记 VL 流程(并入「记录指标」)。
|
||||
IndicatorQuickSheet(onRequestCamera: {
|
||||
|
||||
@@ -174,28 +174,56 @@ actor CaptureService {
|
||||
return s
|
||||
}
|
||||
|
||||
/// VL 推理 + JSON 解析的纯阶段。assets 必须已写入 Vault。
|
||||
/// 整份报告识别 + JSON 解析的纯阶段。assets 必须已写入 Vault。
|
||||
///
|
||||
/// hybrid 双路:
|
||||
/// - **云端可用(Gemini)**:图片直传 Gemini 多模态读图(真·VL,恢复 source_box 证据高亮),
|
||||
/// OCR 文本作数字「抄写员」一并注入。这是端侧 Gemma-3n(MLX 文本版,无视觉)拿不到的能力。
|
||||
/// - **离线/未开云端**:回退 Vision OCR(本地,<1s/页)→ 端侧 Gemma 文本 LLM 抽 meta+指标。
|
||||
/// 云端任何失败(离线/超时/解析失败)都静默回退端侧,绝不卡死(§3.2 失败回退红线)。
|
||||
private func runVL(on assets: [FileVault.SavedAsset]) async throws -> ParsedReport {
|
||||
let urls = assets.map { FileVault.shared.rootURL.appendingPathComponent($0.relativePath) }
|
||||
// Vision OCR 出纯文本(失败/空都视作无法识别 —— 影像类报告本就无文字,不该清空旧数据)。
|
||||
let ocr = await Self.ocrReference(for: urls)
|
||||
|
||||
// 云端优先:Gemini 多模态直读图。影像类报告 OCR 可能为空,但 Gemini 仍能读图,故不卡 OCR 门槛。
|
||||
if AIRuntime.shared.cloudAvailable {
|
||||
do {
|
||||
let raw = try await AIRuntime.shared.analyzeReportCloud(
|
||||
imageURLs: urls,
|
||||
prompt: VLPrompts.reportExtraction(ocrText: ocr),
|
||||
maxTokens: 2048
|
||||
)
|
||||
return try CaptureService.parseReportJSON(
|
||||
CaptureService.stripThink(raw), pageCount: assets.count)
|
||||
} catch {
|
||||
// 落到端侧回退,不抛 —— 保证断网/额度耗尽时仍可用。
|
||||
}
|
||||
}
|
||||
|
||||
// 端侧回退:需有 OCR 文本(端侧 Gemma 无视觉)。
|
||||
guard !ocr.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
|
||||
throw CaptureError.inferenceFailed(String(appLoc: "未识别到文字,无法解读"))
|
||||
}
|
||||
do {
|
||||
try await AIRuntime.shared.prepareVL()
|
||||
try await AIRuntime.shared.prepare() // 载文本 LLM(OOM 闸门处理卸载)
|
||||
} catch {
|
||||
throw CaptureError.modelNotReady
|
||||
}
|
||||
let urls = assets.map { FileVault.shared.rootURL.appendingPathComponent($0.relativePath) }
|
||||
// OCR 参考(Vision 本地,<1s/页):给 2B 多模态当数字「抄写员」,降低小字误读。
|
||||
// 任何失败都静默回退为空串,绝不阻断识别主流程(§3.2)。
|
||||
let ocr = await Self.ocrReference(for: urls)
|
||||
let raw: String
|
||||
var collected = ""
|
||||
do {
|
||||
raw = try await AIRuntime.shared.analyzeReport(
|
||||
imageURLs: urls,
|
||||
prompt: VLPrompts.reportExtraction(ocrText: ocr)
|
||||
// 整份报告十余项,给足 token;与任何 VL/文本解码由 AIRuntime 闸门串行。
|
||||
let stream = await AIRuntime.shared.generate(
|
||||
prompt: VLPrompts.reportExtractionFromText(ocr),
|
||||
maxTokens: 2048
|
||||
)
|
||||
for try await chunk in stream { collected += chunk.text }
|
||||
} catch {
|
||||
throw CaptureError.inferenceFailed("\(error)")
|
||||
}
|
||||
let cleaned = CaptureService.stripThink(collected)
|
||||
do {
|
||||
return try CaptureService.parseReportJSON(raw, pageCount: assets.count)
|
||||
return try CaptureService.parseReportJSON(cleaned, pageCount: assets.count)
|
||||
} catch let CaptureError.parseFailed(msg) {
|
||||
throw CaptureError.parseFailed(msg)
|
||||
} catch {
|
||||
|
||||
206
康康/Services/ConsultationRecorder.swift
Normal file
206
康康/Services/ConsultationRecorder.swift
Normal file
@@ -0,0 +1,206 @@
|
||||
import Foundation
|
||||
import Speech
|
||||
import AVFoundation
|
||||
|
||||
/// 「记录问诊」录音 + 端侧转写(2026-06-28)。
|
||||
///
|
||||
/// 与 `SpeechDictationService`(写日记口述,刻意**不落盘音频**)的区别:问诊要留一段可回放的录音,
|
||||
/// 所以本服务在**同一个 AVAudioEngine tap** 里做两件事:
|
||||
/// ① `request.append(buffer)` → SFSpeech 端侧流式转写(实时字幕,`requiresOnDeviceRecognition = true`,红线:识别不出设备);
|
||||
/// ② `audioFile.write(buffer)` → 落一份 m4a(AAC)到 tmp,停止后由调用方 `FileVault.importFile` 搬进加密 Vault。
|
||||
///
|
||||
/// 音频落盘是**尽力而为**:任何一步失败都 `try?` 吞掉,最坏情况只是没有录音文件——
|
||||
/// 转写稿来自独立的 SFSpeech 流,照常返回,记录照常保存(守红线 #5:失败回退,不卡死)。
|
||||
///
|
||||
/// 工程默认 MainActor 隔离,本类型即 MainActor;tap 与识别回调在系统线程,
|
||||
/// 闭包内只碰局部捕获对象,回主线程统一走 `Task { @MainActor }`(同 SpeechDictationService)。
|
||||
final class ConsultationRecorder {
|
||||
|
||||
enum RecorderError: Error, LocalizedError {
|
||||
case unavailable
|
||||
case audioEngineStartFailed(String)
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .unavailable:
|
||||
return String(appLoc: "本机不支持端侧语音识别")
|
||||
case .audioEngineStartFailed(let m):
|
||||
return String(appLoc: "录音启动失败:\(m)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 停止后返回:最终转写稿 + 录音临时文件(可能为 nil,音频落盘失败时)。
|
||||
struct Result {
|
||||
let transcript: String
|
||||
/// tmp 目录里的 m4a;调用方负责 `FileVault.importFile` 搬进 Vault 或丢弃。
|
||||
let audioTempURL: URL?
|
||||
}
|
||||
|
||||
/// 优先系统语言;系统语言不支持端侧时兜底中文(同 SpeechDictationService)。
|
||||
private static func makeRecognizer() -> SFSpeechRecognizer? {
|
||||
if let r = SFSpeechRecognizer(locale: .current), r.supportsOnDeviceRecognition {
|
||||
return r
|
||||
}
|
||||
if let r = SFSpeechRecognizer(locale: Locale(identifier: "zh-CN")),
|
||||
r.supportsOnDeviceRecognition {
|
||||
return r
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
/// 本机是否支持端侧识别。false(模拟器 / 老机型)时 UI 退化为手动文字录入。
|
||||
static var isAvailable: Bool { makeRecognizer() != nil }
|
||||
|
||||
private let audioEngine = AVAudioEngine()
|
||||
private var request: SFSpeechAudioBufferRecognitionRequest?
|
||||
private var task: SFSpeechRecognitionTask?
|
||||
/// 第一帧 buffer 到达时按其真实格式惰性建文件,保证写入格式与 tap 完全一致(不会因格式不匹配静默丢音)。
|
||||
private var audioFile: AVAudioFile?
|
||||
private var audioTempURL: URL?
|
||||
|
||||
private var latestText = ""
|
||||
private var didFinish = false
|
||||
|
||||
private(set) var isRecording = false
|
||||
|
||||
/// 麦克风 + 语音识别两个权限一起申请。任一被拒返回 false。
|
||||
func requestAuthorization() async -> Bool {
|
||||
let speech = await withCheckedContinuation { (c: CheckedContinuation<SFSpeechRecognizerAuthorizationStatus, Never>) in
|
||||
SFSpeechRecognizer.requestAuthorization { c.resume(returning: $0) }
|
||||
}
|
||||
guard speech == .authorized else { return false }
|
||||
return await AVAudioApplication.requestRecordPermission()
|
||||
}
|
||||
|
||||
/// 开始录音 + 流式识别。partial 结果在主线程回调(实时字幕)。
|
||||
func start(onPartial: @escaping (String) -> Void) throws {
|
||||
guard !isRecording else { return }
|
||||
guard let recognizer = Self.makeRecognizer(), recognizer.isAvailable else {
|
||||
throw RecorderError.unavailable
|
||||
}
|
||||
|
||||
let session = AVAudioSession.sharedInstance()
|
||||
do {
|
||||
// .record + 默认模式(非 .measurement):问诊录音要尽量保真,不做语音增强裁剪。
|
||||
try session.setCategory(.record, mode: .default, options: .duckOthers)
|
||||
try session.setActive(true, options: .notifyOthersOnDeactivation)
|
||||
} catch {
|
||||
throw RecorderError.audioEngineStartFailed(error.localizedDescription)
|
||||
}
|
||||
|
||||
let request = SFSpeechAudioBufferRecognitionRequest()
|
||||
request.requiresOnDeviceRecognition = true // 红线:识别不出设备
|
||||
request.shouldReportPartialResults = true
|
||||
request.addsPunctuation = true
|
||||
self.request = request
|
||||
latestText = ""
|
||||
didFinish = false
|
||||
audioFile = nil
|
||||
|
||||
// 录音先落 tmp(不加密、可边录边写,不受锁屏文件保护影响);停止后再搬进加密 Vault。
|
||||
let tempURL = FileManager.default.temporaryDirectory
|
||||
.appendingPathComponent("consult-\(UUID().uuidString).m4a")
|
||||
self.audioTempURL = tempURL
|
||||
|
||||
let input = audioEngine.inputNode
|
||||
let format = input.outputFormat(forBus: 0)
|
||||
// 录音文件在装 tap 前就按 tap 格式建好:AAC 文件的 processingFormat = 该采样率/声道的 float 格式,
|
||||
// 正是 tap buffer 的格式 → write 不会因格式不符静默丢音。建失败(try? 为 nil)只是没录音文件,转写照常(尽力而为)。
|
||||
let settings: [String: Any] = [
|
||||
AVFormatIDKey: kAudioFormatMPEG4AAC,
|
||||
AVSampleRateKey: format.sampleRate,
|
||||
AVNumberOfChannelsKey: format.channelCount,
|
||||
AVEncoderAudioQualityKey: AVAudioQuality.medium.rawValue,
|
||||
]
|
||||
let audioFile = try? AVAudioFile(forWriting: tempURL, settings: settings)
|
||||
self.audioFile = audioFile // 留一份引用给 stop() flush
|
||||
// tap 在音频线程跑:只碰**局部捕获**的 request / audioFile,绝不碰 self(避免跨线程数据竞争,同 SpeechDictationService)。
|
||||
input.installTap(onBus: 0, bufferSize: 1024, format: format) { buffer, _ in
|
||||
request.append(buffer)
|
||||
try? audioFile?.write(from: buffer)
|
||||
}
|
||||
audioEngine.prepare()
|
||||
do {
|
||||
try audioEngine.start()
|
||||
} catch {
|
||||
input.removeTap(onBus: 0)
|
||||
deactivateSession()
|
||||
throw RecorderError.audioEngineStartFailed(error.localizedDescription)
|
||||
}
|
||||
|
||||
task = recognizer.recognitionTask(with: request) { [weak self] result, error in
|
||||
Task { @MainActor in
|
||||
guard let self else { return }
|
||||
if let result {
|
||||
self.latestText = result.bestTranscription.formattedString
|
||||
onPartial(self.latestText)
|
||||
if result.isFinal { self.didFinish = true }
|
||||
}
|
||||
if error != nil { self.didFinish = true }
|
||||
}
|
||||
}
|
||||
isRecording = true
|
||||
}
|
||||
|
||||
/// 停止录音,等待最终识别结果(最多 1.5s,超时用最新 partial),返回转写稿 + 录音文件。
|
||||
func stop() async -> Result {
|
||||
guard isRecording else {
|
||||
return Result(transcript: latestText, audioTempURL: finishedAudioURL())
|
||||
}
|
||||
isRecording = false
|
||||
|
||||
audioEngine.stop()
|
||||
audioEngine.inputNode.removeTap(onBus: 0)
|
||||
request?.endAudio()
|
||||
|
||||
let deadline = Date().addingTimeInterval(1.5)
|
||||
while !didFinish && Date() < deadline {
|
||||
try? await Task.sleep(nanoseconds: 100_000_000)
|
||||
}
|
||||
task?.cancel()
|
||||
task = nil
|
||||
request = nil
|
||||
let url = finishedAudioURL() // 关文件(置 nil)后再取 URL,确保已 flush 落盘
|
||||
deactivateSession()
|
||||
return Result(transcript: latestText, audioTempURL: url)
|
||||
}
|
||||
|
||||
/// 用户直接关闭时的清理:不关心结果,立即停;顺手删掉半截录音临时文件。
|
||||
func abort() {
|
||||
guard isRecording else {
|
||||
cleanupTempFile()
|
||||
return
|
||||
}
|
||||
isRecording = false
|
||||
audioEngine.stop()
|
||||
audioEngine.inputNode.removeTap(onBus: 0)
|
||||
request?.endAudio()
|
||||
task?.cancel()
|
||||
task = nil
|
||||
request = nil
|
||||
audioFile = nil
|
||||
deactivateSession()
|
||||
cleanupTempFile()
|
||||
}
|
||||
|
||||
/// 关掉写文件句柄(flush),返回有内容的录音 URL;文件没建成/为空则返回 nil。
|
||||
private func finishedAudioURL() -> URL? {
|
||||
audioFile = nil // 释放写句柄,数据 flush 到磁盘
|
||||
guard let url = audioTempURL,
|
||||
let attrs = try? FileManager.default.attributesOfItem(atPath: url.path),
|
||||
let bytes = attrs[.size] as? Int, bytes > 0 else {
|
||||
return nil
|
||||
}
|
||||
return url
|
||||
}
|
||||
|
||||
private func cleanupTempFile() {
|
||||
if let url = audioTempURL { try? FileManager.default.removeItem(at: url) }
|
||||
audioTempURL = nil
|
||||
}
|
||||
|
||||
private func deactivateSession() {
|
||||
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
|
||||
}
|
||||
}
|
||||
@@ -201,4 +201,29 @@ struct DiaryAssistService {
|
||||
guard !text.isEmpty else { throw AssistError.empty }
|
||||
return (text, lastRate)
|
||||
}
|
||||
|
||||
/// 把问诊录音转写稿整理成结构化问诊小结(2026-06-28,见 ConsultationPrompts)。
|
||||
/// 与 organize 同样走 AIRuntime actor 队列、同样失败回退原话(调用方处理),只是 prompt/产物不同。
|
||||
/// maxTokens 给到 700:问诊小结按多小节分行,比日记长。
|
||||
func organizeConsultation(transcript: String) async throws -> (text: String, decodeRate: Double) {
|
||||
do {
|
||||
try await AIRuntime.shared.prepare()
|
||||
} catch {
|
||||
throw AssistError.modelNotReady
|
||||
}
|
||||
|
||||
let prompt = ConsultationPrompts.organize(transcript: transcript)
|
||||
var collected = ""
|
||||
var lastRate: Double = 0
|
||||
let stream = await AIRuntime.shared.generate(prompt: prompt, maxTokens: 700)
|
||||
for try await chunk in stream {
|
||||
collected += chunk.text
|
||||
if chunk.decodeRate > 0 { lastRate = chunk.decodeRate }
|
||||
}
|
||||
|
||||
let text = HealthExportService.stripThinkBlocks(collected)
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !text.isEmpty else { throw AssistError.empty }
|
||||
return (text, lastRate)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -647,13 +647,21 @@ struct HealthExportService {
|
||||
return d
|
||||
}
|
||||
|
||||
// diaries
|
||||
// diaries(含问诊记录:带「问诊」tag 的日记)。
|
||||
// 问诊记录标出 kind 并放宽摘录长度,让报告把它当成「既往看医生的问诊记录」重点参考。
|
||||
root["diaries"] = snapshot.diaries.map { d -> [String: Any] in
|
||||
let excerpt = String(d.content.prefix(80))
|
||||
return [
|
||||
let limit = d.isConsultation ? 240 : 80
|
||||
let excerpt = String(d.content.prefix(limit))
|
||||
var item: [String: Any] = [
|
||||
"date": df.string(from: d.createdAt),
|
||||
"excerpt": excerpt
|
||||
]
|
||||
if d.isConsultation {
|
||||
item["kind"] = "问诊"
|
||||
} else if d.isMedicationLog {
|
||||
item["kind"] = "用药"
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
// 时间窗也给 LLM 看
|
||||
|
||||
169
康康/Services/SenseVoiceASRService.swift
Normal file
169
康康/Services/SenseVoiceASRService.swift
Normal file
@@ -0,0 +1,169 @@
|
||||
import Foundation
|
||||
import AVFoundation
|
||||
|
||||
/// 端侧问诊语音转写服务(SenseVoice,经 sherpa-mnn 跑在 MNN 后端)。
|
||||
///
|
||||
/// 「记录问诊」录完整段录音后调它做**离线转写**(非流式,无实时字幕),
|
||||
/// 转写稿再交本地 LLM(DiaryAssistService.organizeConsultation)整理成问诊小结。
|
||||
///
|
||||
/// 守红线:
|
||||
/// - 全程本机,无任何网络(§1 / §10 #1)。
|
||||
/// - 与 LLM/VL 经 `AIRuntime.runExclusiveForASR` 闸门**互斥**:转写前卸掉常驻文本/视觉模型腾内存,
|
||||
/// 避免两个模型同时常驻冲过单 App 内存上限被 jetsam 杀(§3.1 OOM 防护,见 [[airuntime-llm-vl-oom-gate]])。
|
||||
/// - 模型或引擎未就绪时本服务抛错,调用方自动回退系统端侧识别(SFSpeech),App 不卡死(§10 #5)。
|
||||
///
|
||||
/// 模型与 LLM 的 `ModelKind` 解耦,自管 `Models/SenseVoice/` 目录(model.mnn + tokens.txt)。
|
||||
/// 转换/接入步骤见 docs/release/sensevoice-integration.md。
|
||||
struct SenseVoiceASRService {
|
||||
static let shared = SenseVoiceASRService()
|
||||
private init() {}
|
||||
|
||||
enum ASRError: Error, LocalizedError {
|
||||
case modelNotInstalled
|
||||
case engineUnavailable
|
||||
case decodeFailed
|
||||
case empty
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .modelNotInstalled: return String(appLoc: "问诊转写模型未就绪")
|
||||
case .engineUnavailable: return String(appLoc: "本机暂不支持本地语音转写")
|
||||
case .decodeFailed: return String(appLoc: "录音解码失败")
|
||||
case .empty: return String(appLoc: "没识别到语音内容")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - 模型位置(独立于 LLM 的 ModelKind,问诊 ASR 自管目录)
|
||||
|
||||
/// `Application Support/Models/SenseVoice/`,与 LLM 模型同根目录、互不干扰。
|
||||
nonisolated static var modelDir: URL {
|
||||
ModelStore.shared.rootURL.appendingPathComponent("SenseVoice", isDirectory: true)
|
||||
}
|
||||
/// MNNConvert 产出的 SenseVoice 图。
|
||||
nonisolated static var modelFile: URL { modelDir.appendingPathComponent("model.mnn") }
|
||||
/// tokens.txt(id↔token 映射)。
|
||||
nonisolated static var tokensFile: URL { modelDir.appendingPathComponent("tokens.txt") }
|
||||
|
||||
/// 模型两件套是否都已就位(下载 / 旁路导入后)。
|
||||
nonisolated static var isModelInstalled: Bool {
|
||||
let fm = FileManager.default
|
||||
return fm.fileExists(atPath: modelFile.path) && fm.fileExists(atPath: tokensFile.path)
|
||||
}
|
||||
|
||||
/// 端侧 SenseVoice 是否可用 = 引擎已链接(sherpa-mnn)**且**模型已就位。
|
||||
/// false 时「记录问诊」自动回退系统端侧识别(SFSpeech)。
|
||||
nonisolated static var isAvailable: Bool {
|
||||
SenseVoiceBridge.isAvailable() && isModelInstalled
|
||||
}
|
||||
|
||||
// MARK: - 转写
|
||||
|
||||
/// 把一段录音(m4a/wav/caf 等)离线转写成文字。失败抛错,调用方回退。
|
||||
/// language:"auto" 自动判别(中英日韩粤),也可固定 "zh"。
|
||||
func transcribe(audioFileURL: URL, language: String = "auto") async throws -> String {
|
||||
guard SenseVoiceBridge.isAvailable() else { throw ASRError.engineUnavailable }
|
||||
guard Self.isModelInstalled else { throw ASRError.modelNotInstalled }
|
||||
|
||||
let modelPath = Self.modelFile.path
|
||||
let tokensPath = Self.tokensFile.path
|
||||
|
||||
// 解码 + 解码推理都重(CPU),全部放进闸门内的后台线程:与 LLM/VL 串行,且先卸常驻模型腾内存。
|
||||
let raw = try await AIRuntime.shared.runExclusiveForASR {
|
||||
try await Self.runOnBackground {
|
||||
let decoded = try Self.decodeToMonoFloat(url: audioFileURL)
|
||||
guard !decoded.samples.isEmpty else { throw ASRError.decodeFailed }
|
||||
guard let bridge = SenseVoiceBridge(modelPath: modelPath,
|
||||
tokensPath: tokensPath,
|
||||
language: language) else {
|
||||
throw ASRError.engineUnavailable
|
||||
}
|
||||
let text = decoded.samples.withUnsafeBufferPointer { buf -> String? in
|
||||
guard let base = buf.baseAddress else { return nil }
|
||||
return bridge.transcribeSamples(base,
|
||||
count: Int32(buf.count),
|
||||
sampleRate: Int32(decoded.sampleRate))
|
||||
}
|
||||
return text ?? ""
|
||||
}
|
||||
}
|
||||
|
||||
let cleaned = Self.cleanTranscript(raw)
|
||||
guard !cleaned.isEmpty else { throw ASRError.empty }
|
||||
return cleaned
|
||||
}
|
||||
|
||||
// MARK: - 纯函数(单测覆盖)
|
||||
|
||||
/// 清洗 SenseVoice 输出:去掉可能残留的 `<|zh|><|NEUTRAL|><|Speech|><|woitn|>` 等标签后 trim。
|
||||
/// sherpa 多数情况下已剥成纯文本,这里再兜一层,确保给 LLM 的转写稿干净。
|
||||
nonisolated static func cleanTranscript(_ raw: String) -> String {
|
||||
let stripped = raw.replacingOccurrences(
|
||||
of: "<\\|[^|]*\\|>", with: "", options: .regularExpression)
|
||||
return stripped.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
}
|
||||
|
||||
// MARK: - 音频解码(任意容器 → 16kHz 单声道 float32)
|
||||
|
||||
private struct Decoded { let samples: [Float]; let sampleRate: Int }
|
||||
|
||||
/// 用 AVAudioConverter 把录音重采样/混音成 16kHz 单声道 float32(SenseVoice 期望的输入域)。
|
||||
/// 一次性整段转换:本调用已在闸门内、LLM 已卸,瞬时内存可控。
|
||||
nonisolated private static func decodeToMonoFloat(url: URL,
|
||||
targetSampleRate: Double = 16000) throws -> Decoded {
|
||||
let file = try AVAudioFile(forReading: url)
|
||||
let inFormat = file.processingFormat
|
||||
let frameCount = AVAudioFrameCount(file.length)
|
||||
guard frameCount > 0 else { return Decoded(samples: [], sampleRate: Int(targetSampleRate)) }
|
||||
|
||||
guard let targetFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32,
|
||||
sampleRate: targetSampleRate,
|
||||
channels: 1,
|
||||
interleaved: false),
|
||||
let converter = AVAudioConverter(from: inFormat, to: targetFormat),
|
||||
let inBuffer = AVAudioPCMBuffer(pcmFormat: inFormat, frameCapacity: frameCount) else {
|
||||
throw ASRError.decodeFailed
|
||||
}
|
||||
try file.read(into: inBuffer)
|
||||
|
||||
// 目标采样率通常低于源(48k/44.1k→16k),输出更短;源若低于 16k 则 ratio>1,按比例 + 余量留足。
|
||||
let ratio = targetSampleRate / inFormat.sampleRate
|
||||
let outCapacity = AVAudioFrameCount(Double(frameCount) * ratio) + 1024
|
||||
guard let outBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: outCapacity) else {
|
||||
throw ASRError.decodeFailed
|
||||
}
|
||||
|
||||
var fed = false
|
||||
var convError: NSError?
|
||||
let status = converter.convert(to: outBuffer, error: &convError) { _, inStatus in
|
||||
if fed {
|
||||
inStatus.pointee = .endOfStream // 已喂完整段,通知 flush 余下重采样样本
|
||||
return nil
|
||||
}
|
||||
fed = true
|
||||
inStatus.pointee = .haveData
|
||||
return inBuffer
|
||||
}
|
||||
if let convError { throw convError }
|
||||
guard status != .error else { throw ASRError.decodeFailed }
|
||||
|
||||
guard let channel = outBuffer.floatChannelData?[0] else {
|
||||
return Decoded(samples: [], sampleRate: Int(targetSampleRate))
|
||||
}
|
||||
let n = Int(outBuffer.frameLength)
|
||||
let samples = Array(UnsafeBufferPointer(start: channel, count: n))
|
||||
return Decoded(samples: samples, sampleRate: Int(targetSampleRate))
|
||||
}
|
||||
|
||||
/// 把一段阻塞的同步工作放到后台 QoS 队列跑(转写解码同步阻塞,绝不能占住 actor/主线程)。
|
||||
nonisolated private static func runOnBackground<T: Sendable>(
|
||||
_ work: @escaping @Sendable () throws -> T
|
||||
) async throws -> T {
|
||||
try await withCheckedThrowingContinuation { cont in
|
||||
DispatchQueue.global(qos: .userInitiated).async {
|
||||
do { cont.resume(returning: try work()) }
|
||||
catch { cont.resume(throwing: error) }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,3 +4,4 @@
|
||||
//
|
||||
|
||||
#import "AI/MNN/MNNLLMBridge.h"
|
||||
#import "AI/MNN/SenseVoiceBridge.h"
|
||||
|
||||
Reference in New Issue
Block a user