diff --git a/scripts/build-mnn-xcframework.sh b/scripts/build-mnn-xcframework.sh
index 2fb7884..63d0937 100644
--- a/scripts/build-mnn-xcframework.sh
+++ b/scripts/build-mnn-xcframework.sh
@@ -7,16 +7,17 @@
# 需求:CMake 3.14+、Xcode、约 10-40 分钟。
#
# 关键 flag:
-# MNN_BUILD_LLM=ON —— 编入 llm 引擎(并导出 llm/llm.hpp),自动开 MNN_LOW_MEMORY
-# MNN_SME2=ON —— CMake 默认 ON,A19/iPhone17 运行时经 KleidiAI 自动启用,A17 回退 NEON
-# MNN_METAL=OFF —— 考核走 CPU+SME2,关 Metal 保持精简
-# MNN_BUILD_LLM_OMNI —— 如需 VL(图→文)再开,会额外拉 OpenCV/Audio(本脚本默认不开,文本优先)
+# MNN_BUILD_LLM=ON —— 编入 llm 引擎(并导出 llm/llm.hpp),自动开 MNN_LOW_MEMORY
+# MNN_BUILD_LLM_OMNI=ON —— VL(图→文)所需:多模态 Omni + OpenCV 图像解码。
+# 统一模型(Qwen3.5-4B-MNN 一肩挑文本+视觉)必须开。
+# MNN_SME2=ON —— CMake 默认 ON,A19/iPhone17 运行时经 KleidiAI 自动启用,A17 回退 NEON
+# MNN_METAL=OFF —— 考核走 CPU+SME2,关 Metal 保持精简
set -e
MNN_SRC="${MNN_SRC:-/Users/xuhuayong/apps/MNN-src}"
OUT_DIR="$(cd "$(dirname "$0")/.." && pwd)/Frameworks"
TOOLCHAIN_NEW="${MNN_SRC}/cmake/ios.toolchain.new.cmake"
-EXTRA="-DMNN_BUILD_LLM=ON -DMNN_METAL=OFF -DMNN_ARM82=true -DMNN_SME2=ON"
+EXTRA="-DMNN_BUILD_LLM=ON -DMNN_BUILD_LLM_OMNI=ON -DMNN_METAL=OFF -DMNN_ARM82=true -DMNN_SME2=ON"
COMMON="-DCMAKE_BUILD_TYPE=Release -DENABLE_BITCODE=0 -DMNN_AAPL_FMWK=1 -DMNN_SEP_BUILD=0 -DMNN_BUILD_SHARED_LIBS=false -DMNN_USE_THREAD_POOL=OFF"
export DEVELOPER_DIR="/Applications/Xcode.app/Contents/Developer"
diff --git a/康康/AI/AIRuntime.swift b/康康/AI/AIRuntime.swift
index e9d2153..862e588 100644
--- a/康康/AI/AIRuntime.swift
+++ b/康康/AI/AIRuntime.swift
@@ -255,6 +255,11 @@ actor AIRuntime {
/// 加载 VL 模型。幂等,首调真正 load。
func prepareVL() async throws {
+ // 选了 MNN 且多模态模型就绪:VL 复用同一个 MNN 模型(文本+视觉一肩挑),走 prepareMNN。
+ if InferenceEngine.current == .mnn, ModelStore.shared.isComplete(for: .mnnLLM) {
+ try await prepareMNN()
+ return
+ }
while vlStatus == .loading {
try await Task.sleep(nanoseconds: 80_000_000)
}
@@ -314,6 +319,16 @@ actor AIRuntime {
func analyzeReport(imageURLs: [URL],
prompt: String,
maxTokens: Int = 512) async throws -> String {
+ // 选了 MNN 且就绪:图→文走同一个 MNN 多模态模型。
+ if InferenceEngine.current == .mnn, mnnStatus == .ready {
+ await acquireGate()
+ defer { releaseGate() }
+ do {
+ return try await mnn.analyze(imageURLs: imageURLs, prompt: prompt, maxTokens: maxTokens)
+ } catch {
+ throw AIRuntimeError.inferenceFailed("\(error)")
+ }
+ }
guard vlStatus == .ready, let session = vlSession else {
throw AIRuntimeError.notReady
}
diff --git a/康康/AI/MNN/MNNLLMBridge.mm b/康康/AI/MNN/MNNLLMBridge.mm
index 8e134e2..ca089fb 100644
--- a/康康/AI/MNN/MNNLLMBridge.mm
+++ b/康康/AI/MNN/MNNLLMBridge.mm
@@ -135,28 +135,39 @@ private:
- (void)cancel { _cancel = true; }
-- (MNNGenerateStats *)generateText:(NSString *)prompt
- maxTokens:(int)maxTokens
- onToken:(void (^)(NSString *))onToken {
+// 统一生成:full 已是最终 prompt(文本,或含 ![]()
路径 标签)。
+// 多模态模型 createLLM 返回 Omni,response 解析 ![]()
标签并对路径 CV::imread(OMNI 框架内)。
+- (MNNGenerateStats *)runResponse:(NSString *)full
+ maxTokens:(int)maxTokens
+ onToken:(void (^)(NSString *))onToken {
_cancel = false;
TokenStreamBuf buf(onToken, &_cancel);
std::ostream os(&buf);
if (_llm) {
- _llm->response(std::string(prompt.UTF8String), &os, nullptr, maxTokens);
+ _llm->response(std::string(full.UTF8String), &os, nullptr, maxTokens);
}
buf.flush();
return [self statsFromContext];
}
+- (MNNGenerateStats *)generateText:(NSString *)prompt
+ maxTokens:(int)maxTokens
+ onToken:(void (^)(NSString *))onToken {
+ return [self runResponse:prompt maxTokens:maxTokens onToken:onToken];
+}
+
- (nullable MNNGenerateStats *)analyzeImages:(NSArray *)imagePaths
prompt:(NSString *)prompt
maxTokens:(int)maxTokens
onToken:(void (^)(NSString *))onToken
error:(NSError **)error {
- // VL 需 MNN_BUILD_LLM_OMNI 构建(OpenCV 解码图像)。当前文本构建不含,显式报错。
- if (error) *error = [NSError errorWithDomain:@"MNN" code:-2
- userInfo:@{NSLocalizedDescriptionKey: @"当前 MNN 构建未含 VL(OMNI),请用 OMNI 框架"}];
- return nil;
+ // 在 prompt 前拼 ![]()
本地路径;Omni 解析标签并对路径 imread(需 OMNI 框架)。
+ NSMutableString *full = [NSMutableString string];
+ for (NSString *p in imagePaths) {
+ [full appendFormat:@"![]()
%@", p];
+ }
+ [full appendString:prompt];
+ return [self runResponse:full maxTokens:maxTokens onToken:onToken];
}
- (MNNGenerateStats *)statsFromContext {
diff --git a/康康/AI/MNNBackend.swift b/康康/AI/MNNBackend.swift
index 92bfbd4..57bcae5 100644
--- a/康康/AI/MNNBackend.swift
+++ b/康康/AI/MNNBackend.swift
@@ -47,12 +47,34 @@ actor MNNBackend {
}
}
- /// 图→文(VL)。当前 MNN 文本构建未含 OMNI,直接抛错让上层回退 MLX VL。
- func analyze(imageURLs: [URL], prompt: String, maxTokens: Int) throws -> String {
- throw AIRuntimeError.inferenceFailed("MNN 当前构建不支持 VL(需 OMNI)")
+ /// 图→文(VL)。一次性收集(JSON 抽取不需流式)。桥接里把图片路径拼成 ![]()
标签,
+ /// MNN Omni 内部 imread 加载(需 OMNI 框架);blocking 调用放 detached 线程。
+ func analyze(imageURLs: [URL], prompt: String, maxTokens: Int) async throws -> String {
+ guard let bridge else { throw AIRuntimeError.notReady }
+ let paths = imageURLs.map(\.path)
+ let box = MNNUncheckedBox(bridge)
+ return try await withCheckedThrowingContinuation { cont in
+ Task.detached(priority: .userInitiated) {
+ let sink = MNNTextSink()
+ do {
+ _ = try box.value.analyzeImages(paths, prompt: prompt, maxTokens: Int32(maxTokens)) { piece in
+ sink.append(piece)
+ }
+ cont.resume(returning: sink.text)
+ } catch {
+ cont.resume(throwing: AIRuntimeError.inferenceFailed(error.localizedDescription))
+ }
+ }
+ }
}
}
+/// 单线程串行回调聚合文本,无竞争。
+private nonisolated final class MNNTextSink: @unchecked Sendable {
+ private(set) var text = ""
+ func append(_ s: String) { text += s }
+}
+
/// 把非 Sendable 的 ObjC 桥对象安全带过 detached 边界。
/// 安全性来自 `AIRuntime` 闸门:同一时刻只有一个生成在跑,桥不会被并发访问。
private nonisolated struct MNNUncheckedBox: @unchecked Sendable {