From ddfd474bb394483fb3670535ef72000cb1d2af6d Mon Sep 17 00:00:00 2001 From: link2026 Date: Mon, 8 Jun 2026 20:52:58 +0800 Subject: [PATCH] =?UTF-8?q?feat(AI):=20MNN=204B=20=E5=A4=9A=E6=A8=A1?= =?UTF-8?q?=E6=80=81=E4=B8=80=E8=82=A9=E6=8C=91=E6=96=87=E6=9C=AC+?= =?UTF-8?q?=E8=A7=86=E8=A7=89,=E5=90=88=E5=B9=B6=E4=B8=BA=E5=8D=95?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B(MLX=20=E4=BB=8D=E5=85=9C=E5=BA=95)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 利用 Qwen3.5-4B-MNN 本身是多模态(含 visual.mnn),让同一个 MNN 模型 同时做文本生成与拍照识别 → MNN 路径只需下 1 个模型(7.4GB→2.64GB)。 MLX(.llm/.vl)保留作兜底,尤其开发机 iPhone 15 Pro(A17 无 SME2)。 - MNN.xcframework 重建为 OMNI(MNN_BUILD_LLM_OMNI=ON,加 OpenCV 图像解码); 构建脚本同步加 OMNI flag - MNNLLMBridge.analyzeImages:把图片路径拼成 路径 标签 + response, Omni 内部 CV::imread 加载(无需桥接 include OpenCV);与 generateText 共用 runResponse - MNNBackend.analyze:detached 线程跑 blocking VL 调用,聚合为字符串 - AIRuntime:engine=.mnn 且就绪时,prepareVL→prepareMNN、analyzeReport→mnn.analyze; 否则回退 MLX VL device + 模拟器 BUILD SUCCEEDED,0 error,OMNI 框架链接干净。 VL 实际识别质量需真机用化验单 A/B(demo 核心)。 Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/build-mnn-xcframework.sh | 11 ++++++----- 康康/AI/AIRuntime.swift | 15 +++++++++++++++ 康康/AI/MNN/MNNLLMBridge.mm | 27 +++++++++++++++++++-------- 康康/AI/MNNBackend.swift | 28 +++++++++++++++++++++++++--- 4 files changed, 65 insertions(+), 16 deletions(-) diff --git a/scripts/build-mnn-xcframework.sh b/scripts/build-mnn-xcframework.sh index 2fb7884..63d0937 100644 --- a/scripts/build-mnn-xcframework.sh +++ b/scripts/build-mnn-xcframework.sh @@ -7,16 +7,17 @@ # 需求:CMake 3.14+、Xcode、约 10-40 分钟。 # # 关键 flag: -# MNN_BUILD_LLM=ON —— 编入 llm 引擎(并导出 llm/llm.hpp),自动开 MNN_LOW_MEMORY -# MNN_SME2=ON —— CMake 默认 ON,A19/iPhone17 运行时经 KleidiAI 自动启用,A17 回退 NEON -# MNN_METAL=OFF —— 考核走 CPU+SME2,关 Metal 保持精简 -# MNN_BUILD_LLM_OMNI —— 如需 VL(图→文)再开,会额外拉 OpenCV/Audio(本脚本默认不开,文本优先) +# MNN_BUILD_LLM=ON —— 编入 llm 引擎(并导出 llm/llm.hpp),自动开 MNN_LOW_MEMORY +# MNN_BUILD_LLM_OMNI=ON —— VL(图→文)所需:多模态 Omni + OpenCV 图像解码。 +# 统一模型(Qwen3.5-4B-MNN 一肩挑文本+视觉)必须开。 +# MNN_SME2=ON —— CMake 默认 ON,A19/iPhone17 运行时经 KleidiAI 自动启用,A17 回退 NEON +# MNN_METAL=OFF —— 考核走 CPU+SME2,关 Metal 保持精简 set -e MNN_SRC="${MNN_SRC:-/Users/xuhuayong/apps/MNN-src}" OUT_DIR="$(cd "$(dirname "$0")/.." && pwd)/Frameworks" TOOLCHAIN_NEW="${MNN_SRC}/cmake/ios.toolchain.new.cmake" -EXTRA="-DMNN_BUILD_LLM=ON -DMNN_METAL=OFF -DMNN_ARM82=true -DMNN_SME2=ON" +EXTRA="-DMNN_BUILD_LLM=ON -DMNN_BUILD_LLM_OMNI=ON -DMNN_METAL=OFF -DMNN_ARM82=true -DMNN_SME2=ON" COMMON="-DCMAKE_BUILD_TYPE=Release -DENABLE_BITCODE=0 -DMNN_AAPL_FMWK=1 -DMNN_SEP_BUILD=0 -DMNN_BUILD_SHARED_LIBS=false -DMNN_USE_THREAD_POOL=OFF" export DEVELOPER_DIR="/Applications/Xcode.app/Contents/Developer" diff --git a/康康/AI/AIRuntime.swift b/康康/AI/AIRuntime.swift index e9d2153..862e588 100644 --- a/康康/AI/AIRuntime.swift +++ b/康康/AI/AIRuntime.swift @@ -255,6 +255,11 @@ actor AIRuntime { /// 加载 VL 模型。幂等,首调真正 load。 func prepareVL() async throws { + // 选了 MNN 且多模态模型就绪:VL 复用同一个 MNN 模型(文本+视觉一肩挑),走 prepareMNN。 + if InferenceEngine.current == .mnn, ModelStore.shared.isComplete(for: .mnnLLM) { + try await prepareMNN() + return + } while vlStatus == .loading { try await Task.sleep(nanoseconds: 80_000_000) } @@ -314,6 +319,16 @@ actor AIRuntime { func analyzeReport(imageURLs: [URL], prompt: String, maxTokens: Int = 512) async throws -> String { + // 选了 MNN 且就绪:图→文走同一个 MNN 多模态模型。 + if InferenceEngine.current == .mnn, mnnStatus == .ready { + await acquireGate() + defer { releaseGate() } + do { + return try await mnn.analyze(imageURLs: imageURLs, prompt: prompt, maxTokens: maxTokens) + } catch { + throw AIRuntimeError.inferenceFailed("\(error)") + } + } guard vlStatus == .ready, let session = vlSession else { throw AIRuntimeError.notReady } diff --git a/康康/AI/MNN/MNNLLMBridge.mm b/康康/AI/MNN/MNNLLMBridge.mm index 8e134e2..ca089fb 100644 --- a/康康/AI/MNN/MNNLLMBridge.mm +++ b/康康/AI/MNN/MNNLLMBridge.mm @@ -135,28 +135,39 @@ private: - (void)cancel { _cancel = true; } -- (MNNGenerateStats *)generateText:(NSString *)prompt - maxTokens:(int)maxTokens - onToken:(void (^)(NSString *))onToken { +// 统一生成:full 已是最终 prompt(文本,或含 路径 标签)。 +// 多模态模型 createLLM 返回 Omni,response 解析 标签并对路径 CV::imread(OMNI 框架内)。 +- (MNNGenerateStats *)runResponse:(NSString *)full + maxTokens:(int)maxTokens + onToken:(void (^)(NSString *))onToken { _cancel = false; TokenStreamBuf buf(onToken, &_cancel); std::ostream os(&buf); if (_llm) { - _llm->response(std::string(prompt.UTF8String), &os, nullptr, maxTokens); + _llm->response(std::string(full.UTF8String), &os, nullptr, maxTokens); } buf.flush(); return [self statsFromContext]; } +- (MNNGenerateStats *)generateText:(NSString *)prompt + maxTokens:(int)maxTokens + onToken:(void (^)(NSString *))onToken { + return [self runResponse:prompt maxTokens:maxTokens onToken:onToken]; +} + - (nullable MNNGenerateStats *)analyzeImages:(NSArray *)imagePaths prompt:(NSString *)prompt maxTokens:(int)maxTokens onToken:(void (^)(NSString *))onToken error:(NSError **)error { - // VL 需 MNN_BUILD_LLM_OMNI 构建(OpenCV 解码图像)。当前文本构建不含,显式报错。 - if (error) *error = [NSError errorWithDomain:@"MNN" code:-2 - userInfo:@{NSLocalizedDescriptionKey: @"当前 MNN 构建未含 VL(OMNI),请用 OMNI 框架"}]; - return nil; + // 在 prompt 前拼 本地路径;Omni 解析标签并对路径 imread(需 OMNI 框架)。 + NSMutableString *full = [NSMutableString string]; + for (NSString *p in imagePaths) { + [full appendFormat:@"%@", p]; + } + [full appendString:prompt]; + return [self runResponse:full maxTokens:maxTokens onToken:onToken]; } - (MNNGenerateStats *)statsFromContext { diff --git a/康康/AI/MNNBackend.swift b/康康/AI/MNNBackend.swift index 92bfbd4..57bcae5 100644 --- a/康康/AI/MNNBackend.swift +++ b/康康/AI/MNNBackend.swift @@ -47,12 +47,34 @@ actor MNNBackend { } } - /// 图→文(VL)。当前 MNN 文本构建未含 OMNI,直接抛错让上层回退 MLX VL。 - func analyze(imageURLs: [URL], prompt: String, maxTokens: Int) throws -> String { - throw AIRuntimeError.inferenceFailed("MNN 当前构建不支持 VL(需 OMNI)") + /// 图→文(VL)。一次性收集(JSON 抽取不需流式)。桥接里把图片路径拼成 标签, + /// MNN Omni 内部 imread 加载(需 OMNI 框架);blocking 调用放 detached 线程。 + func analyze(imageURLs: [URL], prompt: String, maxTokens: Int) async throws -> String { + guard let bridge else { throw AIRuntimeError.notReady } + let paths = imageURLs.map(\.path) + let box = MNNUncheckedBox(bridge) + return try await withCheckedThrowingContinuation { cont in + Task.detached(priority: .userInitiated) { + let sink = MNNTextSink() + do { + _ = try box.value.analyzeImages(paths, prompt: prompt, maxTokens: Int32(maxTokens)) { piece in + sink.append(piece) + } + cont.resume(returning: sink.text) + } catch { + cont.resume(throwing: AIRuntimeError.inferenceFailed(error.localizedDescription)) + } + } + } } } +/// 单线程串行回调聚合文本,无竞争。 +private nonisolated final class MNNTextSink: @unchecked Sendable { + private(set) var text = "" + func append(_ s: String) { text += s } +} + /// 把非 Sendable 的 ObjC 桥对象安全带过 detached 边界。 /// 安全性来自 `AIRuntime` 闸门:同一时刻只有一个生成在跑,桥不会被并发访问。 private nonisolated struct MNNUncheckedBox: @unchecked Sendable {