From bf90dcd0ad822141e563d45daeccb70200521530 Mon Sep 17 00:00:00 2001 From: twlddd Date: Mon, 18 May 2026 14:32:40 +0800 Subject: [PATCH 1/7] Fix QNN AOT runtime and lowering issues --- mllm/backends/qnn/QNNAllocator.cpp | 50 +++++++++++++++++-- mllm/backends/qnn/QNNBackend.cpp | 18 ++++--- mllm/backends/qnn/QNNModel.cpp | 32 ++++++++++++ mllm/backends/qnn/QNNUtils.hpp | 3 +- mllm/backends/qnn/aot/QnnWrappersAPI.cpp | 52 +++++++++++++++++--- mllm/backends/qnn/aot/visitor/Matmul.cpp | 10 ++++ mllm/backends/qnn/aot/visitor/Repeat.cpp | 62 ++++++++++++++++-------- mllm/backends/qnn/aot/visitor/View.cpp | 16 ++++++ mllm/core/aops/VisionRoPEOp.cpp | 4 +- mllm/mllm.hpp | 4 +- 10 files changed, 206 insertions(+), 45 deletions(-) diff --git a/mllm/backends/qnn/QNNAllocator.cpp b/mllm/backends/qnn/QNNAllocator.cpp index dc04b5d0b..0f29d5e59 100644 --- a/mllm/backends/qnn/QNNAllocator.cpp +++ b/mllm/backends/qnn/QNNAllocator.cpp @@ -6,6 +6,8 @@ #include "mllm/utils/Common.hpp" #include "mllm/utils/Log.hpp" #include +#include +#include namespace mllm::qnn { @@ -78,7 +80,25 @@ void QNNAllocator::free(Storage* storage) { void QNNAllocator::registerQnnTensorToSharedBuffer(void* ptr, Qnn_Tensor_t& qnn_tensor) { // Make sure there has a memory that we can register to. - MLLM_RT_ASSERT(qnnMemPtrSet_.count(ptr)); + if (!qnnMemPtrSet_.count(ptr)) { + std::ostringstream dims; + dims << "["; + const auto rank = QNN_TENSOR_GET_RANK(qnn_tensor); + const auto* shape = QNN_TENSOR_GET_DIMENSIONS(qnn_tensor); + for (uint32_t i = 0; i < rank; ++i) { dims << (i == 0 ? "" : ",") << shape[i]; } + dims << "]"; + std::fprintf(stderr, + "QNN shared-buffer register failed: tensor='%s', ptr=%p, dtype=%d, rank=%u, dims=%s is not owned by " + "QNNAllocator (owned ptr count=%zu)\n", + QNN_TENSOR_GET_NAME(qnn_tensor) ? QNN_TENSOR_GET_NAME(qnn_tensor) : "", ptr, + static_cast(QNN_TENSOR_GET_DATA_TYPE(qnn_tensor)), rank, dims.str().c_str(), qnnMemPtrSet_.size()); + std::fflush(stderr); + MLLM_ERROR("QNN shared-buffer register failed: tensor='{}', ptr={}, dtype={}, rank={}, dims={} is not owned by " + "QNNAllocator (owned ptr count={})", + QNN_TENSOR_GET_NAME(qnn_tensor) ? QNN_TENSOR_GET_NAME(qnn_tensor) : "", ptr, + static_cast(QNN_TENSOR_GET_DATA_TYPE(qnn_tensor)), rank, dims.str(), qnnMemPtrSet_.size()); + MLLM_RT_ASSERT(qnnMemPtrSet_.count(ptr)); + } // if already registered, just set the mem handle if (ptrToFdAndMemHandleMap_.count(ptr) > 0) { @@ -90,7 +110,14 @@ void QNNAllocator::registerQnnTensorToSharedBuffer(void* ptr, Qnn_Tensor_t& qnn_ // Get the file id of this memory space. int mem_fd = rpcmem_to_fd(ptr); - MLLM_RT_ASSERT(mem_fd != -1); + if (mem_fd == -1) { + std::fprintf(stderr, "QNN shared-buffer register failed: rpcmem_to_fd returned -1 for tensor='%s', ptr=%p\n", + QNN_TENSOR_GET_NAME(qnn_tensor) ? QNN_TENSOR_GET_NAME(qnn_tensor) : "", ptr); + std::fflush(stderr); + MLLM_ERROR("QNN shared-buffer register failed: rpcmem_to_fd returned -1 for tensor='{}', ptr={}", + QNN_TENSOR_GET_NAME(qnn_tensor) ? QNN_TENSOR_GET_NAME(qnn_tensor) : "", ptr); + MLLM_RT_ASSERT(mem_fd != -1); + } // Make qnn memory descriptor. Set ION. Qnn_MemDescriptor_t mem_descriptor = QNN_MEM_DESCRIPTOR_INIT; @@ -106,7 +133,24 @@ void QNNAllocator::registerQnnTensorToSharedBuffer(void* ptr, Qnn_Tensor_t& qnn_ // Register to QNN memory Qnn_MemHandle_t mem_handle = QNN_TENSOR_GET_MEM_HANDLE(qnn_tensor); - MLLM_RT_ASSERT_EQ(QNN_SUCCESS, qnnInterface_.memRegister(context_, &mem_descriptor, 1u, &mem_handle)); + Qnn_ErrorHandle_t status = qnnInterface_.memRegister(context_, &mem_descriptor, 1u, &mem_handle); + if (QNN_SUCCESS != status) { + std::ostringstream dims; + dims << "["; + const auto rank = QNN_TENSOR_GET_RANK(qnn_tensor); + const auto* shape = QNN_TENSOR_GET_DIMENSIONS(qnn_tensor); + for (uint32_t i = 0; i < rank; ++i) { dims << (i == 0 ? "" : ",") << shape[i]; } + dims << "]"; + std::fprintf(stderr, "QNN memRegister failed: status=%lu, tensor='%s', ptr=%p, fd=%d, dtype=%d, rank=%u, dims=%s\n", + static_cast(status), + QNN_TENSOR_GET_NAME(qnn_tensor) ? QNN_TENSOR_GET_NAME(qnn_tensor) : "", ptr, mem_fd, + static_cast(QNN_TENSOR_GET_DATA_TYPE(qnn_tensor)), rank, dims.str().c_str()); + std::fflush(stderr); + MLLM_ERROR("QNN memRegister failed: status={}, tensor='{}', ptr={}, fd={}, dtype={}, rank={}, dims={}", status, + QNN_TENSOR_GET_NAME(qnn_tensor) ? QNN_TENSOR_GET_NAME(qnn_tensor) : "", ptr, mem_fd, + static_cast(QNN_TENSOR_GET_DATA_TYPE(qnn_tensor)), rank, dims.str()); + MLLM_RT_ASSERT_EQ(QNN_SUCCESS, status); + } QNN_TENSOR_SET_MEM_HANDLE(qnn_tensor, mem_handle); diff --git a/mllm/backends/qnn/QNNBackend.cpp b/mllm/backends/qnn/QNNBackend.cpp index 3900afc35..5cbccb1c2 100644 --- a/mllm/backends/qnn/QNNBackend.cpp +++ b/mllm/backends/qnn/QNNBackend.cpp @@ -657,13 +657,17 @@ void QNNBackend::graphExecute(const std::string& graphName, std::vector& inputs.size(), graphName); return; } + if (outputs.size() != model->getGraphOutputTensorWrappers().size()) { + MLLM_ERROR("Output size mismatch: expected {}, got {} for graph '{}'", model->getGraphOutputTensorWrappers().size(), + outputs.size(), graphName); + return; + } std::vector qnn_inputs; std::vector qnn_outputs; // Prepare QNN inputs for (int i = 0; i < model->getGraphInputTensorWrappers().size(); i++) { auto wrapper = model->getGraphInputTensorWrappers()[i]; - auto& wrapper_tensor = wrapper->getDataContainer(); const auto& runtime_input = inputs[i]; // Validate input tensors @@ -672,9 +676,9 @@ void QNNBackend::graphExecute(const std::string& graphName, std::vector& return; } - // Case of executing retrieved graph created by AOT - // input wrapper is empty, set wrapper's dataContainer(mllm::Tensor) - if (!wrapper->isAlloc()) { wrapper->__setDataContainer(runtime_input); } + // Retrieved AOT graphs may be executed repeatedly with different runtime buffers + // in diagnostic paths. Rebind on every execution so QNN sees the current tensor. + wrapper->__setDataContainer(runtime_input); // Allocate and register the wrapper tensor with QNN allocator // QNNAllocator will handle registered memory descriptor when needed @@ -684,7 +688,6 @@ void QNNBackend::graphExecute(const std::string& graphName, std::vector& // Prepare QNN outputs for (int j = 0; j < model->getGraphOutputTensorWrappers().size(); j++) { auto wrapper = model->getGraphOutputTensorWrappers()[j]; - auto& wrapper_tensor = wrapper->getDataContainer(); const auto& runtime_output = outputs[j]; // Validate output tensors @@ -693,8 +696,9 @@ void QNNBackend::graphExecute(const std::string& graphName, std::vector& return; } - // output wrapper is empty, set wrapper's dataContainer(mllm::Tensor) - if (!wrapper->isAlloc()) { wrapper->__setDataContainer(runtime_output); } + // Retrieved AOT graphs may be executed repeatedly with different runtime buffers + // in diagnostic paths. Rebind on every execution so QNN writes to the current tensor. + wrapper->__setDataContainer(runtime_output); // alloc and register qnn tensor wrapper->alloc(); // QNNAllocator will handle registered memory descriptor diff --git a/mllm/backends/qnn/QNNModel.cpp b/mllm/backends/qnn/QNNModel.cpp index 6fc6110bf..75221e89d 100644 --- a/mllm/backends/qnn/QNNModel.cpp +++ b/mllm/backends/qnn/QNNModel.cpp @@ -3,12 +3,34 @@ #include "mllm/backends/qnn/QNNModel.hpp" #include +#include +#include #include "mllm/backends/qnn/QNNTypeMacros.hpp" #include "mllm/backends/qnn/QNNUtils.hpp" #include "mllm/utils/Log.hpp" namespace mllm::qnn { +namespace { + +bool shouldDumpQnnIO() { + const char* flag = std::getenv("MLLM_QNN_DUMP_IO"); + return flag != nullptr && std::string(flag) != "0"; +} + +std::string dimsToString(const std::vector& dims) { + std::ostringstream oss; + oss << "["; + for (size_t i = 0; i < dims.size(); ++i) { + if (i > 0) { oss << ", "; } + oss << dims[i]; + } + oss << "]"; + return oss.str(); +} + +} // namespace + template void freeMultiPtr(Args... args) { (free(args), ...); @@ -112,6 +134,11 @@ ModelError_t QNNModel::loadGraphTensorInfo(const Qnn_Tensor_t* inputTensors, uin inputTensorWrappers_.push_back(wrapper); tensorWrapperMap_[tensorName] = wrapper; + + if (shouldDumpQnnIO()) { + MLLM_INFO("QNN graph {} input[{}]: name='{}', dtype={}, dims={}", graphName_, i, tensorName, + static_cast(QNN_TENSOR_GET_DATA_TYPE(tensor)), dimsToString(dimensions)); + } } // Create wrappers for output tensors @@ -134,6 +161,11 @@ ModelError_t QNNModel::loadGraphTensorInfo(const Qnn_Tensor_t* inputTensors, uin outputTensorWrappers_.push_back(wrapper); tensorWrapperMap_[tensorName] = wrapper; + + if (shouldDumpQnnIO()) { + MLLM_INFO("QNN graph {} output[{}]: name='{}', dtype={}, dims={}", graphName_, i, tensorName, + static_cast(QNN_TENSOR_GET_DATA_TYPE(tensor)), dimsToString(dimensions)); + } } MLLM_INFO("QNNModel::loadGraphTensorInfo() loaded {} input tensors and {} output tensors for graph: {}", numInputTensors, diff --git a/mllm/backends/qnn/QNNUtils.hpp b/mllm/backends/qnn/QNNUtils.hpp index 36fb6a91c..e798b29dd 100644 --- a/mllm/backends/qnn/QNNUtils.hpp +++ b/mllm/backends/qnn/QNNUtils.hpp @@ -209,9 +209,8 @@ class QNNTensorWrapper { bool isAlloc() { return isAlloc_; } void __setDataContainer(const Tensor& tensor) { - MLLM_RT_ASSERT(dataContainer_.isNil()) dataContainer_ = tensor; - if (!tensor.isNil()) { isAlloc_ = true; } + isAlloc_ = !tensor.isNil(); } // Helper to set complex quantization params and manage memory diff --git a/mllm/backends/qnn/aot/QnnWrappersAPI.cpp b/mllm/backends/qnn/aot/QnnWrappersAPI.cpp index 2a2e6010f..8d7a9e501 100644 --- a/mllm/backends/qnn/aot/QnnWrappersAPI.cpp +++ b/mllm/backends/qnn/aot/QnnWrappersAPI.cpp @@ -23,9 +23,18 @@ namespace mllm::qnn::aot { +namespace { + +std::string qnnTensorNameFromIR(const ir::tensor::TensorValue::ptr_t& v) { + if (v && v->hasSymbolAttr()) { return v->getSymbolAttr()->str(); } + return v ? v->name() : ""; +} + +} // namespace + QnnAOTNodeTensor::QnnAOTNodeTensor(const ir::tensor::TensorValue::ptr_t& v, bool force_static_weight) { auto type = parseQnnTensorTypeFromIR(v); - auto name = v->name(); + auto name = parseQnnTensorNameFromIR(v); auto quant = parseQnnQuantizeParamFromIR(v); if (force_static_weight || type == QNN_TENSOR_TYPE_STATIC) { @@ -103,7 +112,9 @@ Qnn_DataType_t QnnAOTNodeTensor::parseQnnDataTypeFromIR(const ir::tensor::Tensor return mllm::qnn::mllmDataTypeToQnnDataType(v->tensor_.dtype()); } -std::string QnnAOTNodeTensor::parseQnnTensorNameFromIR(const ir::tensor::TensorValue::ptr_t& v) { return v->name(); } +std::string QnnAOTNodeTensor::parseQnnTensorNameFromIR(const ir::tensor::TensorValue::ptr_t& v) { + return qnnTensorNameFromIR(v); +} Qnn_QuantizeParams_t QnnAOTNodeTensor::parseQnnQuantizeParamFromIR(const ir::tensor::TensorValue::ptr_t& v) { Qnn_QuantizeParams_t ret = QNN_QUANTIZE_PARAMS_INIT; @@ -139,10 +150,30 @@ Qnn_QuantizeParams_t QnnAOTNodeTensor::parseQnnQuantizeParamFromIR(const ir::ten MLLM_ERROR_EXIT(ExitCode::kCoreError, "SymPerTensor quant recipe has no scale. tensor: {}", v->name()); } - MLLM_RT_ASSERT_EQ(cfg->quant_to_type, kUInt8); + int32_t offset = 0; + switch (cfg->quant_to_type) { + case kUInt8: { + offset = -128; + break; + } + case kUInt16: { + offset = -32768; + break; + } + case kInt8: + case kInt16: { + offset = 0; + break; + } + default: { + MLLM_ERROR_EXIT(ExitCode::kCoreError, "Unsupported SymPerTensor quant target type {} for tensor: {}", + nameOfType(cfg->quant_to_type), v->name()); + } + } - ret.scaleOffsetEncoding = Qnn_ScaleOffset_t{.scale = cfg->scale.item(), .offset = -128}; - MLLM_INFO("Configuring SymPerTensor quantization for tensor: {}, scale: {}", v->name(), cfg->scale.item()); + ret.scaleOffsetEncoding = Qnn_ScaleOffset_t{.scale = cfg->scale.item(), .offset = offset}; + MLLM_INFO("Configuring SymPerTensor quantization for tensor: {}, scale: {}, offset: {}", v->name(), + cfg->scale.item(), offset); break; } default: { @@ -335,8 +366,13 @@ void QnnAOTGraph::addOperation(const QnnAOTNodeOperation::ptr_t& qnn_op) { for (auto& in : qnn_op->inputs) qnn_model_->addTensorWrapper(in->getWrapper()); for (auto& out : qnn_op->outputs) qnn_model_->addTensorWrapper(out->getWrapper()); - qnn_model_->addNode(QNN_OPCONFIG_VERSION_1, qnn_op->name_, qnn_op->package_name_, qnn_op->op_name_, qnn_op->param_tensor, - qnn_op->param_scalar, inputNames, outputNames); + auto add_node_status = + qnn_model_->addNode(QNN_OPCONFIG_VERSION_1, qnn_op->name_, qnn_op->package_name_, qnn_op->op_name_, + qnn_op->param_tensor, qnn_op->param_scalar, inputNames, outputNames); + if (add_node_status != mllm::qnn::MODEL_NO_ERROR) { + MLLM_ERROR_EXIT(ExitCode::kCoreError, "QNN AOT failed to add node {} (op type {}) to graph.", qnn_op->name_, + qnn_op->op_name_); + } op_node_.insert({qnn_op->getName(), qnn_op}); } @@ -686,7 +722,7 @@ void QnnAOTEnv::captureAOTNodeOp(const std::string& qnn_context_name, const std: QnnAOTNodeTensor::ptr_t QnnAOTEnv::captureQnnAOTNodeTensor(const std::string& qnn_context_name, const std::string& graph_name, const ir::tensor::TensorValue::ptr_t& v, bool force_static_weight) { - auto __qnn_tensor_name = v->name(); + auto __qnn_tensor_name = qnnTensorNameFromIR(v); bool __qnn_enable_static_weight = force_static_weight; diff --git a/mllm/backends/qnn/aot/visitor/Matmul.cpp b/mllm/backends/qnn/aot/visitor/Matmul.cpp index b44afd780..90e3f90bf 100644 --- a/mllm/backends/qnn/aot/visitor/Matmul.cpp +++ b/mllm/backends/qnn/aot/visitor/Matmul.cpp @@ -4,6 +4,7 @@ #include "mllm/utils/Common.hpp" #include "mllm/compile/ir/linalg/Op.hpp" #include "mllm/compile/ir/builtin/Attribute.hpp" +#include "mllm/core/aops/MatMulOp.hpp" #include "mllm/backends/qnn/aot/QnnWrappersAPI.hpp" #include "mllm/backends/qnn/aot/visitor/Matmul.hpp" #include "mllm/backends/qnn/aot/passes/AOTCompileContext.hpp" @@ -22,6 +23,11 @@ bool QnnAOTMatMulPattern::rewrite(ir::IRWriter& writer, const ir::op_ptr_t& op) MLLM_ERROR("Failed to cast to linalg::MatMulOp"); return false; } + auto aop = dynamic_cast(matmul_op->getAOp()); + if (!aop) { + MLLM_ERROR("Failed to cast AOp to aops::MatMulOp"); + return false; + } MLLM_RETURN_FALSE_IF_NOT(op->getAttr("qnn_graph_name")); auto qnn_graph_name = op->getAttr("qnn_graph_name")->cast_()->data(); @@ -44,6 +50,10 @@ bool QnnAOTMatMulPattern::rewrite(ir::IRWriter& writer, const ir::op_ptr_t& op) ->emplaceOutput(env->captureQnnAOTNodeTensor(qnn_context_name, qnn_graph_name, output)) ->setName(matmul_op->getAOp()->getName()); + const auto& options = aop->options(); + qnn_op_node->emplaceParamScalar(QNNParamScalarWrapper::create("transpose_in0", options.transpose_a)); + qnn_op_node->emplaceParamScalar(QNNParamScalarWrapper::create("transpose_in1", options.transpose_b)); + // Register this op node into one graph. env->captureAOTNodeOp(qnn_context_name, qnn_graph_name, qnn_op_node); diff --git a/mllm/backends/qnn/aot/visitor/Repeat.cpp b/mllm/backends/qnn/aot/visitor/Repeat.cpp index e6eb0542a..27a482e8e 100644 --- a/mllm/backends/qnn/aot/visitor/Repeat.cpp +++ b/mllm/backends/qnn/aot/visitor/Repeat.cpp @@ -8,6 +8,7 @@ #include "mllm/backends/qnn/aot/visitor/Repeat.hpp" #include "mllm/backends/qnn/aot/passes/AOTCompileContext.hpp" #include "mllm/core/aops/RepeatOp.hpp" +#include "mllm/core/Tensor.hpp" #include namespace mllm::qnn::aot { @@ -49,36 +50,55 @@ bool QnnAOTRepeatPattern::rewrite(ir::IRWriter& writer, const ir::op_ptr_t& op) if (dim < 0) { dim += rank; } - std::vector multiples(rank, 1); + std::vector multiples(rank + 1, 1); if (dim >= 0 && dim < rank) { - multiples[dim] = (uint32_t)repeat_times; + multiples[dim + 1] = (uint32_t)repeat_times; } else { MLLM_ERROR("Invalid dimension for RepeatOp: {}", dim); return false; } - // Create QNN Op Node - // QNN uses "Tile" for repeat - auto qnn_op_node = QnnAOTNodeOperation::create("Tile"); - qnn_op_node->setPackageName("qti.aisw"); - - // Add Input - qnn_op_node->emplaceInput(env->captureQnnAOTNodeTensor(qnn_context_name, qnn_graph_name, input)); - - // Add multiples Param + // mllm Repeat semantics are repeat_interleave along one dimension. QNN Tile + // repeats whole dimensions, so use reshape + tile + reshape: + // [.., D, ..] -> [.., D, 1, ..] -> tile inserted dim -> [.., D * repeat, ..]. + auto expanded_shape = input_shape; + expanded_shape.insert(expanded_shape.begin() + dim + 1, 1); + auto tiled_shape = expanded_shape; + tiled_shape[dim + 1] = repeat_times; + + auto expanded = writer.getContext()->create( + Tensor::empty(expanded_shape, input->tensor_.dtype(), input->tensor_.device())); + expanded->setAttr("quant_recipe", input->getAttr("quant_recipe")); + auto tiled = writer.getContext()->create( + Tensor::empty(tiled_shape, input->tensor_.dtype(), input->tensor_.device())); + tiled->setAttr("quant_recipe", input->getAttr("quant_recipe")); + + auto reshape_in = QnnAOTNodeOperation::create("Reshape"); + reshape_in->setPackageName("qti.aisw"); + reshape_in->emplaceInput(env->captureQnnAOTNodeTensor(qnn_context_name, qnn_graph_name, input)) + ->emplaceOutput(env->captureQnnAOTNodeTensor(qnn_context_name, qnn_graph_name, expanded)) + ->setName(base_op->getName() + ".repeat_interleave_reshape_in"); + env->captureAOTNodeOp(qnn_context_name, qnn_graph_name, reshape_in); + + auto tile = QnnAOTNodeOperation::create("Tile"); + tile->setPackageName("qti.aisw"); + tile->emplaceInput(env->captureQnnAOTNodeTensor(qnn_context_name, qnn_graph_name, expanded)); auto multiplesName = base_op->getName() + ".multiples"; - auto multiplesParam = - QNNParamTensorWrapper::create("multiples", multiplesName, QNN_DATATYPE_UINT_32, std::vector{(uint32_t)rank}); + auto multiplesParam = QNNParamTensorWrapper::create("multiples", multiplesName, QNN_DATATYPE_UINT_32, + std::vector{(uint32_t)multiples.size()}); uint32_t* multiplesData = static_cast(multiplesParam->alloc()); - std::memcpy(multiplesData, multiples.data(), rank * sizeof(uint32_t)); - qnn_op_node->emplaceParamTensor(multiplesParam); - - // Add Output - qnn_op_node->emplaceOutput(env->captureQnnAOTNodeTensor(qnn_context_name, qnn_graph_name, output)) + std::memcpy(multiplesData, multiples.data(), multiples.size() * sizeof(uint32_t)); + tile->emplaceParamTensor(multiplesParam); + tile->emplaceOutput(env->captureQnnAOTNodeTensor(qnn_context_name, qnn_graph_name, tiled)) + ->setName(base_op->getName() + ".repeat_interleave_tile"); + env->captureAOTNodeOp(qnn_context_name, qnn_graph_name, tile); + + auto reshape_out = QnnAOTNodeOperation::create("Reshape"); + reshape_out->setPackageName("qti.aisw"); + reshape_out->emplaceInput(env->captureQnnAOTNodeTensor(qnn_context_name, qnn_graph_name, tiled)) + ->emplaceOutput(env->captureQnnAOTNodeTensor(qnn_context_name, qnn_graph_name, output)) ->setName(base_op->getName()); - - // Register - env->captureAOTNodeOp(qnn_context_name, qnn_graph_name, qnn_op_node); + env->captureAOTNodeOp(qnn_context_name, qnn_graph_name, reshape_out); return true; } diff --git a/mllm/backends/qnn/aot/visitor/View.cpp b/mllm/backends/qnn/aot/visitor/View.cpp index e3447edd9..279a16b53 100644 --- a/mllm/backends/qnn/aot/visitor/View.cpp +++ b/mllm/backends/qnn/aot/visitor/View.cpp @@ -35,6 +35,22 @@ bool QnnAOTViewPattern::rewrite(ir::IRWriter& writer, const ir::op_ptr_t& op) { // Output auto output = op->outputs().front()->cast_(); + // mllm ViewOp can be a metadata-only no-op and reuse the same TensorValue name + // for input/output. QNN Reshape cannot write back to the exact same graph + // tensor, so keep true no-op views as aliases. Shape-changing views should be + // traced with enable_ssa=true by model code so that QNN receives a distinct + // output tensor with the new shape. + if (input->name() == output->name()) { + if (input->tensor_.shape() == output->tensor_.shape()) { + env->captureQnnAOTNodeTensor(qnn_context_name, qnn_graph_name, input); + return true; + } + MLLM_ERROR("QNN AOT ViewOp {} changes shape from [{}] to [{}] but input/output share tensor name {}. " + "Use Tensor::view(..., true) for this path.", + view_op->getAOp()->getName(), input->tensor_.shape(), output->tensor_.shape(), input->name()); + return false; + } + // Create Shape Tensor auto output_shape = output->tensor_.shape(); std::vector shape_data; diff --git a/mllm/core/aops/VisionRoPEOp.cpp b/mllm/core/aops/VisionRoPEOp.cpp index fdd0e9200..6197788d5 100644 --- a/mllm/core/aops/VisionRoPEOp.cpp +++ b/mllm/core/aops/VisionRoPEOp.cpp @@ -9,7 +9,7 @@ namespace mllm::aops { -VisionRoPEOp::VisionRoPEOp(const VisionRoPEOpOptions& options) : BaseOp(OpTypes::kSiLU), options_(options) {} +VisionRoPEOp::VisionRoPEOp(const VisionRoPEOpOptions& options) : BaseOp(OpTypes::kVisionRoPE), options_(options) {} void VisionRoPEOp::load(const ParameterFile::ptr_t& ploader) { MLLM_EMPTY_SCOPE; } @@ -31,4 +31,4 @@ void VisionRoPEOp::reshape(const std::vector& inputs, std::vector& inputs, std::vector& outputs) { BaseOp::setup(inputs, outputs); } -} // namespace mllm::aops \ No newline at end of file +} // namespace mllm::aops diff --git a/mllm/mllm.hpp b/mllm/mllm.hpp index 27ea0abe0..35331c1cd 100644 --- a/mllm/mllm.hpp +++ b/mllm/mllm.hpp @@ -330,12 +330,12 @@ inline void print_stack_trace() { const char* name = demangled ? demangled : info.dli_sname; char line[256]; int len = snprintf(line, sizeof(line), "#%d %p %s\n", i, buffer[i], name); - safe_write(line, len); + safe_write(line, std::min(static_cast(std::max(len, 0)), sizeof(line) - 1)); free(demangled); } else { char line[256]; int len = snprintf(line, sizeof(line), "#%d %p\n", i, buffer[i]); - safe_write(line, len); + safe_write(line, std::min(static_cast(std::max(len, 0)), sizeof(line) - 1)); } } #endif From 7436ea39cc9e0031f4d424f3c892d7bbe5e54f67 Mon Sep 17 00:00:00 2001 From: twlddd Date: Mon, 18 May 2026 14:47:43 +0800 Subject: [PATCH 2/7] Add QNN AOT support for visual graph lowering --- mllm/backends/qnn/aot/passes/AOTPipeline.cpp | 18 +++ mllm/backends/qnn/aot/passes/AOTPipeline.hpp | 5 + .../qnn/aot/passes/LLM2QnnLoweringPass.cpp | 119 +++++++++++++++++- .../qnn/aot/passes/LLM2QnnLoweringPass.hpp | 8 +- .../qnn/aot/passes/LLMQuantRecipePass.cpp | 103 +++++++++++++-- .../qnn/aot/passes/LLMQuantRecipePass.hpp | 28 +++++ mllm/backends/qnn/aot/passes/PTQPass.cpp | 78 ++++++++---- mllm/backends/qnn/aot/visitor/Conv2D.cpp | 6 + mllm/backends/qnn/aot/visitor/GELU.cpp | 47 +++++++ mllm/backends/qnn/aot/visitor/GELU.hpp | 23 ++++ mllm/backends/qnn/aot/visitor/LayerNorm.cpp | 82 ++++++++++++ mllm/backends/qnn/aot/visitor/LayerNorm.hpp | 23 ++++ 12 files changed, 499 insertions(+), 41 deletions(-) create mode 100644 mllm/backends/qnn/aot/visitor/GELU.cpp create mode 100644 mllm/backends/qnn/aot/visitor/GELU.hpp create mode 100644 mllm/backends/qnn/aot/visitor/LayerNorm.cpp create mode 100644 mllm/backends/qnn/aot/visitor/LayerNorm.hpp diff --git a/mllm/backends/qnn/aot/passes/AOTPipeline.cpp b/mllm/backends/qnn/aot/passes/AOTPipeline.cpp index 1b43ca551..64e8da406 100644 --- a/mllm/backends/qnn/aot/passes/AOTPipeline.cpp +++ b/mllm/backends/qnn/aot/passes/AOTPipeline.cpp @@ -37,4 +37,22 @@ std::vector> createQnnAOTLoweringPipeline(QnnAOTEnv* e return ret; } + +std::vector> createQnnAOTSimpleLoweringPipeline(QnnAOTEnv* env, const std::string& config_path, + const ParameterFile::ptr_t& pf, + const std::string& qnn_graph_name) { + std::vector ret; + + AOTCompileContext::getInstance().setEnv(env); + AOTCompileContext::getInstance().setConfig(config_path); + AOTCompileContext::getInstance().setParamFile(pf); + + ret.emplace_back(createMarkQnnGraphPass()); + ret.emplace_back(createOpNamingPass()); + ret.emplace_back(createLLMQuantRecipePass()); + ret.emplace_back(createPTQPass()); + ret.emplace_back(createLLM2QnnLoweringPass(qnn_graph_name)); + + return ret; +} } // namespace mllm::qnn::aot diff --git a/mllm/backends/qnn/aot/passes/AOTPipeline.hpp b/mllm/backends/qnn/aot/passes/AOTPipeline.hpp index d854de974..3fd042987 100644 --- a/mllm/backends/qnn/aot/passes/AOTPipeline.hpp +++ b/mllm/backends/qnn/aot/passes/AOTPipeline.hpp @@ -3,6 +3,7 @@ #pragma once +#include #include #include "mllm/backends/qnn/aot/QnnWrappersAPI.hpp" @@ -14,4 +15,8 @@ namespace mllm::qnn::aot { std::vector> createQnnAOTLoweringPipeline(QnnAOTEnv* env, const std::string& config_path, const ParameterFile::ptr_t& pf); +std::vector> createQnnAOTSimpleLoweringPipeline(QnnAOTEnv* env, const std::string& config_path, + const ParameterFile::ptr_t& pf, + const std::string& qnn_graph_name = ""); + } // namespace mllm::qnn::aot diff --git a/mllm/backends/qnn/aot/passes/LLM2QnnLoweringPass.cpp b/mllm/backends/qnn/aot/passes/LLM2QnnLoweringPass.cpp index 69bd93579..dce933cf3 100644 --- a/mllm/backends/qnn/aot/passes/LLM2QnnLoweringPass.cpp +++ b/mllm/backends/qnn/aot/passes/LLM2QnnLoweringPass.cpp @@ -1,7 +1,9 @@ // Copyright (c) MLLM Team. // Licensed under the MIT License. +#include #include +#include #include "mllm/backends/qnn/aot/passes/LLM2QnnLoweringPass.hpp" #include "mllm/backends/qnn/aot/passes/AOTCompileContext.hpp" @@ -16,9 +18,11 @@ #include "mllm/backends/qnn/aot/visitor/Elewise.hpp" #include "mllm/backends/qnn/aot/visitor/Embedding.hpp" #include "mllm/backends/qnn/aot/visitor/Gather.hpp" +#include "mllm/backends/qnn/aot/visitor/GELU.hpp" #include "mllm/backends/qnn/aot/visitor/CastType.hpp" #include "mllm/backends/qnn/aot/visitor/View.hpp" #include "mllm/backends/qnn/aot/visitor/Index.hpp" +#include "mllm/backends/qnn/aot/visitor/LayerNorm.hpp" #include "mllm/backends/qnn/aot/visitor/RMSNorm.hpp" #include "mllm/backends/qnn/aot/visitor/Linear.hpp" #include "mllm/backends/qnn/aot/visitor/Concat.hpp" @@ -34,12 +38,18 @@ namespace mllm::qnn::aot { -LLM2QnnLoweringPass::LLM2QnnLoweringPass() { +namespace { + +} // namespace + +LLM2QnnLoweringPass::LLM2QnnLoweringPass(std::string simple_qnn_graph_name) + : simple_qnn_graph_name_(std::move(simple_qnn_graph_name)) { registerPatterns(); + QnnAOTEqualPattern, QnnAOTWherePattern, QnnAOTSoftmaxPattern, QnnAOTSigmoidPattern, QnnAOTConv2DPattern, + QnnAOTGELUPattern, QnnAOTLayerNormPattern>(); } uint8_t LLM2QnnLoweringPass::run(const ir::node_ptr_t& op) { @@ -66,8 +76,12 @@ uint8_t LLM2QnnLoweringPass::run(const ir::node_ptr_t& op) { // Check call graph op point to a subgraph named "model" auto symbol_attr = call_graph_op->getSymbolAttr(); if (symbol_attr == nullptr || symbol_attr->str() != "model") { - MLLM_ERROR("LLM2QnnLoweringPass: CallGraphOp should point to a subgraph named 'model'"); - return ir::PASS_RET_FAILURE; + if (simple_qnn_graph_name_.empty()) { + MLLM_ERROR("LLM2QnnLoweringPass: CallGraphOp should point to a subgraph named 'model'"); + return ir::PASS_RET_FAILURE; + } + auto root_name = symbol_attr ? symbol_attr->str() : ""; + return runSimpleGraph(op, root_name, simple_qnn_graph_name_); } // Get the "model" subgraph @@ -191,6 +205,101 @@ uint8_t LLM2QnnLoweringPass::run(const ir::node_ptr_t& op) { return ir::PASS_RET_SUCCESS; } -ir::Pass::ptr_t createLLM2QnnLoweringPass() { return std::make_shared(); } +uint8_t LLM2QnnLoweringPass::runSimpleGraph(const ir::node_ptr_t& op, const std::string& root_graph_name, + const std::string& qnn_graph_name) { + MLLM_RT_ASSERT(op->isa_()); + if (root_graph_name.empty()) { + MLLM_ERROR("LLM2QnnLoweringPass: simple graph root name is empty"); + return ir::PASS_RET_FAILURE; + } + + auto root_ir = getCtx()->lookupSymbolTable(root_graph_name); + if (!root_ir || !root_ir->isa_()) { + MLLM_ERROR("LLM2QnnLoweringPass: Cannot find simple root graph {}", root_graph_name); + return ir::PASS_RET_FAILURE; + } + + auto root_graph = root_ir->cast_(); + auto aot_cfg = AOTCompileContext::getInstance().getConfig(); + auto aot_env = AOTCompileContext::getInstance().getEnv(); + + int split_graph = aot_cfg.value("split_graph", 1); + MLLM_RT_ASSERT_EQ(split_graph, 1); + aot_env->createContext("context.0", true); + auto aot_graph = aot_env->captureAOTGraph("context.0", qnn_graph_name); + + for (auto& input : root_graph->getTopRegion()->inputs()) { + auto tensor_input = input->cast_(); + if (tensor_input) { + tensor_input->setAttr("qnn_graph_inputs", getCtx()->create(true)); + aot_env->captureQnnAOTNodeTensor("context.0", qnn_graph_name, tensor_input); + } + } + for (auto& output : root_graph->getTopRegion()->outputs()) { + auto tensor_output = output->cast_(); + if (tensor_output) { + tensor_output->setAttr("qnn_graph_outputs", getCtx()->create(true)); + aot_env->captureQnnAOTNodeTensor("context.0", qnn_graph_name, tensor_output); + } + } + + std::function lower_subgraph_inline; + lower_subgraph_inline = [&](ir::graph::SubGraphOp::ptr_t subgraph) { + auto region = subgraph->getTopRegion(); + if (!region) { return; } + + auto subgraph_writer = ir::IRWriter(getCtx(), region); + subgraph_writer.walk( + [&](ir::IRWriter& this_tough_writer, const ir::Op::ptr_t& some_op) -> ir::IRWriter::WalkResult { + if (some_op->isa_()) { + auto call_op = some_op->cast_(); + auto called = getCtx()->lookupSymbolTable(call_op->getSymbolAttr()->str()); + if (!called || !called->isa_()) { + MLLM_ERROR_EXIT(ExitCode::kCoreError, "Cannot find called subgraph {}", call_op->getSymbolAttr()->str()); + } + lower_subgraph_inline(called->cast_()); + return ir::IRWriter::WalkResult::WALK_CONTINUE; + } + + if (!some_op->isa_()) { return ir::IRWriter::WalkResult::WALK_CONTINUE; } + + auto linalg_op = some_op->cast_(); + if (!linalg_op->getAttr("using_qnn")) { + MLLM_WARN("Found non-qnn op: {} in graph: {}", linalg_op->getAOp()->getName(), qnn_graph_name); + return ir::IRWriter::WalkResult::WALK_BREAK; + } + + linalg_op->setAttr("qnn_context_name", getCtx()->create("context.0")); + linalg_op->setAttr("qnn_graph_name", getCtx()->create(qnn_graph_name)); + + bool processed = false; + for (auto& [op_type, pass] : named_pattern_) { + if (pass->isMatch(linalg_op)) { + if (!pass->rewrite(this_tough_writer, linalg_op)) { + MLLM_ERROR_EXIT(ExitCode::kCoreError, "Failed when processing op {} with pass {}", + linalg_op->getAOp()->getName(), optype2Str(op_type)); + } + processed = true; + break; + } + } + + if (!processed) { + MLLM_ERROR_EXIT(ExitCode::kCoreError, "Failed processing op {} on all passes", linalg_op->getAOp()->getName()); + } + + return ir::IRWriter::WALK_CONTINUE; + }); + }; + + lower_subgraph_inline(root_graph); + + MLLM_RT_ASSERT(aot_graph->compile()); + return ir::PASS_RET_SUCCESS; +} + +ir::Pass::ptr_t createLLM2QnnLoweringPass(std::string simple_qnn_graph_name) { + return std::make_shared(std::move(simple_qnn_graph_name)); +} } // namespace mllm::qnn::aot diff --git a/mllm/backends/qnn/aot/passes/LLM2QnnLoweringPass.hpp b/mllm/backends/qnn/aot/passes/LLM2QnnLoweringPass.hpp index 55e6fe282..19e1065d1 100644 --- a/mllm/backends/qnn/aot/passes/LLM2QnnLoweringPass.hpp +++ b/mllm/backends/qnn/aot/passes/LLM2QnnLoweringPass.hpp @@ -3,6 +3,7 @@ #pragma once +#include #include #include "mllm/core/OpTypes.hpp" @@ -15,12 +16,14 @@ namespace mllm::qnn::aot { class LLM2QnnLoweringPass final : public ir::Pass { public: - LLM2QnnLoweringPass(); + explicit LLM2QnnLoweringPass(std::string simple_qnn_graph_name = ""); ~LLM2QnnLoweringPass() override = default; uint8_t run(const ir::node_ptr_t& op) override; + uint8_t runSimpleGraph(const ir::node_ptr_t& op, const std::string& root_graph_name, const std::string& qnn_graph_name); + private: template void registerPatterns() { @@ -29,8 +32,9 @@ class LLM2QnnLoweringPass final : public ir::Pass { std::unordered_map> named_pattern_; std::unordered_map subgraph_map_; + std::string simple_qnn_graph_name_; }; -ir::Pass::ptr_t createLLM2QnnLoweringPass(); +ir::Pass::ptr_t createLLM2QnnLoweringPass(std::string simple_qnn_graph_name = ""); } // namespace mllm::qnn::aot diff --git a/mllm/backends/qnn/aot/passes/LLMQuantRecipePass.cpp b/mllm/backends/qnn/aot/passes/LLMQuantRecipePass.cpp index 8320776b6..209517118 100644 --- a/mllm/backends/qnn/aot/passes/LLMQuantRecipePass.cpp +++ b/mllm/backends/qnn/aot/passes/LLMQuantRecipePass.cpp @@ -14,6 +14,8 @@ #include "mllm/compile/ir/linalg/Attribute.hpp" #include "mllm/backends/qnn/aot/passes/AOTCompileContext.hpp" #include "mllm/backends/qnn/aot/passes/LLMQuantRecipePass.hpp" +#include "mllm/core/aops/LayerNormOp.hpp" +#include "mllm/core/aops/LinearOp.hpp" namespace mllm::qnn::aot { @@ -499,6 +501,58 @@ bool LLMQuantRecipeRMSNormPattern::rewrite(ir::IRWriter& writer, const ir::op_pt return true; } +//===----------------------------------------------------------------------===// +// LayerNorm Pattern +//===----------------------------------------------------------------------===// +bool LLMQuantRecipeLayerNormPattern::isMatch(const mllm::ir::op_ptr_t& op) { + if (op->isa_()) { return true; } + return false; +} + +bool LLMQuantRecipeLayerNormPattern::rewrite(ir::IRWriter& writer, const ir::op_ptr_t& node) { + auto ret = noSharingSingleInAndSingleOutQuantAnnoAttr(writer.getContext(), node->cast_()); + if (!ret) return false; + + auto layer_norm_ir = node->cast_(); + auto layer_norm_aop = dynamic_cast(layer_norm_ir->getAOp()); + MLLM_RETURN_FALSE_IF_NOT(layer_norm_aop); + + auto annotation_attr = + node->getAttr("quant_recipe")->cast_(); + + auto add_raw_weight_recipe = [&](const std::string& suffix) -> bool { + auto reg_tensor_ir = writer.getContext()->lookupSymbolTable(layer_norm_ir->getAOp()->getName() + suffix); + MLLM_RETURN_FALSE_IF_NOT(reg_tensor_ir); + MLLM_RETURN_FALSE_IF_NOT(reg_tensor_ir->isa_()); + MLLM_RETURN_FALSE_IF_NOT(reg_tensor_ir->outputs().front()->isa_()); + + auto t = reg_tensor_ir->outputs().front()->cast_(); + auto weight_spec_attr = writer.create( + ir::linalg::QuantizationSpecRaw::create(t->tensor_.dtype())); + t->setAttr("quant_recipe", weight_spec_attr); + annotation_attr->annotation_.weights.insert({suffix.substr(1), weight_spec_attr->spec_}); + + return true; + }; + + if (layer_norm_aop->options().elementwise_affine && !add_raw_weight_recipe(".weight")) { return false; } + if (layer_norm_aop->options().bias && !add_raw_weight_recipe(".bias")) { return false; } + + return true; +} + +//===----------------------------------------------------------------------===// +// GELU Pattern +//===----------------------------------------------------------------------===// +bool LLMQuantRecipeGELUPattern::isMatch(const mllm::ir::op_ptr_t& op) { + if (op->isa_()) { return true; } + return false; +} + +bool LLMQuantRecipeGELUPattern::rewrite(ir::IRWriter& writer, const ir::op_ptr_t& node) { + return noSharingSingleInAndSingleOutQuantAnnoAttr(writer.getContext(), node->cast_()); +} + //===----------------------------------------------------------------------===// // SiLU Pattern //===----------------------------------------------------------------------===// @@ -609,9 +663,7 @@ bool LLMQuantRecipeElementwisePattern::rewrite(ir::IRWriter& writer, const ir::o i_0->getAttr("quant_recipe")->cast_())); } else { - MLLM_WARN("LLMQuantRecipeEqualPattern Only support constant Value as second inputs right now. Pls send us a issue or PR " - "if you want to compare two normal tensor(rather than static-tensor)."); - return false; + i_1->setAttr("quant_recipe", genSimpleQuantizationSpecAttr(writer.getContext(), i_1->cast_())); } } @@ -899,7 +951,26 @@ bool LLMQuantRecipeLinearPattern::rewrite(ir::IRWriter& writer, const ir::op_ptr auto input_spec = linear_ir->inputs().front()->getAttr("quant_recipe")->cast_(); annotation_attr->annotation_.inputs.emplace_back(input_spec->spec_); - if (use_config["method"] == "LPBQ") { + auto weight_name = linear_ir->getAOp()->getName() + ".weight"; + auto weight_reg_tensor_ir = writer.getContext()->lookupSymbolTable(weight_name); + MLLM_RETURN_FALSE_IF_NOT(weight_reg_tensor_ir); + MLLM_RETURN_FALSE_IF_NOT(weight_reg_tensor_ir->isa_()); + MLLM_RETURN_FALSE_IF_NOT(weight_reg_tensor_ir->outputs().front()->isa_()); + auto weight_tensor = weight_reg_tensor_ir->outputs().front()->cast_(); + + const bool allow_raw_float_linear = use_config.value("allow_raw_float_linear", false); + const bool float_weight = weight_tensor->tensor_.dtype() == kFloat32 || weight_tensor->tensor_.dtype() == kFloat16 + || weight_tensor->tensor_.dtype() == kBFloat16; + + if (allow_raw_float_linear && float_weight) { + auto weight_quant_spec = ir::linalg::QuantizationSpecRaw::create(weight_tensor->tensor_.dtype()); + weight_tensor->setAttr("quant_recipe", writer.create(weight_quant_spec)); + annotation_attr->annotation_.weights.insert({"weight", weight_quant_spec}); + + auto out_quant_spec = ir::linalg::QuantizationSpecRaw::create(linear_ir->outputs().front()->cast_()->tensor_.dtype()); + linear_ir->outputs().front()->setAttr("quant_recipe", writer.create(out_quant_spec)); + annotation_attr->annotation_.outputs.emplace_back(out_quant_spec); + } else if (use_config["method"] == "LPBQ") { // Unpack std::string precision = use_config["precision"]; bool sym = use_config["sym"]; @@ -922,18 +993,26 @@ bool LLMQuantRecipeLinearPattern::rewrite(ir::IRWriter& writer, const ir::op_ptr annotation_attr->annotation_.weights.insert({"weight", weight_quant_spec}); } - auto weight_name = linear_ir->getAOp()->getName() + ".weight"; - auto weight_reg_tensor_ir = writer.getContext()->lookupSymbolTable(weight_name); - MLLM_RETURN_FALSE_IF_NOT(weight_reg_tensor_ir); - MLLM_RETURN_FALSE_IF_NOT(weight_reg_tensor_ir->isa_()); - MLLM_RETURN_FALSE_IF_NOT(weight_reg_tensor_ir->outputs().front()->isa_()); - auto t = weight_reg_tensor_ir->outputs().front()->cast_(); - t->setAttr("quant_recipe", writer.create(weight_quant_spec)); + weight_tensor->setAttr("quant_recipe", writer.create(weight_quant_spec)); } else { std::string s = use_config["method"]; MLLM_WARN("Currently not support method: {}", s); } + auto real_linear_op = dynamic_cast(linear_ir->getAOp()); + if (real_linear_op && real_linear_op->options().bias) { + auto bias_name = linear_ir->getAOp()->getName() + ".bias"; + auto bias_reg_tensor_ir = writer.getContext()->lookupSymbolTable(bias_name); + if (bias_reg_tensor_ir) { + MLLM_RETURN_FALSE_IF_NOT(bias_reg_tensor_ir->isa_()); + MLLM_RETURN_FALSE_IF_NOT(bias_reg_tensor_ir->outputs().front()->isa_()); + auto bias_tensor = bias_reg_tensor_ir->outputs().front()->cast_(); + auto bias_quant_spec = ir::linalg::QuantizationSpecRaw::create(bias_tensor->tensor_.dtype()); + bias_tensor->setAttr("quant_recipe", writer.create(bias_quant_spec)); + annotation_attr->annotation_.weights.insert({"bias", bias_quant_spec}); + } + } + linear_ir->setAttr("quant_recipe", annotation_attr); } @@ -1116,6 +1195,8 @@ LLMQuantRecipePass::LLMQuantRecipePass() { addPattern(LLMQuantRecipeRoPEPattern::create(), "rope", 0); addPattern(LLMQuantRecipeCastTypePattern::create(), "cast_type", 0); addPattern(LLMQuantRecipeRMSNormPattern::create(), "rms_norm", 0); + addPattern(LLMQuantRecipeLayerNormPattern::create(), "layer_norm", 0); + addPattern(LLMQuantRecipeGELUPattern::create(), "gelu", 0); addPattern(LLMQuantRecipeSiLUPattern::create(), "silu", 0); addPattern(LLMQuantRecipeIndexPattern::create(), "index", 0); addPattern(LLMQuantRecipeElementwisePattern::create(), "elementwise", 0); diff --git a/mllm/backends/qnn/aot/passes/LLMQuantRecipePass.hpp b/mllm/backends/qnn/aot/passes/LLMQuantRecipePass.hpp index 9ce60b8c4..6545f0ba4 100644 --- a/mllm/backends/qnn/aot/passes/LLMQuantRecipePass.hpp +++ b/mllm/backends/qnn/aot/passes/LLMQuantRecipePass.hpp @@ -126,6 +126,34 @@ class LLMQuantRecipeRMSNormPattern : public ir::Pattern { } }; +//===----------------------------------------------------------------------===// +// LayerNorm Pattern +//===----------------------------------------------------------------------===// +class LLMQuantRecipeLayerNormPattern : public ir::Pattern { + public: + bool isMatch(const mllm::ir::op_ptr_t& op) override; + + bool rewrite(ir::IRWriter& writer, const ir::op_ptr_t& node) override; + + static inline std::shared_ptr create() { + return std::make_shared(); + } +}; + +//===----------------------------------------------------------------------===// +// GELU Pattern +//===----------------------------------------------------------------------===// +class LLMQuantRecipeGELUPattern : public ir::Pattern { + public: + bool isMatch(const mllm::ir::op_ptr_t& op) override; + + bool rewrite(ir::IRWriter& writer, const ir::op_ptr_t& node) override; + + static inline std::shared_ptr create() { + return std::make_shared(); + } +}; + //===----------------------------------------------------------------------===// // SiLU Pattern //===----------------------------------------------------------------------===// diff --git a/mllm/backends/qnn/aot/passes/PTQPass.cpp b/mllm/backends/qnn/aot/passes/PTQPass.cpp index 43e591fec..c97aa7bc6 100644 --- a/mllm/backends/qnn/aot/passes/PTQPass.cpp +++ b/mllm/backends/qnn/aot/passes/PTQPass.cpp @@ -33,31 +33,38 @@ void checkTypeLimits(Tensor in, int quant_min, int quant_max) { // NOLINT void solveLinearWeight(const ir::IRContext::ptr_t& ctx, const ParameterFile::ptr_t& pf, const ir::linalg::LinalgIROp::ptr_t& op) { auto mllm_op = op->getAOp(); - MLLM_INFO("PTQPass working on Op: {}'s weight", mllm_op->getName()); - auto weight_spec = - op->getAttr("quant_recipe")->cast_()->annotation_.weights.at("weight"); - - if (weight_spec->solved) return; - - switch (weight_spec->type) { - case ir::linalg::QuantizationSpecType::kLPBQ: { - auto this_spec = std::static_pointer_cast(weight_spec); - auto scale1 = pf->pull(mllm_op->getName() + ".scale1"); // using uint8 to store uint4 - auto scale2 = pf->pull(mllm_op->getName() + ".scale2"); - auto weight = pf->pull(mllm_op->getName() + ".weight"); + MLLM_INFO("PTQPass working on Op: {}'s linear weights", mllm_op->getName()); + auto annotation = + op->getAttr("quant_recipe")->cast_()->annotation_; - // FIXME weight maybe error, Check qnn eats int8 or uint8. Here weight using int8 to store int4. - checkTypeLimits(weight, 0, 15); // Int4 - checkTypeLimits(scale1, 0, 16); // UInt4 + for (const auto& [name, weight_spec] : annotation.weights) { + if (weight_spec->solved) continue; - this_spec->scale_level_0_int = scale1; - this_spec->scale_level_1_fp = scale2; - - weight_spec->solved = true; - break; - } - default: { - NYI("quant recipe type not support"); + switch (weight_spec->type) { + case ir::linalg::QuantizationSpecType::kRaw: { + weight_spec->solved = true; + break; + } + case ir::linalg::QuantizationSpecType::kLPBQ: { + if (name != "weight") { NYI("LPBQ quant recipe only supports Linear weight, got '{}'.", name); } + auto this_spec = std::static_pointer_cast(weight_spec); + auto scale1 = pf->pull(mllm_op->getName() + ".scale1"); // using uint8 to store uint4 + auto scale2 = pf->pull(mllm_op->getName() + ".scale2"); + auto weight = pf->pull(mllm_op->getName() + ".weight"); + + // FIXME weight maybe error, Check qnn eats int8 or uint8. Here weight using int8 to store int4. + checkTypeLimits(weight, 0, 15); // Int4 + checkTypeLimits(scale1, 0, 16); // UInt4 + + this_spec->scale_level_0_int = scale1; + this_spec->scale_level_1_fp = scale2; + + weight_spec->solved = true; + break; + } + default: { + NYI("quant recipe type not support"); + } } } } @@ -98,6 +105,28 @@ void solveRMSNormWeight(const ir::IRContext::ptr_t& ctx, const ParameterFile::pt } } +void solveLayerNormWeights(const ir::IRContext::ptr_t& ctx, const ParameterFile::ptr_t& pf, + const ir::linalg::LinalgIROp::ptr_t& op) { + auto mllm_op = op->getAOp(); + MLLM_INFO("PTQPass working on Op: {}'s layernorm weights", mllm_op->getName()); + auto annotation = + op->getAttr("quant_recipe")->cast_()->annotation_; + + for (const auto& [name, weight_spec] : annotation.weights) { + if (weight_spec->solved) continue; + + switch (weight_spec->type) { + case ir::linalg::QuantizationSpecType::kRaw: { + weight_spec->solved = true; + break; + } + default: { + NYI("LayerNorm weight '{}' only supports raw quant recipe for now.", name); + } + } + } +} + void solveEmbeddingWeight(const ir::IRContext::ptr_t& ctx, const ParameterFile::ptr_t& pf, const ir::linalg::LinalgIROp::ptr_t& op) { auto mllm_op = op->getAOp(); @@ -144,6 +173,9 @@ void recursiveSolveWeights(const std::shared_ptr& ir_ctx, const i solveLinearWeight(w.getContext(), pf, op->cast_()); } if (op->isa_()) { solveRMSNormWeight(w.getContext(), pf, op->cast_()); } + if (op->isa_()) { + solveLayerNormWeights(w.getContext(), pf, op->cast_()); + } if (op->isa_()) { solveEmbeddingWeight(w.getContext(), pf, op->cast_()); } if (op->isa_()) { auto ns = op->cast_()->getSymbolAttr()->str(); diff --git a/mllm/backends/qnn/aot/visitor/Conv2D.cpp b/mllm/backends/qnn/aot/visitor/Conv2D.cpp index 083a12b38..8ea1a801f 100644 --- a/mllm/backends/qnn/aot/visitor/Conv2D.cpp +++ b/mllm/backends/qnn/aot/visitor/Conv2D.cpp @@ -9,6 +9,7 @@ #include "mllm/backends/qnn/aot/visitor/Conv2D.hpp" #include "mllm/backends/qnn/aot/passes/AOTCompileContext.hpp" #include "mllm/core/aops/Conv2DOp.hpp" +#include "mllm/compile/ir/linalg/Attribute.hpp" namespace mllm::qnn::aot { @@ -61,6 +62,11 @@ bool QnnAOTConv2DPattern::rewrite(ir::IRWriter& writer, const ir::op_ptr_t& op) ->outputs() .front() ->cast_(); + if (!bias_val->getAttr("quant_recipe")) { + bias_val->setAttr("quant_recipe", + writer.create( + ir::linalg::QuantizationSpecRaw::create(bias_val->tensor_.dtype()))); + } qnn_op_node->emplaceInput(env->captureQnnAOTNodeTensor(qnn_context_name, qnn_graph_name, bias_val, true)); } diff --git a/mllm/backends/qnn/aot/visitor/GELU.cpp b/mllm/backends/qnn/aot/visitor/GELU.cpp new file mode 100644 index 000000000..6b36f9276 --- /dev/null +++ b/mllm/backends/qnn/aot/visitor/GELU.cpp @@ -0,0 +1,47 @@ +// Copyright (c) MLLM Team. +// Licensed under the MIT License. + +#include "mllm/backends/qnn/aot/visitor/GELU.hpp" + +#include "mllm/backends/qnn/aot/QnnWrappersAPI.hpp" +#include "mllm/backends/qnn/aot/passes/AOTCompileContext.hpp" +#include "mllm/compile/ir/builtin/Attribute.hpp" +#include "mllm/compile/ir/linalg/Op.hpp" +#include "mllm/utils/Common.hpp" + +namespace mllm::qnn::aot { + +bool QnnAOTGELUPattern::isMatch(const mllm::ir::op_ptr_t& op) { + return op->isa_() && (op->getAttr("using_qnn") != nullptr); +} + +bool QnnAOTGELUPattern::rewrite(ir::IRWriter& writer, const ir::op_ptr_t& op) { + auto env = AOTCompileContext::getInstance().getEnv(); + + MLLM_RETURN_FALSE_IF_NOT(op->getAttr("quant_recipe")); + auto gelu_op = op->cast_(); + if (!gelu_op) { + MLLM_ERROR("Failed to cast to linalg::GELUOp"); + return false; + } + + MLLM_RETURN_FALSE_IF_NOT(op->getAttr("qnn_graph_name")); + auto qnn_graph_name = op->getAttr("qnn_graph_name")->cast_()->data(); + MLLM_RETURN_FALSE_IF_NOT(op->getAttr("qnn_context_name")); + auto qnn_context_name = op->getAttr("qnn_context_name")->cast_()->data(); + + auto input = op->inputs().front()->cast_(); + auto output = op->outputs().front()->cast_(); + + auto qnn_op_node = QnnAOTNodeOperation::create("Gelu"); + qnn_op_node->setPackageName("qti.aisw"); + qnn_op_node->emplaceInput(env->captureQnnAOTNodeTensor(qnn_context_name, qnn_graph_name, input)) + ->emplaceOutput(env->captureQnnAOTNodeTensor(qnn_context_name, qnn_graph_name, output)) + ->setName(gelu_op->getAOp()->getName()); + + env->captureAOTNodeOp(qnn_context_name, qnn_graph_name, qnn_op_node); + + return true; +} + +} // namespace mllm::qnn::aot diff --git a/mllm/backends/qnn/aot/visitor/GELU.hpp b/mllm/backends/qnn/aot/visitor/GELU.hpp new file mode 100644 index 000000000..2949287fe --- /dev/null +++ b/mllm/backends/qnn/aot/visitor/GELU.hpp @@ -0,0 +1,23 @@ +// Copyright (c) MLLM Team. +// Licensed under the MIT License. + +#pragma once + +#include "mllm/core/OpTypes.hpp" +#include "mllm/compile/ir/Node.hpp" +#include "mllm/backends/qnn/aot/visitor/Base.hpp" + +namespace mllm::qnn::aot { + +class QnnAOTGELUPattern : public QnnAOTBasePattern { + public: + bool isMatch(const mllm::ir::op_ptr_t& op) override; + + bool rewrite(ir::IRWriter& writer, const ir::op_ptr_t& op) override; + + static inline std::pair> create() { + return {OpTypes::kGELU, std::make_shared()}; + } +}; + +} // namespace mllm::qnn::aot diff --git a/mllm/backends/qnn/aot/visitor/LayerNorm.cpp b/mllm/backends/qnn/aot/visitor/LayerNorm.cpp new file mode 100644 index 000000000..546ed54f4 --- /dev/null +++ b/mllm/backends/qnn/aot/visitor/LayerNorm.cpp @@ -0,0 +1,82 @@ +// Copyright (c) MLLM Team. +// Licensed under the MIT License. + +#include "mllm/backends/qnn/aot/visitor/LayerNorm.hpp" + +#include "mllm/backends/qnn/aot/QnnWrappersAPI.hpp" +#include "mllm/backends/qnn/aot/passes/AOTCompileContext.hpp" +#include "mllm/compile/ir/builtin/Attribute.hpp" +#include "mllm/compile/ir/linalg/Op.hpp" +#include "mllm/core/aops/LayerNormOp.hpp" +#include "mllm/utils/Common.hpp" + +namespace mllm::qnn::aot { + +bool QnnAOTLayerNormPattern::isMatch(const mllm::ir::op_ptr_t& op) { + return op->isa_() && (op->getAttr("using_qnn") != nullptr); +} + +bool QnnAOTLayerNormPattern::rewrite(ir::IRWriter& writer, const ir::op_ptr_t& op) { + auto env = AOTCompileContext::getInstance().getEnv(); + + MLLM_RETURN_FALSE_IF_NOT(op->getAttr("quant_recipe")); + auto layer_norm_op = op->cast_(); + if (!layer_norm_op) { + MLLM_ERROR("Failed to cast to linalg::LayerNormOp"); + return false; + } + + MLLM_RETURN_FALSE_IF_NOT(op->getAttr("qnn_graph_name")); + auto qnn_graph_name = op->getAttr("qnn_graph_name")->cast_()->data(); + MLLM_RETURN_FALSE_IF_NOT(op->getAttr("qnn_context_name")); + auto qnn_context_name = op->getAttr("qnn_context_name")->cast_()->data(); + + auto base_op = layer_norm_op->getAOp(); + auto real_layer_norm_op = dynamic_cast(base_op); + if (!real_layer_norm_op) { + MLLM_ERROR("Failed to cast BaseOp to mllm::aops::LayerNormOp"); + return false; + } + + auto input = op->inputs().front()->cast_(); + auto output = op->outputs().front()->cast_(); + + auto qnn_op_node = QnnAOTNodeOperation::create("LayerNorm"); + qnn_op_node->setPackageName("qti.aisw"); + qnn_op_node->emplaceInput(env->captureQnnAOTNodeTensor(qnn_context_name, qnn_graph_name, input)); + + if (real_layer_norm_op->options().elementwise_affine) { + auto weight = writer.getContext() + ->lookupSymbolTable(base_op->getName() + ".weight") + ->outputs() + .front() + ->cast_(); + qnn_op_node->emplaceInput(env->captureQnnAOTNodeTensor(qnn_context_name, qnn_graph_name, weight, true)); + } + + if (real_layer_norm_op->options().bias) { + auto bias = writer.getContext() + ->lookupSymbolTable(base_op->getName() + ".bias") + ->outputs() + .front() + ->cast_(); + qnn_op_node->emplaceInput(env->captureQnnAOTNodeTensor(qnn_context_name, qnn_graph_name, bias, true)); + } + + qnn_op_node->emplaceParamScalar(QNNParamScalarWrapper::create("epsilon", real_layer_norm_op->options().eps)); + + std::vector axes_dims = {1}; + auto axes_param = QNNParamTensorWrapper::create("axes", base_op->getName() + "_axes", QNN_DATATYPE_UINT_32, axes_dims); + auto* axes_data = reinterpret_cast(axes_param->alloc()); + axes_data[0] = input->tensor_.shape().size() - 1; + qnn_op_node->emplaceParamTensor(axes_param); + + qnn_op_node->emplaceOutput(env->captureQnnAOTNodeTensor(qnn_context_name, qnn_graph_name, output)) + ->setName(base_op->getName()); + + env->captureAOTNodeOp(qnn_context_name, qnn_graph_name, qnn_op_node); + + return true; +} + +} // namespace mllm::qnn::aot diff --git a/mllm/backends/qnn/aot/visitor/LayerNorm.hpp b/mllm/backends/qnn/aot/visitor/LayerNorm.hpp new file mode 100644 index 000000000..587df5567 --- /dev/null +++ b/mllm/backends/qnn/aot/visitor/LayerNorm.hpp @@ -0,0 +1,23 @@ +// Copyright (c) MLLM Team. +// Licensed under the MIT License. + +#pragma once + +#include "mllm/core/OpTypes.hpp" +#include "mllm/compile/ir/Node.hpp" +#include "mllm/backends/qnn/aot/visitor/Base.hpp" + +namespace mllm::qnn::aot { + +class QnnAOTLayerNormPattern : public QnnAOTBasePattern { + public: + bool isMatch(const mllm::ir::op_ptr_t& op) override; + + bool rewrite(ir::IRWriter& writer, const ir::op_ptr_t& op) override; + + static inline std::pair> create() { + return {OpTypes::kLayerNorm, std::make_shared()}; + } +}; + +} // namespace mllm::qnn::aot From 8d71c4dc3784ea42b20baa3282f608923edc7466 Mon Sep 17 00:00:00 2001 From: twlddd Date: Mon, 18 May 2026 15:00:57 +0800 Subject: [PATCH 3/7] Add Qwen2-VL full QNN AOT example --- .gitignore | 1 + examples/CMakeLists.txt | 1 + examples/qwen2vl/config_2B_qnn_lpbq.json | 33 + examples/qwen2vl/main.cpp | 1 + examples/qwen2vl/qnn_aot_cfg_2B.json | 51 + examples/qwen2vl/qnn_aot_cfg_2B_visual.json | 30 + examples/qwen2vl_qnn_aot/CMakeLists.txt | 26 + examples/qwen2vl_qnn_aot/README.md | 76 + examples/qwen2vl_qnn_aot/aot_run.cpp | 1767 +++++++++++++++++ examples/qwen2vl_qnn_aot/compile.cpp | 398 ++++ examples/qwen2vl_qnn_aot/compile_visual.cpp | 552 +++++ .../modeling_qwen2vl_qnn_aot.hpp | 558 ++++++ .../qwen2vl_qnn_aot/visual_aot_helpers.hpp | 359 ++++ examples/qwen2vl_qnn_aot/visual_aot_run.cpp | 590 ++++++ .../visual_padding_ab_diag.cpp | 484 +++++ .../qwen2vl_qnn_aot/visual_padding_diag.cpp | 296 +++ mllm/backends/cpu/ops/VisionRoPEOp.cpp | 52 +- mllm/models/ARGeneration.cpp | 40 +- .../qwen2vl/image_preprocessor_qwen2vl.hpp | 15 +- mllm/models/qwen2vl/modeling_qwen2vl.hpp | 6 +- .../qwen2vl/modeling_qwen2vl_traceable.hpp | 56 +- mllm/models/qwen2vl/tokenization_qwen2vl.hpp | 6 +- scripts/qwen2vl_qnn/common.sh | 43 + scripts/qwen2vl_qnn/push_eval_assets.sh | 44 + scripts/qwen2vl_qnn/qwen2vl_eval_cases.tsv | 9 + .../qwen2vl_eval_cases_5bucket.tsv | 7 + scripts/qwen2vl_qnn/run_qnn_eval_fixed_set.sh | 125 ++ scripts/qwen2vl_qnn/run_qnn_interactive.sh | 89 + 28 files changed, 5671 insertions(+), 44 deletions(-) create mode 100644 examples/qwen2vl/config_2B_qnn_lpbq.json create mode 100644 examples/qwen2vl/qnn_aot_cfg_2B.json create mode 100644 examples/qwen2vl/qnn_aot_cfg_2B_visual.json create mode 100644 examples/qwen2vl_qnn_aot/CMakeLists.txt create mode 100644 examples/qwen2vl_qnn_aot/README.md create mode 100644 examples/qwen2vl_qnn_aot/aot_run.cpp create mode 100644 examples/qwen2vl_qnn_aot/compile.cpp create mode 100644 examples/qwen2vl_qnn_aot/compile_visual.cpp create mode 100644 examples/qwen2vl_qnn_aot/modeling_qwen2vl_qnn_aot.hpp create mode 100644 examples/qwen2vl_qnn_aot/visual_aot_helpers.hpp create mode 100644 examples/qwen2vl_qnn_aot/visual_aot_run.cpp create mode 100644 examples/qwen2vl_qnn_aot/visual_padding_ab_diag.cpp create mode 100644 examples/qwen2vl_qnn_aot/visual_padding_diag.cpp create mode 100755 scripts/qwen2vl_qnn/common.sh create mode 100755 scripts/qwen2vl_qnn/push_eval_assets.sh create mode 100644 scripts/qwen2vl_qnn/qwen2vl_eval_cases.tsv create mode 100644 scripts/qwen2vl_qnn/qwen2vl_eval_cases_5bucket.tsv create mode 100755 scripts/qwen2vl_qnn/run_qnn_eval_fixed_set.sh create mode 100755 scripts/qwen2vl_qnn/run_qnn_interactive.sh diff --git a/.gitignore b/.gitignore index b441a62eb..e7104144c 100644 --- a/.gitignore +++ b/.gitignore @@ -38,5 +38,6 @@ autotuner.log # Downloaded models and build artifacts (root-only) /models/ +/logs/ # Keep source model adapters tracked !tools/mllm-llm-benchmark/models/ diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index c1b3a2d36..6a3fdb377 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -25,5 +25,6 @@ endif() if(MLLM_QUALCOMM_QNN_AOT_ON_X86_ENABLE OR MLLM_BUILD_QNN_BACKEND) add_subdirectory(qwen3_qnn_aot) add_subdirectory(qwen2_qnn_aot) + add_subdirectory(qwen2vl_qnn_aot) add_subdirectory(llama_qnn_aot) endif() diff --git a/examples/qwen2vl/config_2B_qnn_lpbq.json b/examples/qwen2vl/config_2B_qnn_lpbq.json new file mode 100644 index 000000000..02e8885fc --- /dev/null +++ b/examples/qwen2vl/config_2B_qnn_lpbq.json @@ -0,0 +1,33 @@ +{ + "architectures": [ + "Qwen2VLForConditionalGeneration" + ], + "visual_in_chans": 3, + "visual_embed_dim": 1280, + "visual_patch_size": 14, + "visual_temporal_patch_size": 2, + "visual_spatial_merge_size": 2, + "visual_mlp_ratio": 4, + "visual_num_heads": 16, + "visual_depth": 32, + "hidden_size": 1536, + "intermediate_size": 8960, + "num_attention_heads": 12, + "num_key_value_heads": 2, + "num_hidden_layers": 28, + "max_position_embeddings": 32786, + "rms_norm_eps": 1e-06, + "vocab_size": 151936, + "max_cache_length": 2048, + "mrope_section": [ + 16, + 24, + 24 + ], + "vision_token_id": 151654, + "eos_token_id": 151645, + "end_of_text_token_id": 151643, + "rope_theta": 1000000.0, + "tie_word_embeddings": true, + "linear_impl_type": "QNN_LPBQ_w4a16o16_G32" +} diff --git a/examples/qwen2vl/main.cpp b/examples/qwen2vl/main.cpp index 601f8ced8..71ce5ec50 100644 --- a/examples/qwen2vl/main.cpp +++ b/examples/qwen2vl/main.cpp @@ -65,6 +65,7 @@ MLLM_MAIN({ // Use for loop for (auto& step : qwen2vl.chat(inputs)) { std::wcout << qwen2vl_tokenizer.detokenize(step.cur_token_id) << std::flush; } + qwen2vl.perfSummary(); // OR // Steam it! diff --git a/examples/qwen2vl/qnn_aot_cfg_2B.json b/examples/qwen2vl/qnn_aot_cfg_2B.json new file mode 100644 index 000000000..3ddf11c9e --- /dev/null +++ b/examples/qwen2vl/qnn_aot_cfg_2B.json @@ -0,0 +1,51 @@ +{ + "target_machine": { + "htp_arch": "V75", + "htp_chipset": "SM8650", + "htp_try_best_performance": "HtpBurst", + "htp_security_pd_session": "HtpSignedPd", + "htp_vtcm_capability_in_mb": 8 + }, + "graph_on_qnn": [ + "model" + ], + "op_on_qnn": [ + "lm_head" + ], + "split_graph": 1, + "quant_recipe": { + "llm_recipe": true, + "layers": 28, + "builtin_llm_pass": { + "model": "qwen2", + "lm_head": { + "fallback": { + "method": "LPBQ", + "sym": true, + "precision": "w4a16", + "block_size": 32 + } + }, + "linear": { + "fallback": { + "method": "LPBQ", + "sym": true, + "precision": "w4a16", + "block_size": 32 + } + }, + "kv_cache": { + "key": { + "method": "per-tensor", + "sym": true, + "precision": "w8a8" + }, + "value": { + "method": "per-tensor", + "sym": true, + "precision": "w8a8" + } + } + } + } +} diff --git a/examples/qwen2vl/qnn_aot_cfg_2B_visual.json b/examples/qwen2vl/qnn_aot_cfg_2B_visual.json new file mode 100644 index 000000000..8ef78e133 --- /dev/null +++ b/examples/qwen2vl/qnn_aot_cfg_2B_visual.json @@ -0,0 +1,30 @@ +{ + "target_machine": { + "htp_arch": "V75", + "htp_chipset": "SM8650", + "htp_try_best_performance": "HtpBurst", + "htp_security_pd_session": "HtpUnsignedPd", + "htp_vtcm_capability_in_mb": 8 + }, + "graph_on_qnn": [ + "visual" + ], + "op_on_qnn": [], + "split_graph": 1, + "quant_recipe": { + "llm_recipe": true, + "layers": 32, + "builtin_llm_pass": { + "model": "qwen2vl_visual", + "linear": { + "fallback": { + "method": "LPBQ", + "sym": true, + "precision": "w4a16", + "block_size": 32, + "allow_raw_float_linear": true + } + } + } + } +} diff --git a/examples/qwen2vl_qnn_aot/CMakeLists.txt b/examples/qwen2vl_qnn_aot/CMakeLists.txt new file mode 100644 index 000000000..e3cbb6b1a --- /dev/null +++ b/examples/qwen2vl_qnn_aot/CMakeLists.txt @@ -0,0 +1,26 @@ +# AOT targets run on x86 and generate QNN context binaries for Android. +if(MLLM_QUALCOMM_QNN_AOT_ON_X86_ENABLE) + add_executable(mllm-qwen2vl-aot-c compile.cpp) + target_link_libraries(mllm-qwen2vl-aot-c PRIVATE MllmRT MllmCPUBackend MllmQNNBackend) + target_include_directories(mllm-qwen2vl-aot-c PRIVATE ${MLLM_INCLUDE_DIR}) + + add_executable(mllm-qwen2vl-visual-aot-diag compile_visual.cpp) + target_link_libraries(mllm-qwen2vl-visual-aot-diag PRIVATE MllmRT MllmCPUBackend MllmQNNBackend) + target_include_directories(mllm-qwen2vl-visual-aot-diag PRIVATE ${MLLM_INCLUDE_DIR}) + + add_executable(mllm-qwen2vl-visual-padding-diag visual_padding_diag.cpp) + target_link_libraries(mllm-qwen2vl-visual-padding-diag PRIVATE MllmRT MllmCPUBackend MllmQNNBackend) + target_include_directories(mllm-qwen2vl-visual-padding-diag PRIVATE ${MLLM_INCLUDE_DIR}) +endif() + +add_executable(mllm-qwen2vl-aot-runner aot_run.cpp) +target_link_libraries(mllm-qwen2vl-aot-runner PRIVATE MllmRT MllmCPUBackend MllmQNNBackend) +target_include_directories(mllm-qwen2vl-aot-runner PRIVATE ${MLLM_INCLUDE_DIR}) + +add_executable(mllm-qwen2vl-visual-aot-runner visual_aot_run.cpp) +target_link_libraries(mllm-qwen2vl-visual-aot-runner PRIVATE MllmRT MllmCPUBackend MllmQNNBackend) +target_include_directories(mllm-qwen2vl-visual-aot-runner PRIVATE ${MLLM_INCLUDE_DIR}) + +add_executable(mllm-qwen2vl-visual-padding-ab-diag visual_padding_ab_diag.cpp) +target_link_libraries(mllm-qwen2vl-visual-padding-ab-diag PRIVATE MllmRT MllmCPUBackend MllmQNNBackend) +target_include_directories(mllm-qwen2vl-visual-padding-ab-diag PRIVATE ${MLLM_INCLUDE_DIR}) diff --git a/examples/qwen2vl_qnn_aot/README.md b/examples/qwen2vl_qnn_aot/README.md new file mode 100644 index 000000000..fe4946344 --- /dev/null +++ b/examples/qwen2vl_qnn_aot/README.md @@ -0,0 +1,76 @@ +# Qwen2-VL QNN AOT Example + +This example demonstrates full QNN AOT inference for Qwen2-VL style models: + +- visual encoder graphs +- LLM prefill graph +- LLM decode graph +- interactive image-text inference on Android + +Model weights and compiled QNN context binaries are not stored in this repository. + +## Build + +Build the x86 AOT compiler: + +```bash +cmake --build build-qnn-aot --target mllm-qwen2vl-aot-c -j2 +``` + +Build the Android runner: + +```bash +cmake --build build-android-arm64-v8a-qnn --target mllm-qwen2vl-aot-runner -j2 +``` + +## Compile QNN Context + +Set `QNN_SDK_ROOT` or pass `--qnn_env_path` explicitly. The compiler uses the +QNN x86 backend to generate a context binary for Android. + +```bash +./build-qnn-aot/bin/mllm-qwen2vl-aot-c \ + --model_path path/to/qwen2vl-qnn-lpbq.mllm \ + --config examples/qwen2vl/config_2B_qnn_lpbq.json \ + --aot_config examples/qwen2vl/qnn_aot_cfg_2B.json \ + --output_context_name qwen2vl-full-qnn.bin \ + --context_len 1024 \ + --prefill_len 32 \ + --include_visual_bundle \ + --visual_model path/to/qwen2vl-visual.mllm \ + --visual_config examples/qwen2vl/config_2B_qnn_lpbq.json \ + --visual_aot_config examples/qwen2vl/qnn_aot_cfg_2B_visual.json \ + --visual_bucket_grids 12x16,16x24,24x24,24x32,18x52,52x18,26x36,36x26 +``` + +The default input embedding quantization parameters are widened for multimodal +embeddings. Pass both `--input_embedding_scale -1` and +`--input_embedding_zero_point -1` to use the model embedding quantization +parameters instead. + +## Android Interactive Runner + +The helper script pushes the runner binary and launches an interactive shell +session. Set paths through environment variables as needed: + +```bash +REMOTE_QNN_DIR=/data/local/tmp/mllm-qwen2vl-qnn \ +REMOTE_CPU_DIR=/data/local/tmp/mllm-qwen2vl \ +CONTEXT=models/qwen2vl-full-qnn.bin \ +QNN_PARAMS=models/qwen2vl-qnn-lpbq.mllm \ +scripts/qwen2vl_qnn/run_qnn_interactive.sh +``` + +At the prompt, enter an image path on device and then the text prompt. + +## Visual Buckets + +The runner chooses the smallest visual bucket that covers the preprocessed image +grid. Padded visual tokens are masked in visual attention, and only valid visual +embeddings are passed to the LLM side. + +The default bucket list is: + +```text +12x16,16x24,24x24,24x32,18x52,52x18,26x36,36x26 +``` diff --git a/examples/qwen2vl_qnn_aot/aot_run.cpp b/examples/qwen2vl_qnn_aot/aot_run.cpp new file mode 100644 index 000000000..f9f267488 --- /dev/null +++ b/examples/qwen2vl_qnn_aot/aot_run.cpp @@ -0,0 +1,1767 @@ +// Copyright (c) MLLM Team. +// Licensed under the MIT License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using mllm::Argparse; +using mllm::Tensor; +using mllm::models::qwen2vl::Qwen2VLConfig; +using mllm::models::qwen2vl::Qwen2VLTokenizer; +using mllm::qnn::aot::KVCacheManager; +using mllm::qnn::aot::QnnAOTConfig; +using mllm::qnn::aot::QnnAOTModule; + +namespace { + +constexpr float kDefaultInputEmbeddingScale = 0.002563515f; +constexpr int32_t kDefaultInputEmbeddingZeroPoint = 15604; + +struct QuantParams { + float scale = 1.f; + int32_t zero_point = 0; +}; + +struct LogitEntry { + int32_t token_id = 0; + uint16_t raw = 0; + float value = 0.f; +}; + +struct RuntimeIO { + int32_t ar_len = 1; + std::unique_ptr module; + std::vector inputs; + std::vector outputs; + bool dump_block_outputs = false; + bool dump_layer0_outputs = false; +}; + +using QnnAOTModuleCache = std::unordered_map>; + +struct PromptFeatures { + Tensor sequence; + Tensor position_ids; + Tensor llm_embedding_sin; + Tensor llm_embedding_cos; + Tensor visual_embeddings; + int32_t vision_token_start = -1; +}; + +struct VisualBucketGrid { + int32_t grid_h = 0; + int32_t grid_w = 0; + + [[nodiscard]] int32_t patchTokens() const { return grid_h * grid_w; } +}; + +struct VisualResizeOverride { + bool enabled = false; + int32_t native_grid_h = 0; + int32_t native_grid_w = 0; + int32_t resize_grid_h = 0; + int32_t resize_grid_w = 0; + VisualBucketGrid bucket; +}; + +std::string trimInteractiveLine(std::string line) { + while (!line.empty() && (line.back() == '\n' || line.back() == '\r' || std::isspace(static_cast(line.back())))) { + line.pop_back(); + } + size_t start = 0; + while (start < line.size() && std::isspace(static_cast(line[start]))) { ++start; } + if (start > 0) { line.erase(0, start); } + return line; +} + +float readScalarFloat(const mllm::ParameterFile::ptr_t& params, const std::string& name) { + return params->pull(name).ptr()[0]; +} + +int32_t readScalarInt32(const mllm::ParameterFile::ptr_t& params, const std::string& name) { + return params->pull(name).ptr()[0]; +} + +QuantParams readQuantParams(const mllm::ParameterFile::ptr_t& params, + const std::string& scale_name, + const std::string& zp_name) { + return {.scale = readScalarFloat(params, scale_name), .zero_point = readScalarInt32(params, zp_name)}; +} + +uint16_t quantizeUInt16(float value, const QuantParams& qp) { + auto q = static_cast(std::llround(value / qp.scale) + qp.zero_point); + q = std::max(0, std::min(65535, q)); + return static_cast(q); +} + +bool shouldDumpQwen2VLAOTStats() { + const char* flag = std::getenv("MLLM_QWEN2VL_AOT_DUMP_STATS"); + return flag != nullptr && std::string(flag) != "0"; +} + +std::vector parseVisualBucketGrids(const std::string& text) { + std::vector buckets; + if (text.empty()) { return buckets; } + + std::stringstream ss(text); + std::string item; + while (std::getline(ss, item, ',')) { + item.erase(std::remove_if(item.begin(), item.end(), [](unsigned char c) { return std::isspace(c); }), item.end()); + if (item.empty()) { continue; } + + auto sep = item.find('x'); + if (sep == std::string::npos) { sep = item.find('X'); } + if (sep == std::string::npos) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "Invalid visual bucket grid '{}', expected HxW.", item); + } + + const int32_t grid_h = std::stoi(item.substr(0, sep)); + const int32_t grid_w = std::stoi(item.substr(sep + 1)); + if (grid_h <= 0 || grid_w <= 0 || grid_h % 2 != 0 || grid_w % 2 != 0) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, + "Invalid visual bucket grid '{}': grid_h/grid_w must be positive even numbers.", + item); + } + buckets.push_back({grid_h, grid_w}); + } + return buckets; +} + +std::string visualGraphSuffixForPatchTokens(int32_t patch_tokens) { return "_s" + std::to_string(patch_tokens); } + +int32_t findCoveringVisualBucketIndex(int32_t grid_h, int32_t grid_w, const std::vector& buckets) { + int32_t best_idx = -1; + int64_t best_area = 0; + int64_t best_padding = 0; + for (int32_t i = 0; i < static_cast(buckets.size()); ++i) { + const auto& bucket = buckets[i]; + if (bucket.grid_h < grid_h || bucket.grid_w < grid_w) { continue; } + const int64_t area = static_cast(bucket.grid_h) * bucket.grid_w; + const int64_t padding = area - static_cast(grid_h) * grid_w; + if (best_idx < 0 || area < best_area || (area == best_area && padding < best_padding)) { + best_idx = i; + best_area = area; + best_padding = padding; + } + } + return best_idx; +} + +int32_t floorEvenGrid(double value) { + int32_t grid = static_cast(std::floor(value + 1e-6)); + grid -= grid % 2; + return std::max(4, grid); +} + +int64_t visualPatchIndex(int32_t t, int32_t h, int32_t w, int32_t grid_h, int32_t grid_w, int32_t merge_size) { + const int32_t h_blocks = grid_h / merge_size; + const int32_t w_blocks = grid_w / merge_size; + return (((static_cast(t) * h_blocks + h / merge_size) * w_blocks + w / merge_size) * merge_size + h % merge_size) + * merge_size + + w % merge_size; +} + +int64_t visualMergedIndex(int32_t t, int32_t h, int32_t w, int32_t grid_h, int32_t grid_w, int32_t merge_size) { + const int32_t h_blocks = grid_h / merge_size; + const int32_t w_blocks = grid_w / merge_size; + return (static_cast(t) * h_blocks + h) * w_blocks + w; +} + +Tensor makeGridThwTensor(int32_t grid_t, int32_t grid_h, int32_t grid_w) { + auto grid = Tensor::empty({1, 3}, mllm::kInt32, mllm::kCPU).alloc(); + grid.ptr()[0] = grid_t; + grid.ptr()[1] = grid_h; + grid.ptr()[2] = grid_w; + return grid; +} + +VisualBucketGrid selectVisualBucket(Tensor grid_thw, const std::vector& buckets) { + const auto* grid = grid_thw.ptr(); + const int32_t grid_h = grid[1]; + const int32_t grid_w = grid[2]; + + const int32_t best_idx = findCoveringVisualBucketIndex(grid_h, grid_w, buckets); + + if (best_idx < 0) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, + "No visual bucket can cover grid {}x{}. Add a larger --visual_bucket_grids entry.", + grid_h, + grid_w); + } + return buckets[best_idx]; +} + +VisualResizeOverride chooseVisualResizeOverrideForOversizedImage(const std::string& image_path, + const std::vector& buckets) { + VisualResizeOverride override; + if (buckets.empty()) { return override; } + + auto image = mllm::Image::open(image_path); + mllm::models::qwen2vl::Qwen2VLImagePreprocessor image_preprocessor(56 * 56, 28 * 28 * 256); + auto [native_h_px, native_w_px] = image_preprocessor.smartResize(image.h(), image.w(), 28, 56 * 56, 28 * 28 * 256); + const int32_t native_grid_h = native_h_px / 14; + const int32_t native_grid_w = native_w_px / 14; + if (findCoveringVisualBucketIndex(native_grid_h, native_grid_w, buckets) >= 0) { return override; } + + int32_t max_patch_tokens = 0; + for (const auto& bucket : buckets) { max_patch_tokens = std::max(max_patch_tokens, bucket.patchTokens()); } + + int32_t best_idx = -1; + double best_scale = -1.0; + double best_aspect_error = 0.0; + const double native_aspect = static_cast(native_grid_h) / native_grid_w; + for (int32_t i = 0; i < static_cast(buckets.size()); ++i) { + const auto& bucket = buckets[i]; + if (bucket.patchTokens() != max_patch_tokens) { continue; } + const double scale = std::min(static_cast(bucket.grid_h) / native_grid_h, + static_cast(bucket.grid_w) / native_grid_w); + const double bucket_aspect = static_cast(bucket.grid_h) / bucket.grid_w; + const double aspect_error = std::abs(std::log(bucket_aspect / native_aspect)); + if (best_idx < 0 || scale > best_scale || (scale == best_scale && aspect_error < best_aspect_error)) { + best_idx = i; + best_scale = scale; + best_aspect_error = aspect_error; + } + } + + if (best_idx < 0 || best_scale <= 0.0) { return override; } + + const auto& bucket = buckets[best_idx]; + int32_t resize_grid_h = floorEvenGrid(native_grid_h * best_scale); + int32_t resize_grid_w = floorEvenGrid(native_grid_w * best_scale); + while (resize_grid_h > bucket.grid_h) { resize_grid_h -= 2; } + while (resize_grid_w > bucket.grid_w) { resize_grid_w -= 2; } + resize_grid_h = std::max(4, resize_grid_h); + resize_grid_w = std::max(4, resize_grid_w); + + override.enabled = true; + override.native_grid_h = native_grid_h; + override.native_grid_w = native_grid_w; + override.resize_grid_h = resize_grid_h; + override.resize_grid_w = resize_grid_w; + override.bucket = bucket; + return override; +} + +Tensor padVisualPatchesToBucket(Tensor img, Tensor grid_thw, const VisualBucketGrid& bucket, const Qwen2VLConfig& cfg) { + const auto* grid = grid_thw.ptr(); + const int32_t grid_t = grid[0]; + const int32_t grid_h = grid[1]; + const int32_t grid_w = grid[2]; + if (grid_h == bucket.grid_h && grid_w == bucket.grid_w) { return img; } + + const int32_t patch_dim = img.shape()[1]; + auto padded = Tensor::empty({grid_t * bucket.grid_h * bucket.grid_w, patch_dim}, img.dtype(), mllm::kCPU).alloc(); + std::fill(padded.ptr(), padded.ptr() + padded.numel(), 0.0f); + + for (int32_t t = 0; t < grid_t; ++t) { + for (int32_t h = 0; h < grid_h; ++h) { + for (int32_t w = 0; w < grid_w; ++w) { + const auto src_idx = visualPatchIndex(t, h, w, grid_h, grid_w, cfg.visual_spatial_merge_size); + const auto dst_idx = visualPatchIndex(t, h, w, bucket.grid_h, bucket.grid_w, cfg.visual_spatial_merge_size); + std::memcpy(padded.offsettedPtr({static_cast(dst_idx), 0}), + img.offsettedPtr({static_cast(src_idx), 0}), + patch_dim * sizeof(float)); + } + } + } + return padded; +} + +Tensor makeVisualAttentionMaskForBucket(Tensor original_grid_thw, const VisualBucketGrid& bucket, const Qwen2VLConfig& cfg) { + const auto* grid = original_grid_thw.ptr(); + const int32_t grid_t = grid[0]; + const int32_t grid_h = grid[1]; + const int32_t grid_w = grid[2]; + const int32_t bucket_tokens = grid_t * bucket.grid_h * bucket.grid_w; + + auto mask = Tensor::empty({1, 1, 1, bucket_tokens}, mllm::kFloat32, mllm::kCPU).alloc(); + std::fill(mask.ptr(), mask.ptr() + mask.numel(), -10000.0f); + + for (int32_t t = 0; t < grid_t; ++t) { + for (int32_t h = 0; h < grid_h; ++h) { + for (int32_t w = 0; w < grid_w; ++w) { + const auto idx = visualPatchIndex(t, h, w, bucket.grid_h, bucket.grid_w, cfg.visual_spatial_merge_size); + *mask.offsettedPtr({0, 0, 0, static_cast(idx)}) = 0.0f; + } + } + } + return mask; +} + +Tensor makeAllValidVisualAttentionMask(int32_t patch_tokens) { + auto mask = Tensor::empty({1, 1, 1, patch_tokens}, mllm::kFloat32, mllm::kCPU).alloc(); + std::fill(mask.ptr(), mask.ptr() + mask.numel(), 0.0f); + return mask; +} + +Tensor cropVisualEmbeddingsFromBucket(Tensor bucket_embeddings, + Tensor original_grid_thw, + Tensor bucket_grid_thw, + const Qwen2VLConfig& cfg) { + const auto* original_grid = original_grid_thw.ptr(); + const auto* bucket_grid = bucket_grid_thw.ptr(); + const int32_t grid_t = original_grid[0]; + const int32_t original_h = original_grid[1]; + const int32_t original_w = original_grid[2]; + const int32_t bucket_h = bucket_grid[1]; + const int32_t bucket_w = bucket_grid[2]; + if (original_h == bucket_h && original_w == bucket_w) { return bucket_embeddings; } + + const int32_t merge = cfg.visual_spatial_merge_size; + const int32_t hidden_size = bucket_embeddings.shape()[1]; + const int32_t original_merged_h = original_h / merge; + const int32_t original_merged_w = original_w / merge; + auto cropped = Tensor::empty({grid_t * original_merged_h * original_merged_w, hidden_size}, bucket_embeddings.dtype(), mllm::kCPU).alloc(); + + for (int32_t t = 0; t < grid_t; ++t) { + for (int32_t h = 0; h < original_merged_h; ++h) { + for (int32_t w = 0; w < original_merged_w; ++w) { + const auto src_idx = visualMergedIndex(t, h, w, bucket_h, bucket_w, merge); + const auto dst_idx = visualMergedIndex(t, h, w, original_h, original_w, merge); + std::memcpy(cropped.offsettedPtr({static_cast(dst_idx), 0}), + bucket_embeddings.offsettedPtr({static_cast(src_idx), 0}), + hidden_size * sizeof(float)); + } + } + } + return cropped; +} + +Tensor makeQnnTensor(const std::vector& shape, mllm::DataTypes dtype, const std::string& name) { + auto tensor = Tensor::empty(shape, dtype, mllm::kQNN).setName(name).alloc(); + return tensor; +} + +Tensor copyToQnn(const Tensor& cpu_tensor, const std::string& name) { + auto qnn_tensor = makeQnnTensor(cpu_tensor.shape(), cpu_tensor.dtype(), name); + std::memcpy(qnn_tensor.ptr(), cpu_tensor.ptr(), cpu_tensor.bytes()); + return qnn_tensor; +} + +QnnAOTModule& getOrCreateCachedModule(QnnAOTModuleCache& cache, const std::string& graph_name) { + auto it = cache.find(graph_name); + if (it != cache.end()) { return *it->second; } + + auto module = std::make_unique(graph_name); + module->to(mllm::kQNN); + auto [inserted, _] = cache.emplace(graph_name, std::move(module)); + return *inserted->second; +} + +std::vector visualGraphNamesForLayout(const std::string& bundle_layout, const std::string& graph_suffix) { + if (bundle_layout == "single") { return {"visual_full" + graph_suffix}; } + if (bundle_layout == "6x8") { + return {"visual_patch_embed" + graph_suffix, + "visual_blocks_0_8" + graph_suffix, + "visual_blocks_8_16" + graph_suffix, + "visual_blocks_16_24" + graph_suffix, + "visual_blocks_24_32" + graph_suffix, + "visual_merger" + graph_suffix}; + } + if (bundle_layout == "early2") { + return {"visual_patch_embed" + graph_suffix, + "visual_blocks_0_2" + graph_suffix, + "visual_blocks_2_4" + graph_suffix, + "visual_blocks_4_6" + graph_suffix, + "visual_blocks_6_8" + graph_suffix, + "visual_blocks_8_16" + graph_suffix, + "visual_blocks_16_24" + graph_suffix, + "visual_blocks_24_32" + graph_suffix, + "visual_merger" + graph_suffix}; + } + if (bundle_layout == "tail4") { + return {"visual_patch_embed" + graph_suffix, + "visual_blocks_0_8" + graph_suffix, + "visual_blocks_8_16" + graph_suffix, + "visual_blocks_16_20" + graph_suffix, + "visual_blocks_20_24" + graph_suffix, + "visual_blocks_24_28" + graph_suffix, + "visual_blocks_28_32" + graph_suffix, + "visual_merger" + graph_suffix}; + } + + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "--visual_bundle_layout must be single, 6x8, early2 or tail4."); +} + +void initializeVisualQnnModules(const std::string& bundle_layout, + const std::vector& buckets, + QnnAOTModuleCache& cache) { + std::vector suffixes; + if (buckets.empty()) { + suffixes.push_back(""); + } else { + std::unordered_set seen_patch_tokens; + for (const auto& bucket : buckets) { + if (seen_patch_tokens.insert(bucket.patchTokens()).second) { + suffixes.push_back(visualGraphSuffixForPatchTokens(bucket.patchTokens())); + } + } + } + + const auto before = cache.size(); + for (const auto& suffix : suffixes) { + for (const auto& graph_name : visualGraphNamesForLayout(bundle_layout, suffix)) { + (void)getOrCreateCachedModule(cache, graph_name); + } + } + fmt::print("[Qwen2VL AOT] initialized {} visual QNN module wrappers (layout={}, bucket_suffixes={})\n", + cache.size() - before, + bundle_layout, + suffixes.size()); +} + +Tensor runQnnGraph1(const std::string& graph_name, + Tensor hidden, + Tensor visual_embedding_sin, + Tensor visual_embedding_cos, + Tensor visual_attention_mask, + Tensor output, + QnnAOTModuleCache* module_cache = nullptr) { + std::unique_ptr local_module; + QnnAOTModule* module = nullptr; + if (module_cache != nullptr) { + module = &getOrCreateCachedModule(*module_cache, graph_name); + } else { + local_module = std::make_unique(graph_name); + local_module->to(mllm::kQNN); + module = local_module.get(); + } + + module->setOutputTensors({output}); + auto inputs = visual_attention_mask.isNil() ? std::vector{hidden, visual_embedding_sin, visual_embedding_cos} + : std::vector{hidden, + visual_embedding_sin, + visual_embedding_cos, + visual_attention_mask}; + auto outputs = (*module)(inputs); + MLLM_RT_ASSERT_EQ(outputs.size(), 1); + return outputs[0]; +} + +Tensor runVisualQnnBundle(Tensor img, + Tensor visual_embedding_sin, + Tensor visual_embedding_cos, + Tensor visual_attention_mask, + const Qwen2VLConfig& cfg, + const std::string& bundle_layout, + const std::string& graph_suffix, + QnnAOTModuleCache* module_cache = nullptr) { + const int32_t visual_patch_tokens = img.shape()[0]; + const int32_t merged_tokens = visual_patch_tokens / (cfg.visual_spatial_merge_size * cfg.visual_spatial_merge_size); + + auto img_qnn = copyToQnn(img, "visual_img"); + auto sin_qnn = copyToQnn(visual_embedding_sin, "visual_embedding_sin"); + auto cos_qnn = copyToQnn(visual_embedding_cos, "visual_embedding_cos"); + auto mask_qnn = copyToQnn(visual_attention_mask, "visual_attention_mask"); + + auto hidden_a = makeQnnTensor({visual_patch_tokens, cfg.visual_embed_dim}, mllm::kFloat32, "visual_hidden_a"); + auto hidden_b = makeQnnTensor({visual_patch_tokens, cfg.visual_embed_dim}, mllm::kFloat32, "visual_hidden_b"); + auto visual_embeddings = makeQnnTensor({merged_tokens, cfg.hidden_size}, mllm::kFloat32, "visual_embeddings"); + + Tensor hidden = Tensor::nil(); + if (bundle_layout == "single") { + auto output = + runQnnGraph1("visual_full" + graph_suffix, img_qnn, sin_qnn, cos_qnn, mask_qnn, visual_embeddings, module_cache); + return output.to(mllm::kCPU).clone(); + } else if (bundle_layout == "6x8") { + hidden = runQnnGraph1("visual_patch_embed" + graph_suffix, img_qnn, sin_qnn, cos_qnn, Tensor::nil(), hidden_a, module_cache); + hidden = runQnnGraph1("visual_blocks_0_8" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b, module_cache); + hidden = runQnnGraph1("visual_blocks_8_16" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a, module_cache); + hidden = runQnnGraph1("visual_blocks_16_24" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b, module_cache); + hidden = runQnnGraph1("visual_blocks_24_32" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a, module_cache); + } else if (bundle_layout == "early2") { + hidden = runQnnGraph1("visual_patch_embed" + graph_suffix, img_qnn, sin_qnn, cos_qnn, Tensor::nil(), hidden_a, module_cache); + hidden = runQnnGraph1("visual_blocks_0_2" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b, module_cache); + hidden = runQnnGraph1("visual_blocks_2_4" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a, module_cache); + hidden = runQnnGraph1("visual_blocks_4_6" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b, module_cache); + hidden = runQnnGraph1("visual_blocks_6_8" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a, module_cache); + hidden = runQnnGraph1("visual_blocks_8_16" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b, module_cache); + hidden = runQnnGraph1("visual_blocks_16_24" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a, module_cache); + hidden = runQnnGraph1("visual_blocks_24_32" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b, module_cache); + } else if (bundle_layout == "tail4") { + hidden = runQnnGraph1("visual_patch_embed" + graph_suffix, img_qnn, sin_qnn, cos_qnn, Tensor::nil(), hidden_a, module_cache); + hidden = runQnnGraph1("visual_blocks_0_8" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b, module_cache); + hidden = runQnnGraph1("visual_blocks_8_16" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a, module_cache); + hidden = runQnnGraph1("visual_blocks_16_20" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b, module_cache); + hidden = runQnnGraph1("visual_blocks_20_24" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a, module_cache); + hidden = runQnnGraph1("visual_blocks_24_28" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b, module_cache); + hidden = runQnnGraph1("visual_blocks_28_32" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a, module_cache); + } else { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "--visual_bundle_layout must be single, 6x8, early2 or tail4 for full runner."); + } + + auto output = runQnnGraph1("visual_merger" + graph_suffix, + hidden, + sin_qnn, + cos_qnn, + Tensor::nil(), + visual_embeddings, + module_cache); + return output.to(mllm::kCPU).clone(); +} + +int32_t findVisionTokenStart(const Tensor& sequence, int32_t vision_token_id) { + auto* ids = sequence.ptr(); + for (int32_t i = 0; i < sequence.shape()[1]; ++i) { + if (ids[i] == vision_token_id) { return i; } + } + return -1; +} + +Tensor makePositionIdsPrefill(const Tensor& sequence, const Tensor& image_grid_thw, const Qwen2VLConfig& cfg) { + MLLM_RT_ASSERT_EQ(sequence.shape().size(), 2); + MLLM_RT_ASSERT_EQ(image_grid_thw.shape().size(), 2); + MLLM_RT_ASSERT_EQ(sequence.shape()[0], 1); + + const int32_t seq_len = sequence.shape()[1]; + Tensor position_ids = Tensor::empty({3, 1, seq_len}, mllm::kInt64, mllm::kCPU).alloc(); + + const int32_t vision_pad_start = findVisionTokenStart(sequence, cfg.vision_token_id); + MLLM_RT_ASSERT(vision_pad_start >= 0); + + const int* grid = image_grid_thw.ptr(); + const int32_t img_t = grid[0]; + const int32_t img_h = grid[1]; + const int32_t img_w = grid[2]; + const int32_t inputs_t = img_t; + const int32_t inputs_h = img_h / cfg.visual_spatial_merge_size; + const int32_t inputs_w = img_w / cfg.visual_spatial_merge_size; + const int32_t vision_tokens = inputs_t * inputs_h * inputs_w; + + int64_t current_max_position_id = 0; + for (int32_t d = 0; d < 3; ++d) { + auto* ptr = position_ids.offsettedPtr({d, 0, 0}); + for (int64_t k = 0; k < vision_pad_start; ++k) { ptr[k] = k; } + } + current_max_position_id = vision_pad_start - 1; + + int32_t cnt = 0; + const int64_t vision_start_id = current_max_position_id + 1; + for (int32_t ti = 0; ti < inputs_t; ++ti) { + for (int32_t hi = 0; hi < inputs_h; ++hi) { + for (int32_t wi = 0; wi < inputs_w; ++wi) { + *position_ids.offsettedPtr({0, 0, vision_pad_start + cnt}) = vision_start_id + ti; + *position_ids.offsettedPtr({1, 0, vision_pad_start + cnt}) = vision_start_id + hi; + *position_ids.offsettedPtr({2, 0, vision_pad_start + cnt}) = vision_start_id + wi; + ++cnt; + } + } + } + + const int64_t dim_0_tail = *position_ids.offsettedPtr({0, 0, vision_pad_start + vision_tokens - 1}); + const int64_t dim_1_tail = *position_ids.offsettedPtr({1, 0, vision_pad_start + vision_tokens - 1}); + const int64_t dim_2_tail = *position_ids.offsettedPtr({2, 0, vision_pad_start + vision_tokens - 1}); + current_max_position_id = std::max({dim_0_tail, dim_1_tail, dim_2_tail}); + + const int64_t trailing_text_start = vision_pad_start + vision_tokens; + const int64_t trailing_text_count = seq_len - trailing_text_start; + if (trailing_text_count > 0) { + const int64_t start_id = current_max_position_id + 1; + for (int32_t d = 0; d < 3; ++d) { + auto* ptr = position_ids.offsettedPtr({d, 0, 0}); + for (int64_t k = 0; k < trailing_text_count; ++k) { ptr[trailing_text_start + k] = start_id + k; } + } + } + + return position_ids; +} + +Tensor makePositionIdsDecode(Tensor prev_position_ids) { + const int32_t last_idx = prev_position_ids.shape()[2] - 1; + const auto last_pos = *prev_position_ids.offsettedPtr({0, 0, last_idx}); + auto ret = Tensor::empty({3, 1, 1}, mllm::kInt64, mllm::kCPU).alloc(); + for (int32_t d = 0; d < 3; ++d) { *ret.offsettedPtr({d, 0, 0}) = last_pos + 1; } + return ret; +} + +void fillQuantizedFloatChunk(Tensor src, + int32_t src_start, + int32_t valid_len, + Tensor& dst, + const QuantParams& qp, + int32_t feature_dim) { + auto* out = dst.ptr(); + std::fill(out, out + dst.numel(), quantizeUInt16(0.f, qp)); + + for (int32_t s = 0; s < valid_len; ++s) { + const float* in_row = src.offsettedPtr({src_start + s, 0}); + uint16_t* out_row = out + s * feature_dim; + for (int32_t d = 0; d < feature_dim; ++d) { out_row[d] = quantizeUInt16(in_row[d], qp); } + } +} + +void fillEmbeddingChunk(Tensor sequence, + Tensor embedding_weight, + Tensor visual_embeddings, + int32_t vision_start, + int32_t chunk_start, + int32_t valid_len, + Tensor& dst, + const QuantParams& embedding_weight_qp, + const QuantParams& input_embedding_qp, + int32_t hidden_size) { + auto* out = dst.ptr(); + const auto* ids = sequence.ptr(); + const auto* weight = embedding_weight.ptr(); + std::fill(out, out + dst.numel(), quantizeUInt16(0.f, input_embedding_qp)); + + const bool same_qp = embedding_weight_qp.scale == input_embedding_qp.scale + && embedding_weight_qp.zero_point == input_embedding_qp.zero_point; + + for (int32_t s = 0; s < valid_len; ++s) { + const int32_t seq_idx = chunk_start + s; + uint16_t* out_row = out + s * hidden_size; + const int64_t token_id = ids[seq_idx]; + const int32_t visual_idx = seq_idx - vision_start; + if (!visual_embeddings.isNil() && visual_idx >= 0 && visual_idx < visual_embeddings.shape()[0]) { + const float* visual_row = visual_embeddings.offsettedPtr({visual_idx, 0}); + for (int32_t d = 0; d < hidden_size; ++d) { out_row[d] = quantizeUInt16(visual_row[d], input_embedding_qp); } + } else { + const uint16_t* weight_row = weight + token_id * hidden_size; + if (same_qp) { + std::memcpy(out_row, weight_row, hidden_size * sizeof(uint16_t)); + } else { + for (int32_t d = 0; d < hidden_size; ++d) { + const float value = (static_cast(weight_row[d]) - embedding_weight_qp.zero_point) * embedding_weight_qp.scale; + out_row[d] = quantizeUInt16(value, input_embedding_qp); + } + } + } + } +} + +void initQwen2VLIO(RuntimeIO& io, + const QnnAOTConfig& cfg, + KVCacheManager& kv_manager_u8, + KVCacheManager* key_cache_manager_u16, + bool key_cache_uint16, + int32_t hidden_size, + int32_t intermediate_size) { + io.module = std::make_unique("model.0.s" + std::to_string(io.ar_len)); + io.module->to(mllm::kQNN); + + io.inputs.clear(); + io.outputs.clear(); + io.inputs.reserve(4 + 2 * cfg.num_layers); + const int32_t attention_heads = hidden_size / cfg.head_dim; + io.outputs.reserve(1 + 2 * cfg.num_layers + (io.dump_block_outputs ? cfg.num_layers : 0) + + (io.dump_layer0_outputs ? 16 + 4 * attention_heads : 0)); + + io.inputs.push_back(makeQnnTensor({1, io.ar_len, hidden_size}, mllm::kUInt16PerTensorAsy, "input_embeddings")); + io.inputs.push_back(makeQnnTensor({1, io.ar_len, cfg.head_dim}, mllm::kUInt16PerTensorAsy, "llm_embedding_sin")); + io.inputs.push_back(makeQnnTensor({1, io.ar_len, cfg.head_dim}, mllm::kUInt16PerTensorAsy, "llm_embedding_cos")); + io.inputs.push_back(makeQnnTensor({1, 1, io.ar_len, cfg.context_len}, mllm::kUInt16PerTensorAsy, "causal_mask")); + + const auto& v_caches = kv_manager_u8.getVCache(); + if (key_cache_uint16) { + MLLM_RT_ASSERT(key_cache_manager_u16 != nullptr); + const auto& k_caches = key_cache_manager_u16->getKCache(); + for (int32_t l = 0; l < cfg.num_layers; ++l) { + auto k_tensor = + Tensor::empty({1, cfg.num_heads, cfg.head_dim, cfg.context_len - io.ar_len}, mllm::kUInt16PerTensorSym, + mllm::kQNN); + k_tensor.impl()->storage()->ptr_ = k_caches[l].buffer; + k_tensor.impl()->storage()->mem_type_ = mllm::kManual; + k_tensor.setName("past_key_" + std::to_string(l)); + io.inputs.push_back(k_tensor); + } + } else { + const auto& k_caches = kv_manager_u8.getKCache(); + for (int32_t l = 0; l < cfg.num_layers; ++l) { + auto k_tensor = + Tensor::empty({1, cfg.num_heads, cfg.head_dim, cfg.context_len - io.ar_len}, mllm::kUInt8, mllm::kQNN); + k_tensor.impl()->storage()->ptr_ = k_caches[l].buffer; + k_tensor.impl()->storage()->mem_type_ = mllm::kManual; + k_tensor.setName("past_key_" + std::to_string(l)); + io.inputs.push_back(k_tensor); + } + } + for (int32_t l = 0; l < cfg.num_layers; ++l) { + auto v_tensor = Tensor::empty({1, cfg.num_heads, cfg.context_len - io.ar_len, cfg.head_dim}, mllm::kUInt8, mllm::kQNN); + v_tensor.impl()->storage()->ptr_ = v_caches[l].buffer; + v_tensor.impl()->storage()->mem_type_ = mllm::kManual; + v_tensor.setName("past_value_" + std::to_string(l)); + io.inputs.push_back(v_tensor); + } + + io.outputs.push_back(makeQnnTensor({1, 1, io.ar_len, cfg.vocab_size}, mllm::kUInt16PerTensorAsy, "logits")); + if (key_cache_uint16) { + MLLM_RT_ASSERT(key_cache_manager_u16 != nullptr); + const auto& k_caches = key_cache_manager_u16->getKCache(); + for (int32_t l = 0; l < cfg.num_layers; ++l) { + auto k_tensor = + Tensor::empty({1, cfg.num_heads, cfg.head_dim, io.ar_len}, mllm::kUInt16PerTensorSym, mllm::kQNN); + k_tensor.impl()->storage()->ptr_ = k_caches[l].output_buffer; + k_tensor.impl()->storage()->mem_type_ = mllm::kManual; + k_tensor.setName("present_key_" + std::to_string(l)); + io.outputs.push_back(k_tensor); + } + } else { + const auto& k_caches = kv_manager_u8.getKCache(); + for (int32_t l = 0; l < cfg.num_layers; ++l) { + auto k_tensor = Tensor::empty({1, cfg.num_heads, cfg.head_dim, io.ar_len}, mllm::kUInt8, mllm::kQNN); + k_tensor.impl()->storage()->ptr_ = k_caches[l].output_buffer; + k_tensor.impl()->storage()->mem_type_ = mllm::kManual; + k_tensor.setName("present_key_" + std::to_string(l)); + io.outputs.push_back(k_tensor); + } + } + for (int32_t l = 0; l < cfg.num_layers; ++l) { + auto v_tensor = Tensor::empty({1, cfg.num_heads, io.ar_len, cfg.head_dim}, mllm::kUInt8, mllm::kQNN); + v_tensor.impl()->storage()->ptr_ = v_caches[l].output_buffer; + v_tensor.impl()->storage()->mem_type_ = mllm::kManual; + v_tensor.setName("present_value_" + std::to_string(l)); + io.outputs.push_back(v_tensor); + } + if (io.dump_block_outputs) { + for (int32_t l = 0; l < cfg.num_layers; ++l) { + io.outputs.push_back(makeQnnTensor({1, io.ar_len, hidden_size}, mllm::kUInt16PerTensorAsy, + "block_out_" + std::to_string(l))); + } + } + if (io.dump_layer0_outputs) { + const int32_t kv_dim = cfg.num_heads * cfg.head_dim; + const auto push_attn_head_outputs = [&](const std::string& base_name) { + for (int32_t h = 0; h < attention_heads; ++h) { + io.outputs.push_back(makeQnnTensor({1, 1, io.ar_len, cfg.context_len}, mllm::kUInt16PerTensorAsy, + base_name + "_h" + std::to_string(h))); + } + }; + io.outputs.push_back(makeQnnTensor({1, io.ar_len, hidden_size}, mllm::kUInt16PerTensorAsy, + "layer0_input_layernorm_out")); + io.outputs.push_back(makeQnnTensor({1, io.ar_len, hidden_size}, mllm::kUInt16PerTensorAsy, + "layer0_q_proj_out_flat")); + io.outputs.push_back(makeQnnTensor({1, io.ar_len, kv_dim}, mllm::kUInt16PerTensorAsy, "layer0_k_proj_out_flat")); + io.outputs.push_back(makeQnnTensor({1, io.ar_len, kv_dim}, mllm::kUInt16PerTensorAsy, "layer0_v_proj_out_flat")); + io.outputs.push_back(makeQnnTensor({1, io.ar_len, kv_dim}, mllm::kUInt8, "layer0_v_cache_out_flat")); + io.outputs.push_back(makeQnnTensor({1, io.ar_len, hidden_size}, mllm::kUInt16PerTensorAsy, + "layer0_q_rope_out_flat")); + io.outputs.push_back(makeQnnTensor({1, io.ar_len, kv_dim}, mllm::kUInt16PerTensorAsy, "layer0_k_rope_out_flat")); + push_attn_head_outputs("layer0_qk_matmul_out_flat"); + push_attn_head_outputs("layer0_qk_scaled_out_flat"); + push_attn_head_outputs("layer0_qk_masked_out_flat"); + push_attn_head_outputs("layer0_softmax_out_flat"); + io.outputs.push_back(makeQnnTensor({1, io.ar_len, hidden_size}, mllm::kUInt16PerTensorAsy, + "layer0_attn_value_out_flat")); + io.outputs.push_back(makeQnnTensor({1, io.ar_len, hidden_size}, mllm::kUInt16PerTensorAsy, "layer0_o_proj_out")); + io.outputs.push_back(makeQnnTensor({1, io.ar_len, hidden_size}, mllm::kUInt16PerTensorAsy, + "layer0_post_attention_layernorm_out")); + io.outputs.push_back(makeQnnTensor({1, io.ar_len, intermediate_size}, mllm::kUInt16PerTensorAsy, + "layer0_mlp_up_proj_out")); + io.outputs.push_back(makeQnnTensor({1, io.ar_len, intermediate_size}, mllm::kUInt16PerTensorAsy, + "layer0_mlp_gate_proj_out")); + io.outputs.push_back(makeQnnTensor({1, io.ar_len, intermediate_size}, mllm::kUInt16PerTensorAsy, + "layer0_mlp_gate_act_out")); + io.outputs.push_back(makeQnnTensor({1, io.ar_len, intermediate_size}, mllm::kUInt16PerTensorAsy, + "layer0_mlp_down_proj_input")); + io.outputs.push_back(makeQnnTensor({1, io.ar_len, hidden_size}, mllm::kUInt16PerTensorAsy, + "layer0_mlp_down_proj_out")); + io.outputs.push_back(makeQnnTensor({1, io.ar_len, hidden_size}, mllm::kUInt16PerTensorAsy, "layer0_block_out")); + } +} + +int64_t sampleGreedyFromLogitsAt(Tensor& logits, int32_t token_idx) { + auto logits_cpu = logits.to(mllm::kCPU); + auto* data = logits_cpu.ptr(); + const int32_t vocab = logits_cpu.shape().back(); + const auto& shape = logits_cpu.shape(); + if (shape.size() == 4) { + MLLM_RT_ASSERT(token_idx >= 0 && token_idx < shape[2]); + data += token_idx * vocab; + } else if (shape.size() == 3) { + MLLM_RT_ASSERT(token_idx >= 0 && token_idx < shape[1]); + data += token_idx * vocab; + } else { + MLLM_RT_ASSERT_EQ(logits_cpu.numel(), vocab); + } + auto max_it = std::max_element(data, data + vocab); + return static_cast(std::distance(data, max_it)); +} + +std::vector topKFromLogitsAt(Tensor& logits, int32_t token_idx, int32_t k, const QuantParams& qp) { + auto logits_cpu = logits.to(mllm::kCPU); + auto* data = logits_cpu.ptr(); + const int32_t vocab = logits_cpu.shape().back(); + const auto& shape = logits_cpu.shape(); + if (shape.size() == 4) { + MLLM_RT_ASSERT(token_idx >= 0 && token_idx < shape[2]); + data += token_idx * vocab; + } else if (shape.size() == 3) { + MLLM_RT_ASSERT(token_idx >= 0 && token_idx < shape[1]); + data += token_idx * vocab; + } else { + MLLM_RT_ASSERT_EQ(logits_cpu.numel(), vocab); + } + + k = std::max(0, std::min(k, vocab)); + std::vector indices(vocab); + std::iota(indices.begin(), indices.end(), 0); + std::partial_sort(indices.begin(), indices.begin() + k, indices.end(), [&](int32_t lhs, int32_t rhs) { + if (data[lhs] == data[rhs]) { return lhs < rhs; } + return data[lhs] > data[rhs]; + }); + + std::vector ret; + ret.reserve(k); + for (int32_t i = 0; i < k; ++i) { + const int32_t token_id = indices[i]; + const uint16_t raw = data[token_id]; + ret.push_back({.token_id = token_id, .raw = raw, .value = (static_cast(raw) - qp.zero_point) * qp.scale}); + } + return ret; +} + +class Qwen2VLAOTRunner { + public: + Qwen2VLAOTRunner(const Qwen2VLConfig& qnn_cfg, + const Qwen2VLConfig& visual_cfg, + const mllm::ParameterFile::ptr_t& qnn_params, + int32_t ar_len, + int32_t context_len, + float input_embedding_scale, + int32_t input_embedding_zero_point, + bool dump_block_outputs, + bool dump_layer0_outputs, + bool dump_visual_tokens, + bool key_cache_uint16, + bool visual_qnn, + std::string visual_bundle_layout, + std::vector visual_bucket_grids, + int32_t dump_logits_topk, + std::vector dump_token_indices, + std::string dump_path) + : qnn_cfg_(qnn_cfg), + visual_cfg_(visual_cfg), + qnn_params_(qnn_params), + dump_block_outputs_(dump_block_outputs), + dump_layer0_outputs_(dump_layer0_outputs), + dump_visual_tokens_(dump_visual_tokens), + key_cache_uint16_(key_cache_uint16), + visual_qnn_(visual_qnn), + visual_bundle_layout_(std::move(visual_bundle_layout)), + visual_bucket_grids_(std::move(visual_bucket_grids)), + dump_logits_topk_(dump_logits_topk), + dump_token_indices_(std::move(dump_token_indices)), + dump_path_(std::move(dump_path)) { + config_.num_layers = qnn_cfg_.num_hidden_layers; + config_.num_heads = qnn_cfg_.num_key_value_heads; + config_.head_dim = qnn_cfg_.hidden_size / qnn_cfg_.num_attention_heads; + config_.vocab_size = qnn_cfg_.vocab_size; + config_.context_len = context_len; + config_.ar_len = ar_len; + config_.kv_dtype = mllm::kUInt8; + config_.max_cache_len = context_len - 1; + config_.max_ar_len = std::max(1, ar_len); + config_.sliding_window = context_len; + + embedding_weight_qp_ = readQuantParams(qnn_params_, "model.embed_tokens.scale", "model.embed_tokens.zero_point"); + input_embedding_qp_ = embedding_weight_qp_; + if (input_embedding_scale > 0.0f && input_embedding_zero_point >= 0) { + input_embedding_qp_ = {.scale = input_embedding_scale, .zero_point = input_embedding_zero_point}; + fmt::print("[Qwen2VL AOT] override input_embeddings quant params: scale={:.9f}, zero_point={}\n", + input_embedding_qp_.scale, input_embedding_qp_.zero_point); + } + sin_qp_ = readQuantParams(qnn_params_, "model.sin_embedding_input_qdq.fake_quant.scale", + "model.sin_embedding_input_qdq.fake_quant.zero_point"); + cos_qp_ = readQuantParams(qnn_params_, "model.cos_embedding_input_qdq.fake_quant.scale", + "model.cos_embedding_input_qdq.fake_quant.zero_point"); + logits_qp_ = + readQuantParams(qnn_params_, "lm_head_output_qdq.fake_quant.scale", "lm_head_output_qdq.fake_quant.zero_point"); + block_out_qps_.reserve(qnn_cfg_.num_hidden_layers); + for (int32_t l = 0; l < qnn_cfg_.num_hidden_layers; ++l) { + if (l + 1 < qnn_cfg_.num_hidden_layers) { + block_out_qps_.push_back(readQuantParams( + qnn_params_, "model.layers." + std::to_string(l + 1) + ".input_layernorm_input_qdq.fake_quant.scale", + "model.layers." + std::to_string(l + 1) + ".input_layernorm_input_qdq.fake_quant.zero_point")); + } else { + block_out_qps_.push_back(readQuantParams(qnn_params_, "model.norm_input_qdq.fake_quant.scale", + "model.norm_input_qdq.fake_quant.zero_point")); + } + } + initLayer0QuantParams(); + embedding_weight_ = qnn_params_->pull("model.embed_tokens.weight"); + } + + bool load() { + auto backend = mllm::Context::instance().getBackend(mllm::kQNN); + if (!backend) { + MLLM_ERROR("QNN Backend not found"); + return false; + } + + kv_manager_u8_ = std::make_unique>(config_); + kv_manager_u8_->initCache(backend->allocator().get(), config_.ar_len); + if (key_cache_uint16_) { + key_cache_manager_u16_ = std::make_unique>(config_); + key_cache_manager_u16_->initCache(backend->allocator().get(), config_.ar_len); + fmt::print("[Qwen2VL AOT] experimental key cache dtype: uint16, value cache dtype: uint8\n"); + } + prefill_io_.ar_len = config_.ar_len; + decode_io_.ar_len = 1; + prefill_io_.dump_block_outputs = dump_block_outputs_; + decode_io_.dump_block_outputs = dump_block_outputs_; + prefill_io_.dump_layer0_outputs = dump_layer0_outputs_; + decode_io_.dump_layer0_outputs = dump_layer0_outputs_; + initQwen2VLIO(prefill_io_, config_, *kv_manager_u8_, key_cache_manager_u16_.get(), key_cache_uint16_, + qnn_cfg_.hidden_size, qnn_cfg_.intermediate_size); + initQwen2VLIO(decode_io_, config_, *kv_manager_u8_, key_cache_manager_u16_.get(), key_cache_uint16_, + qnn_cfg_.hidden_size, qnn_cfg_.intermediate_size); + if (visual_qnn_) { initializeVisualQnnModules(visual_bundle_layout_, visual_bucket_grids_, visual_modules_); } + return true; + } + + void resetStateForRequest() { + current_pos_ = 0; + prev_position_ids_ = Tensor::nil(); + prefill_tokens_ = 0; + generated_tokens_ = 0; + prompt_token_ids_.clear(); + vision_token_start_ = -1; + vision_token_count_ = 0; + } + + PromptFeatures preparePrompt(const mllm::models::ARGenerationOutputPast& inputs, + mllm::models::qwen2vl::Qwen2VisionTransformerPretrainedModel* visual) { + PromptFeatures features; + features.sequence = inputs.at("sequence"); + auto img = inputs.at("img"); + auto grid_thw = inputs.at("grid_thw"); + if (dump_block_outputs_ || dump_layer0_outputs_ || dump_visual_tokens_) { + prompt_token_ids_ = tensorToTokenIds(features.sequence); + dumpPromptSummary(); + } + + fmt::print("ViT Processing: ...\n"); + mllm::print("Image shape is:", img.shape()); + auto visual_img = img; + auto visual_grid_thw = grid_thw; + auto visual_attention_mask = makeAllValidVisualAttentionMask(img.shape()[0]); + std::string visual_graph_suffix; + if (visual_qnn_ && !visual_bucket_grids_.empty()) { + const auto* original_grid = grid_thw.ptr(); + const auto bucket = selectVisualBucket(grid_thw, visual_bucket_grids_); + visual_img = padVisualPatchesToBucket(img, grid_thw, bucket, visual_cfg_); + visual_grid_thw = makeGridThwTensor(original_grid[0], bucket.grid_h, bucket.grid_w); + visual_attention_mask = makeVisualAttentionMaskForBucket(grid_thw, bucket, visual_cfg_); + visual_graph_suffix = visualGraphSuffixForPatchTokens(bucket.patchTokens()); + fmt::print("[Qwen2VL AOT] visual bucket: original_grid={}x{} patches={} -> bucket_grid={}x{} patches={} graph_suffix={}\n", + original_grid[1], + original_grid[2], + img.shape()[0], + bucket.grid_h, + bucket.grid_w, + visual_img.shape()[0], + visual_graph_suffix); + } + auto inv_freq = mllm::models::qwen2vl::makeVisualRoPEInvFreq(visual_cfg_.visual_embed_dim / visual_cfg_.visual_num_heads, + 10000.0); + auto visual_pos_ids = mllm::models::qwen2vl::makeVisualRotaryPosEmbIds(visual_grid_thw, visual_cfg_.visual_spatial_merge_size); + auto rotary_pos_emb_full = mllm::models::qwen2vl::makeVisualRotaryPosEmbFull(inv_freq, visual_img.shape()[0]); + auto pos_emb = mllm::models::qwen2vl::makeVisualRotaryPosEmb(rotary_pos_emb_full, visual_pos_ids, visual_grid_thw); + auto [visual_embedding_sin, visual_embedding_cos] = mllm::models::qwen2vl::makeVisualRotarySinCos(pos_emb); + + vit_start_ = std::chrono::high_resolution_clock::now(); + if (visual_qnn_) { + const int32_t half_dim = visual_cfg_.visual_embed_dim / visual_cfg_.visual_num_heads / 2; + auto visual_sin_4d = visual_embedding_sin.view({1, -1, 1, half_dim}, false); + auto visual_cos_4d = visual_embedding_cos.view({1, -1, 1, half_dim}, false); + auto bucket_embeddings = + runVisualQnnBundle(visual_img, + visual_sin_4d, + visual_cos_4d, + visual_attention_mask, + visual_cfg_, + visual_bundle_layout_, + visual_graph_suffix, + &visual_modules_); + features.visual_embeddings = cropVisualEmbeddingsFromBucket(bucket_embeddings, grid_thw, visual_grid_thw, visual_cfg_); + } else { + MLLM_RT_ASSERT(visual != nullptr); + features.visual_embeddings = (*visual)(img, visual_embedding_sin, visual_embedding_cos)[0]; + } + vit_end_ = std::chrono::high_resolution_clock::now(); + fmt::print("ViT Processing: done, backend={}, time cost: {:.3f} seconds\n", + visual_qnn_ ? "QNN AOT" : "CPU/KAI", + std::chrono::duration(vit_end_ - vit_start_).count()); + if (shouldDumpQwen2VLAOTStats()) { + const auto* data = features.visual_embeddings.ptr(); + const auto numel = features.visual_embeddings.numel(); + auto [min_it, max_it] = std::minmax_element(data, data + numel); + const float qmin = (0 - input_embedding_qp_.zero_point) * input_embedding_qp_.scale; + const float qmax = (65535 - input_embedding_qp_.zero_point) * input_embedding_qp_.scale; + int64_t clipped = 0; + for (int64_t i = 0; i < numel; ++i) { + if (data[i] < qmin || data[i] > qmax) { ++clipped; } + } + fmt::print("[Qwen2VL AOT] visual_embeddings shape=[{}, {}], range=[{:.6f}, {:.6f}], embedding_qrange=[{:.6f}, " + "{:.6f}], clipped={}/{} ({:.2f}%)\n", + features.visual_embeddings.shape()[0], features.visual_embeddings.shape()[1], *min_it, *max_it, qmin, qmax, + clipped, numel, numel > 0 ? 100.0 * static_cast(clipped) / static_cast(numel) : 0.0); + } + + features.vision_token_start = findVisionTokenStart(features.sequence, qnn_cfg_.vision_token_id); + MLLM_RT_ASSERT(features.vision_token_start >= 0); + vision_token_start_ = features.vision_token_start; + vision_token_count_ = features.visual_embeddings.shape()[0]; + + features.position_ids = makePositionIdsPrefill(features.sequence, grid_thw, qnn_cfg_); + auto inv = mllm::models::qwen2vl::makeMultimodalRoPEInvFreq( + qnn_cfg_.hidden_size / qnn_cfg_.num_attention_heads, qnn_cfg_.rope_theta); + auto [sin, cos] = mllm::models::qwen2vl::makeMultimodalPositionEmbedding( + features.position_ids, inv, qnn_cfg_.max_position_embeddings, qnn_cfg_.hidden_size / qnn_cfg_.num_attention_heads, + qnn_cfg_.mrope_section); + features.llm_embedding_sin = sin; + features.llm_embedding_cos = cos; + return features; + } + + int64_t prefill(const PromptFeatures& features) { + const int64_t num_tokens = features.sequence.shape()[1]; + const auto dump_tokens = selectDumpTokens(num_tokens); + if ((dump_block_outputs_ || dump_layer0_outputs_ || dump_visual_tokens_ || dump_logits_topk_ > 0) && !dump_path_.empty()) { + writeDumpHeader(num_tokens, dump_tokens); + } + int64_t processed_tokens = 0; + int64_t current_pos = 0; + + rearrangeCache(config_.ar_len); + + std::vector attention_map(config_.ar_len); + std::iota(attention_map.begin(), attention_map.end(), -1); + kv_manager_u8_->initAttentionMask(prefill_io_.inputs[3].ptr(), attention_map, config_.ar_len, 0, + config_.sliding_window); + prefill_io_.module->setOutputTensors(prefill_io_.outputs); + + int64_t next_token = 0; + while (processed_tokens < num_tokens) { + const int32_t chunk = static_cast(std::min(config_.ar_len, num_tokens - processed_tokens)); + fillEmbeddingChunk(features.sequence, embedding_weight_, features.visual_embeddings, features.vision_token_start, + processed_tokens, chunk, prefill_io_.inputs[0], embedding_weight_qp_, input_embedding_qp_, + qnn_cfg_.hidden_size); + fillQuantizedFloatChunk(features.llm_embedding_sin, processed_tokens, chunk, prefill_io_.inputs[1], sin_qp_, + config_.head_dim); + fillQuantizedFloatChunk(features.llm_embedding_cos, processed_tokens, chunk, prefill_io_.inputs[2], cos_qp_, + config_.head_dim); + + auto module_inputs = prefill_io_.inputs; + prefill_io_.outputs = (*prefill_io_.module)(module_inputs); + if (dump_block_outputs_ || dump_layer0_outputs_ || dump_visual_tokens_) { + dumpPrefillDebugOutputs(prefill_io_.inputs, prefill_io_.outputs, chunk, processed_tokens, dump_tokens); + } + + updateCache(config_.ar_len, current_pos, chunk); + kv_manager_u8_->updateAttentionMask(prefill_io_.inputs[3].ptr(), config_.ar_len, current_pos, chunk, + config_.sliding_window); + + const int32_t logits_idx = static_cast((processed_tokens + chunk - 1) % config_.ar_len); + next_token = sampleGreedyFromLogitsAt(prefill_io_.outputs[0], logits_idx); + if (dump_logits_topk_ > 0 && processed_tokens + chunk == num_tokens) { + dumpLogitsTopK(prefill_io_.outputs[0], logits_idx, static_cast(processed_tokens + chunk - 1), "prefill"); + } + + processed_tokens += chunk; + current_pos += chunk; + } + + current_pos_ = current_pos; + prev_position_ids_ = features.position_ids; + return next_token; + } + + int64_t decodeOne(int64_t token) { + rearrangeCache(1); + auto position_ids = makePositionIdsDecode(prev_position_ids_); + prev_position_ids_ = position_ids; + + std::vector attention_map(1); + std::iota(attention_map.begin(), attention_map.end(), -1); + kv_manager_u8_->initAttentionMask(decode_io_.inputs[3].ptr(), attention_map, 1, current_pos_, + config_.sliding_window); + + auto token_tensor = Tensor::empty({1, 1}, mllm::kInt64, mllm::kCPU).alloc(); + token_tensor.ptr()[0] = token; + auto visual_nil = Tensor::nil(); + fillEmbeddingChunk(token_tensor, embedding_weight_, visual_nil, 0, 0, 1, decode_io_.inputs[0], embedding_weight_qp_, + input_embedding_qp_, qnn_cfg_.hidden_size); + + auto inv = mllm::models::qwen2vl::makeMultimodalRoPEInvFreq( + qnn_cfg_.hidden_size / qnn_cfg_.num_attention_heads, qnn_cfg_.rope_theta); + auto [sin, cos] = mllm::models::qwen2vl::makeMultimodalPositionEmbedding( + position_ids, inv, qnn_cfg_.max_position_embeddings, qnn_cfg_.hidden_size / qnn_cfg_.num_attention_heads, + qnn_cfg_.mrope_section); + fillQuantizedFloatChunk(sin, 0, 1, decode_io_.inputs[1], sin_qp_, config_.head_dim); + fillQuantizedFloatChunk(cos, 0, 1, decode_io_.inputs[2], cos_qp_, config_.head_dim); + + decode_io_.module->setOutputTensors(decode_io_.outputs); + auto module_inputs = decode_io_.inputs; + decode_io_.outputs = (*decode_io_.module)(module_inputs); + + updateCache(1, current_pos_, 1); + auto next_token = sampleGreedyFromLogitsAt(decode_io_.outputs[0], 0); + ++current_pos_; + return next_token; + } + + void generate(const PromptFeatures& features, + Qwen2VLTokenizer& tokenizer, + int32_t gen_len, + const std::function& token_callback) { + prefill_start_ = std::chrono::high_resolution_clock::now(); + int64_t next_token = prefill(features); + prefill_end_ = std::chrono::high_resolution_clock::now(); + + generated_tokens_ = 1; + emitToken(next_token, tokenizer, token_callback); + + decode_start_ = std::chrono::high_resolution_clock::now(); + for (int32_t i = 1; i < gen_len; ++i) { + if (current_pos_ >= config_.context_len) { break; } + if (eos_ids_.count(next_token)) { break; } + next_token = decodeOne(next_token); + ++generated_tokens_; + emitToken(next_token, tokenizer, token_callback); + if (eos_ids_.count(next_token)) { break; } + } + decode_end_ = std::chrono::high_resolution_clock::now(); + prefill_tokens_ = features.sequence.shape()[1]; + } + + void perfSummary() const { + const auto llm_prefill_us = std::chrono::duration_cast(prefill_end_ - prefill_start_).count(); + const auto decode_us = std::chrono::duration_cast(decode_end_ - decode_start_).count(); + const auto vit_us = std::chrono::duration_cast(vit_end_ - vit_start_).count(); + const auto prefill_total_us = vit_us + llm_prefill_us; + const auto total_us = prefill_total_us + decode_us; + + fmt::print(fg(fmt::color::cyan), "\n{:=^50}\n", " QNN AOT Performance "); + fmt::print("Total time : {:10.3f} s\n", total_us / 1000000.0); + fmt::print("Prefill total time : {:10.3f} s ({:6.2f} tokens/s)\n", prefill_total_us / 1000000.0, + prefill_total_us > 0 ? prefill_tokens_ / (prefill_total_us / 1000000.0) : 0.0); + fmt::print("LLM prefill time : {:10.3f} s ({:6.2f} tokens/s)\n", llm_prefill_us / 1000000.0, + llm_prefill_us > 0 ? prefill_tokens_ / (llm_prefill_us / 1000000.0) : 0.0); + fmt::print("Decode time : {:10.3f} s ({:6.2f} tokens/s)\n", decode_us / 1000000.0, + decode_us > 0 ? generated_tokens_ / (decode_us / 1000000.0) : 0.0); + fmt::print("TTFT : {:10.3f} s\n", prefill_total_us / 1000000.0); + fmt::print("Prefill tokens : {:10d}\n", prefill_tokens_); + fmt::print("Decode steps : {:10d}\n", generated_tokens_); + fmt::print("Avg decode time : {:10.3f} s/token\n", + generated_tokens_ > 0 ? (decode_us / 1000000.0) / generated_tokens_ : 0.0); + fmt::print(fg(fmt::color::cyan), "{:=^50}\n", ""); + fmt::print(fg(fmt::color::magenta), "\n{:=^50}\n", " Custom Events "); + fmt::print("ViT : {:10.2f} μs\n", static_cast(vit_us)); + fmt::print(fg(fmt::color::magenta), "{:=^50}\n", ""); + } + + private: + void rearrangeCache(int32_t ar_len) { + if (key_cache_uint16_) { + MLLM_RT_ASSERT(key_cache_manager_u16_ != nullptr); + key_cache_manager_u16_->rearrangeCache(ar_len); + } + kv_manager_u8_->rearrangeCache(ar_len); + } + + void updateCache(int32_t ar_len, int32_t current_pos, int32_t chunk) { + if (key_cache_uint16_) { + MLLM_RT_ASSERT(key_cache_manager_u16_ != nullptr); + key_cache_manager_u16_->updateCache(ar_len, current_pos, chunk, {}); + } + kv_manager_u8_->updateCache(ar_len, current_pos, chunk, {}); + } + + void emitToken(int64_t token, + Qwen2VLTokenizer& tokenizer, + const std::function& token_callback) { + if (!token_callback) { return; } + auto wstr = tokenizer.detokenize(token); + token_callback(mllm::preprocessor::wideString2Utf8String(wstr)); + } + + const Qwen2VLConfig& qnn_cfg_; + const Qwen2VLConfig& visual_cfg_; + mllm::ParameterFile::ptr_t qnn_params_; + Tensor embedding_weight_; + QuantParams embedding_weight_qp_; + QuantParams input_embedding_qp_; + QuantParams sin_qp_; + QuantParams cos_qp_; + QuantParams logits_qp_; + std::vector block_out_qps_; + std::vector> layer0_output_qps_; + QnnAOTConfig config_; + std::unique_ptr> kv_manager_u8_; + std::unique_ptr> key_cache_manager_u16_; + RuntimeIO prefill_io_; + RuntimeIO decode_io_; + QnnAOTModuleCache visual_modules_; + Tensor prev_position_ids_; + int64_t current_pos_ = 0; + int32_t prefill_tokens_ = 0; + int32_t generated_tokens_ = 0; + bool dump_block_outputs_ = false; + bool dump_layer0_outputs_ = false; + bool dump_visual_tokens_ = false; + bool key_cache_uint16_ = false; + bool visual_qnn_ = false; + std::string visual_bundle_layout_ = "6x8"; + std::vector visual_bucket_grids_; + int32_t dump_logits_topk_ = 0; + std::vector dump_token_indices_; + std::string dump_path_; + std::vector prompt_token_ids_; + int32_t vision_token_start_ = -1; + int32_t vision_token_count_ = 0; + std::unordered_set eos_ids_{151643, 151645}; + std::chrono::high_resolution_clock::time_point vit_start_; + std::chrono::high_resolution_clock::time_point vit_end_; + std::chrono::high_resolution_clock::time_point prefill_start_; + std::chrono::high_resolution_clock::time_point prefill_end_; + std::chrono::high_resolution_clock::time_point decode_start_; + std::chrono::high_resolution_clock::time_point decode_end_; + + static float dequantizeUInt16(uint16_t value, const QuantParams& qp) { + return (static_cast(value) - qp.zero_point) * qp.scale; + } + + static float dequantizeUInt8(uint8_t value, const QuantParams& qp) { + return (static_cast(value) - qp.zero_point) * qp.scale; + } + + static std::vector tensorToTokenIds(Tensor sequence) { + const auto seq_len = sequence.shape()[1]; + const auto* ids = sequence.ptr(); + return std::vector(ids, ids + seq_len); + } + + void initLayer0QuantParams() { + const auto p = [&](const std::string& label, const std::string& base) { + layer0_output_qps_.push_back( + {label, readQuantParams(qnn_params_, base + ".fake_quant.scale", base + ".fake_quant.zero_point")}); + }; + const auto p_heads = [&](const std::string& label, const std::string& base) { + for (int32_t h = 0; h < qnn_cfg_.num_attention_heads; ++h) { p(label + "_h" + std::to_string(h), base); } + }; + p("layer0_input_layernorm_out", "model.layers.0.self_attn.q_proj_input_qdq"); + p("layer0_q_proj_out_flat", "model.layers.0.self_attn.q_proj_output_qdq"); + p("layer0_k_proj_out_flat", "model.layers.0.self_attn.k_proj_output_qdq"); + p("layer0_v_proj_out_flat", "model.layers.0.self_attn.v_cast_to_int16_qdq"); + p("layer0_v_cache_out_flat", "model.layers.0.self_attn.v_cast_to_int8_qdq"); + p("layer0_q_rope_out_flat", "model.layers.0.self_attn.q_rope_add_0_output_qdq"); + p("layer0_k_rope_out_flat", "model.layers.0.self_attn.k_rope_add_0_output_qdq"); + p_heads("layer0_qk_matmul_out_flat", "model.layers.0.self_attn.qk_matmul_output_qdq"); + p_heads("layer0_qk_scaled_out_flat", "model.layers.0.self_attn.mul_0_output_qdq"); + p_heads("layer0_qk_masked_out_flat", "model.layers.0.self_attn.where_attn_qdq"); + p_heads("layer0_softmax_out_flat", "model.layers.0.self_attn.softmax_output_qdq"); + p("layer0_attn_value_out_flat", "model.layers.0.self_attn.attn_value_matmul_output_qdq"); + p("layer0_o_proj_out", "model.layers.0.add_0_lhs_input_qdq"); + p("layer0_post_attention_layernorm_out", "model.layers.0.mlp.up_proj_input_qdq"); + p("layer0_mlp_up_proj_out", "model.layers.0.mlp.up_proj_output_qdq"); + p("layer0_mlp_gate_proj_out", "model.layers.0.mlp.gate_proj_output_qdq"); + p("layer0_mlp_gate_act_out", "model.layers.0.mlp.act_output_qdq"); + p("layer0_mlp_down_proj_input", "model.layers.0.mlp.down_proj_input_qdq"); + p("layer0_mlp_down_proj_out", "model.layers.0.add_1_lhs_input_qdq"); + layer0_output_qps_.push_back( + {"layer0_block_out", readQuantParams(qnn_params_, "model.layers.1.input_layernorm_input_qdq.fake_quant.scale", + "model.layers.1.input_layernorm_input_qdq.fake_quant.zero_point")}); + } + + void dumpPromptSummary() const { + const auto seq_len = static_cast(prompt_token_ids_.size()); + fmt::print("[Qwen2VL AOT Dump] seq_len={} token_ids_first=", seq_len); + const int32_t first = std::min(seq_len, 16); + for (int32_t i = 0; i < first; ++i) { fmt::print("{}{}", i == 0 ? "" : ",", prompt_token_ids_[i]); } + fmt::print(" token_ids_last="); + const int32_t start = std::max(0, seq_len - 16); + for (int32_t i = start; i < seq_len; ++i) { fmt::print("{}{}", i == start ? "" : ",", prompt_token_ids_[i]); } + fmt::print("\n"); + } + + void dumpTokenIdLine(std::ofstream& out, const char* label, int32_t begin, int32_t end) const { + out << label << "="; + for (int32_t i = begin; i < end; ++i) { out << (i == begin ? "" : ",") << prompt_token_ids_[i]; } + out << "\n"; + } + + void dumpVectorLine(std::ofstream& out, const std::string& label, int32_t layer, int32_t token, const std::vector& vec) { + double dot = 0.0; + double sum = 0.0; + double sum_sq = 0.0; + double max_abs = 0.0; + for (float v : vec) { + sum += v; + sum_sq += static_cast(v) * static_cast(v); + max_abs = std::max(max_abs, std::abs(static_cast(v))); + } + const double mean = vec.empty() ? 0.0 : sum / static_cast(vec.size()); + for (float v : vec) { + const double d = static_cast(v) - mean; + dot += d * d; + } + const double stddev = vec.empty() ? 0.0 : std::sqrt(dot / static_cast(vec.size())); + const double rms = vec.empty() ? 0.0 : std::sqrt(sum_sq / static_cast(vec.size())); + out << "[LAYER_DUMP][NPU][label=" << label << "][layer=" << layer << "][token=" << token << "] mean=" << mean + << " std=" << stddev << " rms=" << rms << " maxabs=" << max_abs; + for (int32_t i = 0; i < 4; ++i) { out << " d" << i << "=" << (i < static_cast(vec.size()) ? vec[i] : 0.0f); } + out << "\n"; + + out << "[LAYER_VEC][NPU][label=" << label << "][layer=" << layer << "][token=" << token << "][dims=" + << vec.size() << "] v="; + for (size_t i = 0; i < vec.size(); ++i) { out << (i == 0 ? "" : ",") << vec[i]; } + out << "\n"; + } + + std::vector selectDumpTokens(int64_t num_tokens) const { + std::vector tokens; + const auto normalize = [&](int32_t token) -> int32_t { + if (token < 0) { token = static_cast(num_tokens) + token; } + return token; + }; + + for (auto token : dump_token_indices_) { + token = normalize(token); + if (token >= 0 && token < num_tokens) { tokens.push_back(token); } + } + if (dump_visual_tokens_ && vision_token_start_ >= 0 && vision_token_count_ > 0) { + tokens.push_back(vision_token_start_); + tokens.push_back(vision_token_start_ + vision_token_count_ / 2); + tokens.push_back(vision_token_start_ + vision_token_count_ - 1); + tokens.push_back(static_cast(num_tokens - 1)); + } + if (tokens.empty()) { tokens.push_back(static_cast(num_tokens - 1)); } + + std::sort(tokens.begin(), tokens.end()); + tokens.erase(std::unique(tokens.begin(), tokens.end()), tokens.end()); + return tokens; + } + + void writeDumpHeader(int64_t num_tokens, const std::vector& dump_tokens) const { + std::ofstream out(dump_path_, std::ios::out | std::ios::trunc); + if (!out) { + fmt::print("[Qwen2VL AOT Dump] failed to open dump path: {}\n", dump_path_); + return; + } + out << "# Qwen2-VL QNN AOT debug dump\n"; + out << "seq_len=" << num_tokens << "\n"; + out << "global_token=" << (num_tokens - 1) << "\n"; + out << "vision_token_start=" << vision_token_start_ << "\n"; + out << "vision_token_count=" << vision_token_count_ << "\n"; + out << "dump_tokens="; + for (size_t i = 0; i < dump_tokens.size(); ++i) { out << (i == 0 ? "" : ",") << dump_tokens[i]; } + out << "\n"; + if (!prompt_token_ids_.empty()) { + const int32_t first_end = std::min(prompt_token_ids_.size(), 16); + const int32_t last_begin = std::max(0, static_cast(prompt_token_ids_.size()) - 16); + dumpTokenIdLine(out, "token_ids_first", 0, first_end); + dumpTokenIdLine(out, "token_ids_last", last_begin, static_cast(prompt_token_ids_.size())); + } + } + + std::vector dequantizeRow(Tensor& tensor, int32_t token_in_chunk, const QuantParams& qp) const { + auto tensor_cpu = tensor.to(mllm::kCPU); + const auto& shape = tensor_cpu.shape(); + const auto dtype = tensor_cpu.dtype(); + const bool is_uint8 = dtype == mllm::kUInt8 || dtype == mllm::kUInt8PerTensorSym || dtype == mllm::kUInt8PerTensorAsy; + if (shape.size() == 4) { + MLLM_RT_ASSERT_EQ(shape[0], 1); + MLLM_RT_ASSERT(token_in_chunk >= 0 && token_in_chunk < shape[2]); + const int32_t heads = shape[1]; + const int32_t seq = shape[2]; + const int32_t context = shape[3]; + std::vector vec(heads * context); + if (is_uint8) { + const auto* data = tensor_cpu.ptr(); + for (int32_t h = 0; h < heads; ++h) { + const uint8_t* row = data + (h * seq + token_in_chunk) * context; + for (int32_t d = 0; d < context; ++d) { vec[h * context + d] = dequantizeUInt8(row[d], qp); } + } + } else { + const auto* data = tensor_cpu.ptr(); + for (int32_t h = 0; h < heads; ++h) { + const uint16_t* row = data + (h * seq + token_in_chunk) * context; + for (int32_t d = 0; d < context; ++d) { vec[h * context + d] = dequantizeUInt16(row[d], qp); } + } + } + return vec; + } + const int32_t dims = shape.back(); + MLLM_RT_ASSERT(token_in_chunk >= 0); + MLLM_RT_ASSERT(tensor_cpu.numel() >= static_cast(token_in_chunk + 1) * dims); + std::vector vec(dims); + if (is_uint8) { + const auto* data = tensor_cpu.ptr(); + const uint8_t* row = data + token_in_chunk * dims; + for (int32_t d = 0; d < dims; ++d) { vec[d] = dequantizeUInt8(row[d], qp); } + } else { + const auto* data = tensor_cpu.ptr(); + const uint16_t* row = data + token_in_chunk * dims; + for (int32_t d = 0; d < dims; ++d) { vec[d] = dequantizeUInt16(row[d], qp); } + } + return vec; + } + + static std::string attentionHeadGroupBase(const std::string& label) { + static const std::vector bases = { + "layer0_qk_matmul_out_flat", + "layer0_qk_scaled_out_flat", + "layer0_qk_masked_out_flat", + "layer0_softmax_out_flat", + }; + for (const auto& base : bases) { + if (label == base + "_h0") { return base; } + } + return ""; + } + + void dumpInputEmbedding(std::ofstream& out, Tensor& input_embeddings, int32_t token_in_chunk, int32_t global_token) { + auto vec = dequantizeRow(input_embeddings, token_in_chunk, input_embedding_qp_); + dumpVectorLine(out, "input_embeddings", 0, global_token, vec); + } + + void dumpLogitsTopK(Tensor& logits, int32_t token_in_chunk, int32_t global_token, const std::string& stage) { + if (dump_path_.empty() || dump_logits_topk_ <= 0) { return; } + std::ofstream out(dump_path_, std::ios::out | std::ios::app); + if (!out) { + fmt::print("[Qwen2VL AOT Dump] failed to open dump path: {}\n", dump_path_); + return; + } + auto rows = topKFromLogitsAt(logits, token_in_chunk, dump_logits_topk_, logits_qp_); + for (int32_t rank = 0; rank < static_cast(rows.size()); ++rank) { + const auto& row = rows[rank]; + out << "[LOGITS_TOPK][NPU][stage=" << stage << "][token=" << global_token << "][rank=" << rank + << "] token_id=" << row.token_id << " raw=" << row.raw << " logit=" << row.value << "\n"; + } + fmt::print("[Qwen2VL AOT Dump] wrote logits top-k to {}\n", dump_path_); + } + + void dumpPrefillDebugOutputs(std::vector& inputs, + std::vector& outputs, + int32_t chunk, + int64_t processed_tokens, + const std::vector& dump_tokens) { + if (dump_path_.empty()) { return; } + std::ofstream out(dump_path_, std::ios::out | std::ios::app); + if (!out) { + fmt::print("[Qwen2VL AOT Dump] failed to open dump path: {}\n", dump_path_); + return; + } + std::vector tokens_in_chunk; + for (auto token : dump_tokens) { + if (token >= processed_tokens && token < processed_tokens + chunk) { tokens_in_chunk.push_back(token); } + } + if (tokens_in_chunk.empty()) { return; } + out << "chunk_start=" << processed_tokens << "\n"; + out << "chunk_len=" << chunk << "\n"; + for (auto global_token : tokens_in_chunk) { + const int32_t token_in_chunk = global_token - static_cast(processed_tokens); + dumpInputEmbedding(out, inputs[0], token_in_chunk, global_token); + } + + const int32_t block_offset = 1 + 2 * config_.num_layers; + if (dump_block_outputs_) { + if (outputs.size() < static_cast(block_offset + config_.num_layers)) { + fmt::print("[Qwen2VL AOT Dump] block outputs requested but output count={} is too small\n", outputs.size()); + return; + } + for (auto global_token : tokens_in_chunk) { + const int32_t token_in_chunk = global_token - static_cast(processed_tokens); + for (int32_t l = 0; l < config_.num_layers; ++l) { + auto vec = dequantizeRow(outputs[block_offset + l], token_in_chunk, block_out_qps_[l]); + dumpVectorLine(out, "block_out", l, global_token, vec); + } + } + } + if (dump_layer0_outputs_) { + const int32_t layer0_offset = block_offset + (dump_block_outputs_ ? config_.num_layers : 0); + if (outputs.size() < static_cast(layer0_offset + layer0_output_qps_.size())) { + fmt::print("[Qwen2VL AOT Dump] layer0 outputs requested but output count={} is too small\n", outputs.size()); + return; + } + for (auto global_token : tokens_in_chunk) { + const int32_t token_in_chunk = global_token - static_cast(processed_tokens); + for (size_t i = 0; i < layer0_output_qps_.size();) { + const auto& [label, qp] = layer0_output_qps_[i]; + const auto head_group_base = attentionHeadGroupBase(label); + if (!head_group_base.empty()) { + std::vector vec; + for (int32_t h = 0; h < qnn_cfg_.num_attention_heads; ++h) { + MLLM_RT_ASSERT(i + h < layer0_output_qps_.size()); + auto head_vec = + dequantizeRow(outputs[layer0_offset + static_cast(i) + h], token_in_chunk, + layer0_output_qps_[i + h].second); + vec.insert(vec.end(), head_vec.begin(), head_vec.end()); + } + dumpVectorLine(out, head_group_base, 0, global_token, vec); + i += qnn_cfg_.num_attention_heads; + continue; + } + auto vec = dequantizeRow(outputs[layer0_offset + static_cast(i)], token_in_chunk, qp); + dumpVectorLine(out, label, 0, global_token, vec); + if (label == "layer0_o_proj_out") { dumpVectorLine(out, "layer0_attn_out", 0, global_token, vec); } + if (label == "layer0_mlp_down_proj_out") { dumpVectorLine(out, "layer0_mlp_out", 0, global_token, vec); } + ++i; + } + } + } + fmt::print("[Qwen2VL AOT Dump] wrote debug vectors to {}\n", dump_path_); + } +}; + +std::vector parseDumpTokenIndices(const std::string& text) { + std::vector tokens; + if (text.empty()) { return tokens; } + std::stringstream ss(text); + std::string item; + while (std::getline(ss, item, ',')) { + if (item.empty()) { continue; } + tokens.push_back(std::stoi(item)); + } + return tokens; +} + +bool isInteractiveExitCommand(const std::string& text) { + return text == "/exit" || text == "/quit" || text == "exit" || text == "quit"; +} + +mllm::models::ARGenerationOutputPast makeRequestInputs(Qwen2VLTokenizer& tokenizer, + const std::string& image_path, + const std::string& prompt, + bool visual_qnn, + const std::vector& visual_bucket_grids) { + const auto resize_override = + visual_qnn ? chooseVisualResizeOverrideForOversizedImage(image_path, visual_bucket_grids) : VisualResizeOverride{}; + if (resize_override.enabled) { + fmt::print("[Qwen2VL AOT] visual oversized fallback: native_grid={}x{} patches={} -> resize_grid={}x{} patches={} " + "within bucket_grid={}x{} patches={} graph_suffix={}\n", + resize_override.native_grid_h, + resize_override.native_grid_w, + resize_override.native_grid_h * resize_override.native_grid_w, + resize_override.resize_grid_h, + resize_override.resize_grid_w, + resize_override.resize_grid_h * resize_override.resize_grid_w, + resize_override.bucket.grid_h, + resize_override.bucket.grid_w, + resize_override.bucket.patchTokens(), + visualGraphSuffixForPatchTokens(resize_override.bucket.patchTokens())); + } + return resize_override.enabled + ? tokenizer.convertMessage({.prompt = prompt, .img_file_path = image_path}, + resize_override.resize_grid_h, + resize_override.resize_grid_w) + : tokenizer.convertMessage({.prompt = prompt, .img_file_path = image_path}); +} + +void runOneRequest(Qwen2VLAOTRunner& runner, + Qwen2VLTokenizer& tokenizer, + mllm::models::qwen2vl::Qwen2VisionTransformerPretrainedModel* visual, + const std::string& image_path, + const std::string& prompt, + bool visual_qnn, + const std::vector& visual_bucket_grids, + int32_t gen_len) { + auto inputs = makeRequestInputs(tokenizer, image_path, prompt, visual_qnn, visual_bucket_grids); + runner.resetStateForRequest(); + auto features = runner.preparePrompt(inputs, visual); + fmt::print("\nResponse: "); + runner.generate(features, tokenizer, gen_len, [](const std::string& token) { std::cout << token << std::flush; }); + fmt::print("\n"); + runner.perfSummary(); +} + +void runInteractiveLoop(Qwen2VLAOTRunner& runner, + Qwen2VLTokenizer& tokenizer, + mllm::models::qwen2vl::Qwen2VisionTransformerPretrainedModel* visual, + bool visual_qnn, + const std::vector& visual_bucket_grids, + const std::string& default_prompt, + int32_t gen_len) { + fmt::print("\n[Qwen2VL AOT] interactive mode is ready. Enter an image path per request; /exit quits.\n"); + fmt::print("[Qwen2VL AOT] default prompt: {}\n", default_prompt); + + while (true) { + std::string image_path; + std::string prompt; + + fmt::print("\nImage path> "); + std::cout.flush(); + if (!std::getline(std::cin, image_path)) { break; } + image_path = trimInteractiveLine(image_path); + if (image_path.empty() || isInteractiveExitCommand(image_path)) { break; } + + fmt::print("Prompt> "); + std::cout.flush(); + if (!std::getline(std::cin, prompt)) { break; } + prompt = trimInteractiveLine(prompt); + if (isInteractiveExitCommand(prompt)) { break; } + if (prompt.empty()) { prompt = default_prompt; } + + fmt::print("[Qwen2VL AOT] running request: image={}, gen_len={}\n", image_path, gen_len); + runOneRequest(runner, tokenizer, visual, image_path, prompt, visual_qnn, visual_bucket_grids, gen_len); + } + + fmt::print("\n[Qwen2VL AOT] interactive mode stopped.\n"); +} + +} // namespace + +MLLM_MAIN({ + auto& help = Argparse::add("-h|--help").help("Show help message"); + auto& context_path = Argparse::add("-m|--model").help("QNN AOT context .bin path").required(true); + auto& qnn_params_path = + Argparse::add("--qnn_params").help("LPBQ .mllm file used for embedding and QDQ params").required(true); + auto& visual_model_path = + Argparse::add("--visual_model").help("FP32/W4A32 .mllm file used for CPU ViT when --visual_qnn is not set."); + auto& visual_model_version = + Argparse::add("--visual_model_version").help("visual model file version: v1/v2").def("v2"); + auto& tokenizer_path = Argparse::add("-t|--tokenizer").help("tokenizer.json path").required(true); + auto& qnn_config_path = Argparse::add("-c|--config").help("QNN LPBQ config path").required(true); + auto& visual_config_path = + Argparse::add("--visual_config").help("visual model config path").required(true); + auto& image_path = Argparse::add("-i|--image").help("input image path for single-shot mode").def(""); + auto& prompt = Argparse::add("-p|--prompt").help("prompt text").def("Describe this picture."); + auto& interactive = + Argparse::add("--interactive").help("Run repeated image/prompt requests in one process after model initialization."); + auto& ar_len = Argparse::add("--ar_len").help("prefill graph chunk length").def(32); + auto& context_len = Argparse::add("--context_len").help("QNN context length").def(1024); + auto& gen_len = Argparse::add("--gen_len").help("max generated tokens").def(96); + auto& input_embedding_scale = + Argparse::add("--input_embedding_scale") + .help("input_embeddings UInt16 scale. Defaults to the Qwen2-VL visual-safe wide QP; set both input embedding " + "QP arguments to -1 to use model.embed_tokens QP.") + .def(kDefaultInputEmbeddingScale); + auto& input_embedding_zero_point = + Argparse::add("--input_embedding_zero_point") + .help("input_embeddings UInt16 zero point. Defaults to the Qwen2-VL visual-safe wide QP; set both input " + "embedding QP arguments to -1 to use model.embed_tokens QP.") + .def(kDefaultInputEmbeddingZeroPoint); + auto& dump_block_outputs = + Argparse::add("--dump_block_outputs").help("Dump per-layer block_out vectors from debug QNN context."); + auto& dump_layer0_outputs = + Argparse::add("--dump_layer0_outputs").help("Dump layer0 fine-grained vectors from debug QNN context."); + auto& dump_visual_tokens = + Argparse::add("--dump_visual_tokens").help("Dump first/middle/last visual token and final prefill token."); + auto& dump_token_indices = Argparse::add("--dump_token_indices") + .help("Comma-separated global token indices to dump. Negative values count from end.") + .def(""); + auto& dump_logits_topk = + Argparse::add("--dump_logits_topk").help("Dump top-k logits at the final prefill token.").def(0); + auto& dump_path = Argparse::add("--dump_path") + .help("Path on device for debug vector dump.") + .def("/data/local/tmp/qwen2vl_qnn_aot_debug_dump.log"); + auto& key_cache_dtype = + Argparse::add("--key_cache_dtype") + .help("Key cache dtype for experimental contexts: uint8 or uint16. Value cache remains uint8.") + .def("uint8"); + auto& visual_qnn = + Argparse::add("--visual_qnn").help("Use visual graphs from the loaded combined QNN context instead of CPU/KAI visual."); + auto& visual_bundle_layout = + Argparse::add("--visual_bundle_layout").help("visual QNN bundle layout: single, 6x8, early2 or tail4.").def("6x8"); + auto& visual_bucket_grids = Argparse::add("--visual_bucket_grids") + .help("Comma-separated visual QNN patch-grid buckets HxW. Example: 10x16,12x16,26x36.") + .def(""); + + Argparse::parse(argc, argv); + if (help.isSet()) { + Argparse::printHelp(); + return 0; + } + const bool override_input_embedding_qp = input_embedding_scale.get() > 0.0f || input_embedding_zero_point.get() >= 0; + if (override_input_embedding_qp && (input_embedding_scale.get() <= 0.0f || input_embedding_zero_point.get() < 0)) { + std::cerr << "input embedding override requires both --input_embedding_scale and --input_embedding_zero_point; " + "set both to -1 to use model.embed_tokens QP\n"; + return 1; + } + const bool key_cache_uint16 = key_cache_dtype.get() == "uint16"; + if (!key_cache_uint16 && key_cache_dtype.get() != "uint8") { + std::cerr << "--key_cache_dtype must be uint8 or uint16\n"; + return 1; + } + if (!interactive.isSet() && !image_path.isSet()) { + std::cerr << "-i/--image is required unless --interactive is set\n"; + return 1; + } + + mllm::initQnnBackend(context_path.get()); + + auto qnn_cfg = Qwen2VLConfig(qnn_config_path.get()); + auto visual_cfg = Qwen2VLConfig(visual_config_path.get()); + auto tokenizer = Qwen2VLTokenizer(tokenizer_path.get()); + + auto qnn_params = mllm::load(qnn_params_path.get(), mllm::ModelFileVersion::kV2); + std::unique_ptr visual; + if (!visual_qnn.isSet()) { + if (!visual_model_path.isSet()) { + std::cerr << "--visual_model is required when --visual_qnn is not set\n"; + return 1; + } + mllm::ModelFileVersion visual_file_version = mllm::ModelFileVersion::kV2; + if (visual_model_version.get() == "v1") { visual_file_version = mllm::ModelFileVersion::kV1; } + auto visual_params = mllm::load(visual_model_path.get(), visual_file_version); + visual = std::make_unique("visual", visual_cfg); + visual->load(visual_params); + } + + auto parsed_visual_bucket_grids = parseVisualBucketGrids(visual_bucket_grids.get()); + + Qwen2VLAOTRunner runner(qnn_cfg, visual_cfg, qnn_params, ar_len.get(), context_len.get(), input_embedding_scale.get(), + input_embedding_zero_point.get(), dump_block_outputs.isSet(), dump_layer0_outputs.isSet(), + dump_visual_tokens.isSet(), key_cache_uint16, visual_qnn.isSet(), visual_bundle_layout.get(), + parsed_visual_bucket_grids, + dump_logits_topk.get(), + parseDumpTokenIndices(dump_token_indices.get()), dump_path.get()); + if (!runner.load()) { + std::cerr << "Failed to load Qwen2-VL QNN AOT runner\n"; + return 1; + } + + if (interactive.isSet()) { + runInteractiveLoop(runner, + tokenizer, + visual.get(), + visual_qnn.isSet(), + parsed_visual_bucket_grids, + prompt.get(), + gen_len.get()); + } else { + runOneRequest(runner, + tokenizer, + visual.get(), + image_path.get(), + prompt.get(), + visual_qnn.isSet(), + parsed_visual_bucket_grids, + gen_len.get()); + } + mllm::memoryReport(); + return 0; +}); diff --git a/examples/qwen2vl_qnn_aot/compile.cpp b/examples/qwen2vl_qnn_aot/compile.cpp new file mode 100644 index 000000000..fe15eac49 --- /dev/null +++ b/examples/qwen2vl_qnn_aot/compile.cpp @@ -0,0 +1,398 @@ +// Copyright (c) MLLM Team. +// Licensed under the MIT License. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "modeling_qwen2vl_qnn_aot.hpp" +#include "visual_aot_helpers.hpp" + +using mllm::Argparse; + +namespace { + +constexpr float kDefaultInputEmbeddingScale = 0.002563515f; +constexpr int32_t kDefaultInputEmbeddingZeroPoint = 15604; + +template +void addCausalMaskParams(const ParamsT& params) { + params->push("causal_mask.scale", mllm::Tensor::constant(0.001 / 65535.f, mllm::kFloat32)); + params->push("causal_mask.zero_point", mllm::Tensor::constant(65535, mllm::kInt32)); + params->push("constant_zero.scale", mllm::Tensor::constant(0.001 / 65535.f, mllm::kFloat32)); + params->push("constant_zero.zero_point", mllm::Tensor::constant(65535, mllm::kInt32)); +} + +template +mllm::Tensor makeUInt16AsymTensor(const std::vector& shape, + const std::string& scale_name, + const std::string& zp_name, + const ParamsT& params) { + auto tensor = mllm::Tensor::empty(shape, mllm::kUInt16); + tensor = tensor.__unsafeSetDType(mllm::kUInt16PerTensorAsy); + tensor.attach("scale", params->pull(scale_name).impl(), true); + tensor.attach("zero_point", params->pull(zp_name).impl(), true); + return tensor; +} + +template +mllm::Tensor makeUInt16SymTensor(const std::vector& shape, + const std::string& scale_name, + const ParamsT& params) { + auto tensor = mllm::Tensor::empty(shape, mllm::kUInt16); + tensor = tensor.__unsafeSetDType(mllm::kUInt16PerTensorSym); + tensor.attach("scale", params->pull(scale_name).impl(), true); + return tensor; +} + +inline mllm::Tensor makeUInt16AsymTensor(const std::vector& shape, float scale, int32_t zero_point) { + auto tensor = mllm::Tensor::empty(shape, mllm::kUInt16); + tensor = tensor.__unsafeSetDType(mllm::kUInt16PerTensorAsy); + tensor.attach("scale", mllm::Tensor::constant(scale, mllm::kFloat32).impl(), true); + tensor.attach("zero_point", mllm::Tensor::constant(zero_point, mllm::kInt32).impl(), true); + return tensor; +} + +template +std::unordered_map makeTraceInputs(int seq_len, + int context_len, + const mllm::models::qwen2vl::Qwen2VLConfig& model_cfg, + const ParamsT& params, + bool override_input_embedding_qp, + float input_embedding_scale, + int32_t input_embedding_zero_point, + bool key_cache_uint16) { + const int head_dim = model_cfg.hidden_size / model_cfg.num_attention_heads; + + std::unordered_map trace_inputs; + + if (override_input_embedding_qp) { + trace_inputs["input_embeddings"] = + makeUInt16AsymTensor({1, seq_len, model_cfg.hidden_size}, input_embedding_scale, input_embedding_zero_point); + } else { + trace_inputs["input_embeddings"] = makeUInt16AsymTensor( + {1, seq_len, model_cfg.hidden_size}, "model.embed_tokens.scale", "model.embed_tokens.zero_point", params); + } + trace_inputs["llm_embedding_sin"] = + makeUInt16AsymTensor({1, seq_len, head_dim}, "model.sin_embedding_input_qdq.fake_quant.scale", + "model.sin_embedding_input_qdq.fake_quant.zero_point", params); + trace_inputs["llm_embedding_cos"] = + makeUInt16AsymTensor({1, seq_len, head_dim}, "model.cos_embedding_input_qdq.fake_quant.scale", + "model.cos_embedding_input_qdq.fake_quant.zero_point", params); + trace_inputs["causal_mask"] = + makeUInt16AsymTensor({1, 1, seq_len, context_len}, "causal_mask.scale", "causal_mask.zero_point", params); + + for (int i = 0; i < model_cfg.num_hidden_layers; ++i) { + auto past_key_name = "past_key_" + std::to_string(i); + auto past_value_name = "past_value_" + std::to_string(i); + + if (key_cache_uint16) { + trace_inputs[past_key_name] = makeUInt16SymTensor( + { + 1, + model_cfg.num_key_value_heads, + head_dim, + context_len - seq_len, + }, + "model.layers." + std::to_string(i) + ".self_attn.k_rope_add_0_output_qdq.fake_quant.scale", + params); + } else { + trace_inputs[past_key_name] = mllm::Tensor::empty({ + 1, + model_cfg.num_key_value_heads, + head_dim, + context_len - seq_len, + }, mllm::kUInt8PerTensorSym); + trace_inputs[past_key_name].attach("scale", + params->pull("model.layers." + std::to_string(i) + + ".self_attn.k_cast_to_int8_qdq.fake_quant.scale") + .impl(), + true); + trace_inputs[past_key_name].attach("zero_point", + params->pull("model.layers." + std::to_string(i) + + ".self_attn.k_cast_to_int8_qdq.fake_quant.zero_point") + .impl(), + true); + } + trace_inputs[past_value_name] = mllm::Tensor::empty({ + 1, + model_cfg.num_key_value_heads, + context_len - seq_len, + head_dim, + }, mllm::kUInt8PerTensorSym); + + trace_inputs[past_value_name].attach("scale", + params->pull("model.layers." + std::to_string(i) + + ".self_attn.v_cast_to_int8_qdq.fake_quant.scale") + .impl(), + true); + trace_inputs[past_value_name].attach("zero_point", + params->pull("model.layers." + std::to_string(i) + + ".self_attn.v_cast_to_int8_qdq.fake_quant.zero_point") + .impl(), + true); + } + + return trace_inputs; +} + +void compileVisualBundleGraphs(mllm::qnn::aot::QnnAOTEnv& qnn_aot_env, + const std::string& visual_aot_cfg_path, + const mllm::ParameterFile::ptr_t& visual_params, + const mllm::models::qwen2vl::Qwen2VLConfig& visual_cfg, + int32_t visual_patch_tokens, + int32_t patch_flat_dim, + const std::string& bundle_layout, + const std::string& visual_ir_prefix, + const std::string& graph_suffix) { + const int32_t half_dim = visual_cfg.visual_embed_dim / visual_cfg.visual_num_heads / 2; + auto visual_embedding_sin = mllm::Tensor::empty({1, visual_patch_tokens, 1, half_dim}, mllm::kFloat32, mllm::kCPU).alloc(); + auto visual_embedding_cos = mllm::Tensor::empty({1, visual_patch_tokens, 1, half_dim}, mllm::kFloat32, mllm::kCPU).alloc(); + auto visual_attention_mask = + mllm::Tensor::empty({1, 1, 1, visual_patch_tokens}, mllm::kFloat32, mllm::kCPU).alloc(); + + const auto segments = qwen2vl_qnn_aot::makeVisualBundleSegments(bundle_layout, + visual_cfg.visual_depth, + visual_patch_tokens, + patch_flat_dim, + visual_cfg, + graph_suffix); + + for (const auto& segment : segments) { + fmt::print("\n{:=^72}\n", fmt::format(" Compile {} ", segment.graph_name)); + auto segment_img = mllm::Tensor::empty(segment.input_shape, mllm::kFloat32, mllm::kCPU).alloc(); + auto visual = qwen2vl_qnn_aot::Qwen2VisionTransformerPretrainedModelAOTRewrite("visual", + visual_cfg, + segment.start_block, + segment.visual_blocks, + segment.skip_merger, + segment.skip_patch_embed); + visual.load(visual_params); + + mllm::ir::lowlevel::traceStart(); + auto visual_output = segment.visual_blocks > 0 + ? mllm::ir::lowlevel::traceModule(visual, + segment_img, + visual_embedding_sin, + visual_embedding_cos, + visual_attention_mask)[0] + : mllm::ir::lowlevel::traceModule(visual, segment_img, visual_embedding_sin, visual_embedding_cos)[0]; + auto visual_ir = mllm::ir::lowlevel::traceStop(); + + fmt::print("visual segment output shape: [{}, {}]\n", visual_output.shape()[0], visual_output.shape()[1]); + const auto segment_ir_path = visual_ir_prefix + "." + segment.ir_name + ".mir"; + mllm::redirect(segment_ir_path, [&]() { mllm::print(visual_ir); }); + fmt::print("Visual IR dumped to: {}\n", segment_ir_path); + + mllm::ir::PassManager pm(visual_ir); + pm.reg(mllm::qnn::aot::createQnnAOTSimpleLoweringPipeline(&qnn_aot_env, + visual_aot_cfg_path, + visual_params, + segment.graph_name)); + if (!pm.run()) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "Visual QNN AOT lowering failed for graph {}.", segment.graph_name); + } + } +} + +void compileVisualBundleGraphsFromImage(mllm::qnn::aot::QnnAOTEnv& qnn_aot_env, + const std::string& visual_aot_cfg_path, + const mllm::ParameterFile::ptr_t& visual_params, + const mllm::models::qwen2vl::Qwen2VLConfig& visual_cfg, + const std::string& tokenizer_path, + const std::string& image_path, + const std::string& prompt, + const std::string& bundle_layout, + const std::string& visual_ir_prefix) { + if (tokenizer_path.empty() || image_path.empty()) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "--include_visual_bundle requires --tokenizer and --image."); + } + + auto tokenizer = mllm::models::qwen2vl::Qwen2VLTokenizer(tokenizer_path); + auto inputs = tokenizer.convertMessage({.prompt = prompt, .img_file_path = image_path}); + auto img = inputs.at("img"); + compileVisualBundleGraphs(qnn_aot_env, + visual_aot_cfg_path, + visual_params, + visual_cfg, + img.shape()[0], + img.shape()[1], + bundle_layout, + visual_ir_prefix, + ""); +} + +} // namespace + +MLLM_MAIN({ + auto& help = Argparse::add("-h|--help").help("Show help message"); + auto& model_path = Argparse::add("-m|--model_path").help("Model file path."); + auto& model_cfg_path = Argparse::add("-c|--config").help("Model config file path."); + auto& qnn_aot_cfg_files = Argparse::add("-aot_cfg|--aot_config").help("AOT Config file path."); + auto& qnn_env_path = Argparse::add("-qnn_env|--qnn_env_path") + .def("/opt/qcom/aistack/qairt/2.41.0.251128/lib/x86_64-linux-clang/") + .help("QNN AOT Environment path."); + auto& output_context_path = Argparse::add("-o|--output_context_name").help("Output QNN context path."); + auto& context_len = Argparse::add("--context_len").help("QNN context length.").def(1024); + auto& prefill_len = Argparse::add("--prefill_len").help("Prefill graph sequence length.").def(32); + auto& input_embedding_scale = + Argparse::add("--input_embedding_scale") + .help("input_embeddings UInt16 scale. Defaults to the Qwen2-VL visual-safe wide QP; set both input embedding " + "QP arguments to -1 to use model.embed_tokens QP.") + .def(kDefaultInputEmbeddingScale); + auto& input_embedding_zero_point = + Argparse::add("--input_embedding_zero_point") + .help("input_embeddings UInt16 zero point. Defaults to the Qwen2-VL visual-safe wide QP; set both input " + "embedding QP arguments to -1 to use model.embed_tokens QP.") + .def(kDefaultInputEmbeddingZeroPoint); + auto& dump_block_outputs = + Argparse::add("--dump_block_outputs").help("Expose per-layer block_out tensors as graph outputs for debugging."); + auto& dump_layer0_outputs = + Argparse::add("--dump_layer0_outputs").help("Expose layer0 fine-grained tensors as graph outputs for debugging."); + auto& key_cache_dtype = + Argparse::add("--key_cache_dtype").help("Key cache dtype for experimental contexts: uint8 or uint16.").def("uint8"); + auto& include_visual_bundle = + Argparse::add("--include_visual_bundle") + .help("Also compile the Qwen2-VL visual tower bundle graphs into the same QNN context."); + auto& skip_llm_graphs = + Argparse::add("--skip_llm_graphs") + .help("Only compile requested auxiliary graphs, such as the visual bundle, and skip LLM prefill/decode graphs."); + auto& visual_model_path = Argparse::add("--visual_model") + .help("Optional FP32/W32A32 visual-capable .mllm for visual bundle graphs. Defaults to --model_path."); + auto& visual_config_path = Argparse::add("--visual_config") + .help("Optional visual config. Defaults to --config."); + auto& visual_aot_config_path = Argparse::add("--visual_aot_config") + .help("AOT config for visual bundle graphs. Defaults to --aot_config."); + auto& visual_bundle_layout = + Argparse::add("--visual_bundle_layout") + .def("6x8") + .help("Visual bundle layout: single, 6x8, tail4, early2 or block1."); + auto& visual_bucket_grids = Argparse::add("--visual_bucket_grids") + .def("") + .help("Comma-separated visual patch grid buckets HxW. Example: 10x16,12x16,26x36."); + auto& tokenizer_path = Argparse::add("-t|--tokenizer").help("tokenizer.json path for visual input shape."); + auto& image_path = Argparse::add("-i|--image").help("image path for visual input shape."); + auto& prompt = Argparse::add("-p|--prompt").help("prompt text used with --image.").def("describe this picture"); + auto& visual_ir_prefix = + Argparse::add("--visual_ir_prefix").def("qwen2vl_visual_combined").help("Prefix for visual IR dumps."); + + Argparse::parse(argc, argv); + + if (help.isSet()) { + Argparse::printHelp(); + return 0; + } + if (!model_path.isSet() || !model_cfg_path.isSet() || !qnn_aot_cfg_files.isSet() || !output_context_path.isSet()) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "Missing required argument."); + Argparse::printHelp(); + return -1; + } + if (prefill_len.get() <= 0 || context_len.get() <= prefill_len.get()) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "Invalid context_len/prefill_len: {} / {}", context_len.get(), + prefill_len.get()); + return -1; + } + const bool override_input_embedding_qp = input_embedding_scale.get() > 0.0f || input_embedding_zero_point.get() >= 0; + if (override_input_embedding_qp && (input_embedding_scale.get() <= 0.0f || input_embedding_zero_point.get() < 0)) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, + "input embedding override requires both --input_embedding_scale and --input_embedding_zero_point; " + "set both to -1 to use model.embed_tokens QP."); + } + const bool key_cache_uint16 = key_cache_dtype.get() == "uint16"; + if (!key_cache_uint16 && key_cache_dtype.get() != "uint8") { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "--key_cache_dtype must be uint8 or uint16."); + } + + auto model_cfg = mllm::models::qwen2vl::Qwen2VLConfig(model_cfg_path.get()); + auto params = mllm::load(model_path.get(), mllm::ModelFileVersion::kV2); + addCausalMaskParams(params); + + mllm::models::qwen2vl::qnn_aot::DebugOutputConfig debug_outputs{ + .dump_block_outputs = dump_block_outputs.isSet(), + .dump_layer0_outputs = dump_layer0_outputs.isSet(), + .key_cache_uint16 = key_cache_uint16, + }; + auto model = mllm::models::qwen2vl::qnn_aot::Qwen2VLForCausalLM(model_cfg, debug_outputs); + model.load(params); + + auto qnn_aot_env = mllm::qnn::aot::QnnAOTEnv(qnn_env_path.get(), + mllm::qnn::aot::parseQcomTargetMachineFromJSONFile(qnn_aot_cfg_files.get())); + + auto trace_and_dump = [&](int seq_len, const std::string& mir_path) { + mllm::print("Tracing Qwen2-VL LLM QNN AOT graph, seq=" + std::to_string(seq_len)); + auto trace_inputs = + makeTraceInputs(seq_len, context_len.get(), model_cfg, params, override_input_embedding_qp, + input_embedding_scale.get(), input_embedding_zero_point.get(), key_cache_uint16); + auto ir = model.trace(trace_inputs, {}); + mllm::print("Trace completed, lowering to QNN AOT."); + + mllm::ir::PassManager pm(ir["model"]); + pm.reg(mllm::qnn::aot::createQnnAOTLoweringPipeline(&qnn_aot_env, qnn_aot_cfg_files.get(), params)); + pm.run(); + + mllm::redirect(mir_path, [&]() { mllm::print(ir["model"]); }); + mllm::print("IR dumped to " + mir_path); + }; + + if (skip_llm_graphs.isSet() && !include_visual_bundle.isSet()) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "--skip_llm_graphs requires --include_visual_bundle or another graph family."); + } + + if (!skip_llm_graphs.isSet()) { + trace_and_dump(prefill_len.get(), "qwen2vl_qnn_aot_" + std::to_string(prefill_len.get()) + ".mir"); + trace_and_dump(1, "qwen2vl_qnn_aot_1.mir"); + } else { + mllm::print("Skipping LLM prefill/decode graph compilation."); + } + + if (include_visual_bundle.isSet()) { + const auto visual_model = visual_model_path.isSet() ? visual_model_path.get() : model_path.get(); + const auto visual_config = visual_config_path.isSet() ? visual_config_path.get() : model_cfg_path.get(); + const auto visual_aot_config = visual_aot_config_path.isSet() ? visual_aot_config_path.get() : qnn_aot_cfg_files.get(); + auto visual_cfg = mllm::models::qwen2vl::Qwen2VLConfig(visual_config); + auto visual_params = mllm::load(visual_model, mllm::ModelFileVersion::kV2); + qwen2vl_qnn_aot::reshapePatchEmbedConv3DWeightForLinear(visual_params, visual_cfg); + + const int32_t patch_flat_dim = + visual_cfg.visual_in_chans * visual_cfg.visual_temporal_patch_size * visual_cfg.visual_patch_size * visual_cfg.visual_patch_size; + const auto buckets = qwen2vl_qnn_aot::parseVisualBucketGrids(visual_bucket_grids.get()); + if (!buckets.empty()) { + const auto bucket_tokens = qwen2vl_qnn_aot::uniqueVisualBucketPatchTokens(buckets); + fmt::print("Compiling {} visual bucket shape(s): ", bucket_tokens.size()); + for (size_t i = 0; i < bucket_tokens.size(); ++i) { fmt::print("{}{}", i == 0 ? "" : ",", bucket_tokens[i]); } + fmt::print("\n"); + for (const int32_t patch_tokens : bucket_tokens) { + compileVisualBundleGraphs(qnn_aot_env, + visual_aot_config, + visual_params, + visual_cfg, + patch_tokens, + patch_flat_dim, + visual_bundle_layout.get(), + visual_ir_prefix.get(), + qwen2vl_qnn_aot::visualGraphSuffixForPatchTokens(patch_tokens)); + } + } else { + compileVisualBundleGraphsFromImage(qnn_aot_env, + visual_aot_config, + visual_params, + visual_cfg, + tokenizer_path.get(), + image_path.get(), + prompt.get(), + visual_bundle_layout.get(), + visual_ir_prefix.get()); + } + } + + qnn_aot_env.saveContext("context.0", output_context_path.get()); + mllm::print("Qwen2-VL QNN AOT compilation completed."); + mllm::print("Context: " + output_context_path.get()); +}); diff --git a/examples/qwen2vl_qnn_aot/compile_visual.cpp b/examples/qwen2vl_qnn_aot/compile_visual.cpp new file mode 100644 index 000000000..d981689fb --- /dev/null +++ b/examples/qwen2vl_qnn_aot/compile_visual.cpp @@ -0,0 +1,552 @@ +// Copyright (c) MLLM Team. +// Licensed under the MIT License. + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using mllm::Argparse; + +namespace { + +std::string defaultQnnAOTEnvPath() { + const char* qnn_root = std::getenv("QNN_SDK_ROOT"); + if (qnn_root != nullptr && qnn_root[0] != '\0') { + return std::string(qnn_root) + "/lib/x86_64-linux-clang/"; + } + return "/opt/qcom/aistack/qairt/2.41.0.251128/lib/x86_64-linux-clang/"; +} + +class PatchEmbedLinear final : public mllm::nn::Module { + int32_t patch_dim_ = 0; + int32_t embed_dim_ = 0; + + mllm::nn::Linear proj_; + + public: + PatchEmbedLinear() = default; + + PatchEmbedLinear(const std::string& name, const mllm::models::qwen2vl::Qwen2VLConfig& cfg) : mllm::nn::Module(name) { + patch_dim_ = cfg.visual_in_chans * cfg.visual_temporal_patch_size * cfg.visual_patch_size * cfg.visual_patch_size; + embed_dim_ = cfg.visual_embed_dim; + proj_ = reg("proj", patch_dim_, embed_dim_, false, mllm::aops::LinearImplTypes::kDefault); + } + + std::vector forward(const std::vector& inputs, + const std::vector& /*args*/) override { + auto hidden_states = inputs[0]; + hidden_states = hidden_states.view({-1, patch_dim_}, true); + hidden_states = proj_(hidden_states).view({-1, embed_dim_}, true); + return {hidden_states}; + } +}; + +class Qwen2VisionTransformerPretrainedModelLinearPatch final : public mllm::nn::Module { + PatchEmbedLinear patch_embed_; + mllm::models::qwen2vl::PatchMerger patch_merger_; + mllm::nn::ModuleList blocks_; + + public: + Qwen2VisionTransformerPretrainedModelLinearPatch() = default; + + Qwen2VisionTransformerPretrainedModelLinearPatch(const std::string& name, + const mllm::models::qwen2vl::Qwen2VLConfig& cfg) + : mllm::nn::Module(name) { + patch_embed_ = reg("patch_embed", cfg); + patch_merger_ = reg("merger", cfg); + blocks_ = reg>("blocks", cfg.visual_depth, cfg); + } + + std::vector forward(const std::vector& inputs, + const std::vector& /*args*/) override { + auto hidden_states = inputs[0]; + auto embedding_sin = inputs[1]; + auto embedding_cos = inputs[2]; + + hidden_states = patch_embed_(hidden_states)[0]; + for (auto& b : blocks_.list()) { hidden_states = b(hidden_states, embedding_sin, embedding_cos)[0]; } + hidden_states = patch_merger_(hidden_states)[0]; + + return {hidden_states}; + } +}; + +class VisionMlpPrimitiveQuickGELU final : public mllm::nn::Module { + int32_t dim_ = 0; + int32_t hidden_dim_ = 0; + + mllm::nn::Linear fc_1_; + mllm::nn::Linear fc_2_; + + public: + VisionMlpPrimitiveQuickGELU() = default; + + VisionMlpPrimitiveQuickGELU(const std::string& name, const mllm::models::qwen2vl::Qwen2VLConfig& cfg) + : mllm::nn::Module(name) { + dim_ = cfg.visual_embed_dim; + hidden_dim_ = cfg.visual_embed_dim * cfg.visual_mlp_ratio; + fc_1_ = reg("fc1", dim_, hidden_dim_, true, cfg.linear_impl_type); + fc_2_ = reg("fc2", hidden_dim_, dim_, true, cfg.linear_impl_type); + } + + std::vector forward(const std::vector& inputs, + const std::vector& /*args*/) override { + auto x = fc_1_(inputs[0]); + x = x * mllm::nn::functional::sigmoid(x * 1.702f); + return {fc_2_(x)}; + } +}; + +class Qwen2VLVisionBlockAOTRewrite final : public mllm::nn::Module { + mllm::nn::LayerNorm norm1_; + mllm::nn::LayerNorm norm2_; + + mllm::models::qwen2vl::VisionAttention attn_; + VisionMlpPrimitiveQuickGELU mlp_; + + public: + Qwen2VLVisionBlockAOTRewrite() = default; + + Qwen2VLVisionBlockAOTRewrite(const std::string& name, const mllm::models::qwen2vl::Qwen2VLConfig& cfg) + : mllm::nn::Module(name) { + norm1_ = reg("norm1", std::vector{cfg.visual_embed_dim}, true, true, 1e-6); + norm2_ = reg("norm2", std::vector{cfg.visual_embed_dim}, true, true, 1e-6); + attn_ = reg("attn", cfg); + mlp_ = reg("mlp", cfg); + } + + std::vector forward(const std::vector& inputs, + const std::vector& /*args*/) override { + auto hidden_states = inputs[0]; + auto visual_embedding_sin = inputs[1]; + auto visual_embedding_cos = inputs[2]; + + hidden_states = hidden_states + attn_(norm1_(hidden_states), visual_embedding_sin, visual_embedding_cos)[0]; + hidden_states = hidden_states + mlp_(norm2_(hidden_states))[0]; + return {hidden_states}; + } +}; + +class Qwen2VisionTransformerPretrainedModelAOTRewrite final : public mllm::nn::Module { + PatchEmbedLinear patch_embed_; + mllm::models::qwen2vl::PatchMerger patch_merger_; + mllm::nn::ModuleList blocks_; + int32_t start_block_ = 0; + int32_t active_blocks_ = -1; + bool skip_merger_ = false; + bool skip_patch_embed_ = false; + + public: + Qwen2VisionTransformerPretrainedModelAOTRewrite() = default; + + Qwen2VisionTransformerPretrainedModelAOTRewrite(const std::string& name, + const mllm::models::qwen2vl::Qwen2VLConfig& cfg, + int32_t start_block = 0, + int32_t active_blocks = -1, + bool skip_merger = false, + bool skip_patch_embed = false) + : mllm::nn::Module(name) { + start_block_ = start_block; + active_blocks_ = active_blocks; + skip_merger_ = skip_merger; + skip_patch_embed_ = skip_patch_embed; + patch_embed_ = reg("patch_embed", cfg); + patch_merger_ = reg("merger", cfg); + blocks_ = reg>("blocks", cfg.visual_depth, cfg); + } + + std::vector forward(const std::vector& inputs, + const std::vector& /*args*/) override { + auto hidden_states = inputs[0]; + auto embedding_sin = inputs[1]; + auto embedding_cos = inputs[2]; + + if (!skip_patch_embed_) { hidden_states = patch_embed_(hidden_states)[0]; } + auto num_blocks = active_blocks_ < 0 ? static_cast(blocks_.list().size()) : active_blocks_; + MLLM_RT_ASSERT(start_block_ >= 0); + MLLM_RT_ASSERT(num_blocks >= 0); + MLLM_RT_ASSERT(start_block_ + num_blocks <= static_cast(blocks_.list().size())); + for (int32_t i = 0; i < num_blocks; ++i) { + hidden_states = blocks_.list()[start_block_ + i](hidden_states, embedding_sin, embedding_cos)[0]; + } + if (!skip_merger_) { hidden_states = patch_merger_(hidden_states)[0]; } + + return {hidden_states}; + } +}; + +std::set currentQnnAOTVisitorOps() { + return { + mllm::OpTypes::kEmbedding, + mllm::OpTypes::kCastType, + mllm::OpTypes::kAdd, + mllm::OpTypes::kMul, + mllm::OpTypes::kNeg, + mllm::OpTypes::kReshape, + mllm::OpTypes::kView, + mllm::OpTypes::kIndex, + mllm::OpTypes::kGather, + mllm::OpTypes::kRMSNorm, + mllm::OpTypes::kLinear, + mllm::OpTypes::kTranspose, + mllm::OpTypes::kSlice, + mllm::OpTypes::kConcat, + mllm::OpTypes::kRepeat, + mllm::OpTypes::kMatMul, + mllm::OpTypes::kReduceMax, + mllm::OpTypes::kReduceMin, + mllm::OpTypes::kMean, + mllm::OpTypes::kReduceSum, + mllm::OpTypes::kEqual, + mllm::OpTypes::kWhere, + mllm::OpTypes::kSoftmax, + mllm::OpTypes::kSigmoid, + mllm::OpTypes::kConv2D, + mllm::OpTypes::kGELU, + mllm::OpTypes::kLayerNorm, + }; +} + +struct OpSummary { + std::map counts; + std::map> names; +}; + +OpSummary summarizeLinalgOps(const mllm::ir::IRContext::ptr_t& ir) { + OpSummary summary; + auto module = ir->topLevelOp()->cast_(); + auto module_writer = mllm::ir::IRWriter(ir, module->getTopRegion()); + + module_writer.walk( + [&](mllm::ir::IRWriter& /*writer*/, + const mllm::ir::graph::SubGraphOp::ptr_t& subgraph) -> mllm::ir::IRWriter::WalkResult { + auto graph_writer = mllm::ir::IRWriter(ir, subgraph->getTopRegion()); + graph_writer.walk( + [&](mllm::ir::IRWriter& /*inner_writer*/, + const mllm::ir::linalg::LinalgIROp::ptr_t& op) -> mllm::ir::IRWriter::WalkResult { + const auto type = op->getAOp()->getOpType(); + summary.counts[type] += 1; + auto name = op->getAOp()->getName(); + if (name.empty()) { name = ""; } + auto& names = summary.names[type]; + if (names.size() < 8) { names.push_back(name); } + return mllm::ir::IRWriter::WALK_CONTINUE; + }); + return mllm::ir::IRWriter::WALK_CONTINUE; + }); + + return summary; +} + +void printSummary(const OpSummary& summary) { + const auto supported = currentQnnAOTVisitorOps(); + + fmt::print("\n{:=^72}\n", " Visual IR op summary "); + for (const auto& [type, count] : summary.counts) { + const bool ok = supported.count(type) > 0; + fmt::print("{:<16} {:>5} {}\n", mllm::optype2Str(type), count, ok ? "visitor exists" : "missing visitor"); + const auto names_it = summary.names.find(type); + if (names_it != summary.names.end()) { + for (const auto& name : names_it->second) { fmt::print(" - {}\n", name); } + } + } + + fmt::print("\n{:=^72}\n", " Missing current QNN AOT visitors "); + bool has_missing = false; + for (const auto& [type, count] : summary.counts) { + if (supported.count(type) > 0) { continue; } + has_missing = true; + fmt::print("- {}: {}\n", mllm::optype2Str(type), count); + } + if (!has_missing) { fmt::print("No missing visitor found in this raw visual trace.\n"); } + fmt::print("{:=^72}\n\n", ""); +} + +void reshapePatchEmbedConv3DWeightForLinear(const mllm::ParameterFile::ptr_t& params, + const mllm::models::qwen2vl::Qwen2VLConfig& cfg) { + const std::string weight_name = "visual.patch_embed.proj.weight"; + if (!params->has(weight_name)) { MLLM_ERROR_EXIT(mllm::ExitCode::kIOError, "Missing {}", weight_name); } + + const int32_t patch_dim = cfg.visual_in_chans * cfg.visual_temporal_patch_size * cfg.visual_patch_size * cfg.visual_patch_size; + auto weight = params->pull(weight_name); + MLLM_RT_ASSERT_EQ(weight.numel(), cfg.visual_embed_dim * patch_dim); + + params->remove(weight_name); + params->push(weight_name, weight.view({cfg.visual_embed_dim, patch_dim})); +} + +template +std::pair traceAndReportVisual(VisualModelT& visual, + const std::string& output_ir, + mllm::Tensor img, + mllm::Tensor visual_embedding_sin, + mllm::Tensor visual_embedding_cos) { + mllm::ir::lowlevel::traceStart(); + mllm::ir::lowlevel::traceComment("visual inputs: img, visual_embedding_sin, visual_embedding_cos"); + auto visual_embeddings = mllm::ir::lowlevel::traceModule(visual, img, visual_embedding_sin, visual_embedding_cos)[0]; + auto visual_ir = mllm::ir::lowlevel::traceStop(); + + fmt::print("visual output shape : [{}, {}]\n", visual_embeddings.shape()[0], visual_embeddings.shape()[1]); + + mllm::redirect(output_ir, [&]() { mllm::print(visual_ir); }); + fmt::print("Raw visual IR dumped to : {}\n", output_ir); + + auto summary = summarizeLinalgOps(visual_ir); + printSummary(summary); + + return {visual_ir, visual_embeddings}; +} + +void compileVisualSegment(mllm::qnn::aot::QnnAOTEnv& qnn_aot_env, + const std::string& qnn_aot_cfg_path, + const mllm::ParameterFile::ptr_t& params, + const mllm::ir::IRContext::ptr_t& visual_ir, + const std::string& qnn_graph_name) { + mllm::ir::PassManager pm(visual_ir); + pm.reg(mllm::qnn::aot::createQnnAOTSimpleLoweringPipeline(&qnn_aot_env, qnn_aot_cfg_path, params, qnn_graph_name)); + if (!pm.run()) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "Visual QNN AOT lowering failed for graph {}.", qnn_graph_name); + } +} + +} // namespace + +MLLM_MAIN({ + auto& help = Argparse::add("-h|--help").help("Show help message"); + auto& model_path = Argparse::add("-m|--model_path").help("Visual-capable .mllm model path.").required(true); + auto& model_version = Argparse::add("-mv|--model_version").help("Model file version: v1/v2.").def("v2"); + auto& tokenizer_path = Argparse::add("-t|--tokenizer").help("tokenizer.json path.").required(true); + auto& config_path = Argparse::add("-c|--config").help("Qwen2-VL config path.").required(true); + auto& image_path = Argparse::add("-i|--image").help("Image path used to derive real visual shapes.").required(true); + auto& prompt = Argparse::add("-p|--prompt").help("Prompt text.").def("describe this picture"); + auto& output_ir = Argparse::add("-o|--output_ir").help("Output visual IR dump path.").def("qwen2vl_visual_raw.mir"); + auto& compile_context = + Argparse::add("--compile_context").help("Lower the traced visual tower to QNN. Save context only if --output_context is set."); + auto& qnn_aot_cfg_files = Argparse::add("-aot_cfg|--aot_config").help("AOT config JSON path."); + auto& qnn_env_path = Argparse::add("-qnn_env|--qnn_env_path") + .def(defaultQnnAOTEnvPath()) + .help("QNN AOT environment path."); + auto& output_context_path = + Argparse::add("--output_context").help("Output QNN visual context path."); + auto& qnn_graph_name = + Argparse::add("--qnn_graph_name").def("visual").help("QNN graph name stored in the visual context."); + auto& linear_patch_embed = Argparse::add("--linear_patch_embed") + .help("Trace an AOT-oriented visual model that rewrites Conv3D patch embedding to Linear."); + auto& aot_rewrite = Argparse::add("--aot_rewrite") + .help("Trace the current AOT rewrite: Linear PatchEmbed plus primitive QuickGELU."); + auto& start_block = + Argparse::add("--start_block").def(0).help("First visual block index to trace when --aot_rewrite is set."); + auto& visual_blocks = + Argparse::add("--visual_blocks").def(-1).help("Only trace the first N visual blocks when --aot_rewrite is set."); + auto& skip_patch_embed = Argparse::add("--skip_patch_embed").help("Skip patch embedding when --aot_rewrite is set."); + auto& skip_merger = Argparse::add("--skip_merger").help("Skip the visual merger when --aot_rewrite is set."); + auto& visual_bundle_context = + Argparse::add("--visual_bundle_context") + .help("Compile the standard 6 visual graphs into one context: patch, four 8-block segments, merger."); + auto& visual_bundle_layout = + Argparse::add("--visual_bundle_layout") + .def("6x8") + .help("Visual bundle layout: 6x8, tail4, tail2, early2 or block1."); + + for (int32_t i = 1; i < argc; ++i) { + std::string arg = argv[i]; + if (arg == "-h" || arg == "--help") { + Argparse::printHelp(); + return 0; + } + } + + Argparse::parse(argc, argv); + + if (help.isSet()) { + Argparse::printHelp(); + return 0; + } + + mllm::ModelFileVersion file_version = mllm::ModelFileVersion::kV2; + if (model_version.get() == "v1") { + file_version = mllm::ModelFileVersion::kV1; + } else if (model_version.get() != "v2") { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "--model_version must be v1 or v2."); + } + + auto cfg = mllm::models::qwen2vl::Qwen2VLConfig(config_path.get()); + auto tokenizer = mllm::models::qwen2vl::Qwen2VLTokenizer(tokenizer_path.get()); + auto params = mllm::load(model_path.get(), file_version); + + auto inputs = tokenizer.convertMessage({.prompt = prompt.get(), .img_file_path = image_path.get()}); + auto img = inputs.at("img"); + auto grid_thw = inputs.at("grid_thw"); + const auto visual_patch_tokens = img.shape()[0]; + const auto patch_flat_dim = img.shape()[1]; + if (aot_rewrite.isSet() && skip_patch_embed.isSet()) { + img = mllm::Tensor::empty({visual_patch_tokens, cfg.visual_embed_dim}, mllm::kFloat32, mllm::kCPU).alloc(); + } + + auto inv_freq = mllm::models::qwen2vl::makeVisualRoPEInvFreq(cfg.visual_embed_dim / cfg.visual_num_heads, 10000.0); + auto pos_ids = mllm::models::qwen2vl::makeVisualRotaryPosEmbIds(grid_thw, cfg.visual_spatial_merge_size); + auto rotary_pos_emb_full = mllm::models::qwen2vl::makeVisualRotaryPosEmbFull(inv_freq, visual_patch_tokens); + auto pos_emb = mllm::models::qwen2vl::makeVisualRotaryPosEmb(rotary_pos_emb_full, pos_ids, grid_thw); + auto [visual_embedding_sin, visual_embedding_cos] = mllm::models::qwen2vl::makeVisualRotarySinCos(pos_emb); + if (aot_rewrite.isSet()) { + const int32_t half_dim = cfg.visual_embed_dim / cfg.visual_num_heads / 2; + visual_embedding_sin = visual_embedding_sin.view({1, -1, 1, half_dim}, true); + visual_embedding_cos = visual_embedding_cos.view({1, -1, 1, half_dim}, true); + } + + fmt::print("Tracing Qwen2-VL visual tower.\n"); + fmt::print("linear patch embedding : {}\n", (linear_patch_embed.isSet() || aot_rewrite.isSet()) ? "enabled" : "disabled"); + fmt::print("primitive QuickGELU : {}\n", aot_rewrite.isSet() ? "enabled" : "disabled"); + fmt::print("start block : {}\n", start_block.get()); + fmt::print("visual blocks : {}\n", visual_blocks.get()); + fmt::print("skip patch embedding : {}\n", skip_patch_embed.isSet() ? "enabled" : "disabled"); + fmt::print("skip merger : {}\n", skip_merger.isSet() ? "enabled" : "disabled"); + fmt::print("QNN graph name : {}\n", qnn_graph_name.get()); + fmt::print("img shape : [{}, {}]\n", visual_patch_tokens, patch_flat_dim); + fmt::print("grid_thw shape : [{}, {}]\n", grid_thw.shape()[0], grid_thw.shape()[1]); + fmt::print("visual sin/cos shape :"); + for (auto dim : visual_embedding_sin.shape()) { fmt::print(" {}", dim); } + fmt::print("\n"); + + mllm::ir::IRContext::ptr_t visual_ir; + const bool should_trace_single_graph = !visual_bundle_context.isSet(); + + if (visual_bundle_context.isSet() && !aot_rewrite.isSet()) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "--visual_bundle_context requires --aot_rewrite."); + } + + if (should_trace_single_graph && aot_rewrite.isSet()) { + reshapePatchEmbedConv3DWeightForLinear(params, cfg); + auto visual = Qwen2VisionTransformerPretrainedModelAOTRewrite("visual", cfg, start_block.get(), visual_blocks.get(), + skip_merger.isSet(), skip_patch_embed.isSet()); + visual.load(params); + visual_ir = traceAndReportVisual(visual, output_ir.get(), img, visual_embedding_sin, visual_embedding_cos).first; + } else if (should_trace_single_graph && linear_patch_embed.isSet()) { + reshapePatchEmbedConv3DWeightForLinear(params, cfg); + auto visual = Qwen2VisionTransformerPretrainedModelLinearPatch("visual", cfg); + visual.load(params); + visual_ir = traceAndReportVisual(visual, output_ir.get(), img, visual_embedding_sin, visual_embedding_cos).first; + } else if (should_trace_single_graph) { + auto visual = mllm::models::qwen2vl::Qwen2VisionTransformerPretrainedModel("visual", cfg); + visual.load(params); + visual_ir = traceAndReportVisual(visual, output_ir.get(), img, visual_embedding_sin, visual_embedding_cos).first; + } + + if (compile_context.isSet()) { + if (!aot_rewrite.isSet()) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "--compile_context currently requires --aot_rewrite."); + } + if (!qnn_aot_cfg_files.isSet()) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "--compile_context requires --aot_config."); + } + + auto qnn_aot_env = + mllm::qnn::aot::QnnAOTEnv(qnn_env_path.get(), mllm::qnn::aot::parseQcomTargetMachineFromJSONFile(qnn_aot_cfg_files.get())); + if (visual_bundle_context.isSet()) { + reshapePatchEmbedConv3DWeightForLinear(params, cfg); + struct Segment { + std::string graph_name; + int32_t start_block; + int32_t visual_blocks; + bool skip_patch_embed; + bool skip_merger; + std::vector input_shape; + std::string ir_name; + }; + std::vector segments; + const std::string bundle_layout = visual_bundle_layout.get(); + if (bundle_layout == "6x8") { + segments = { + {"visual_patch_embed", 0, 0, false, true, {visual_patch_tokens, patch_flat_dim}, "patch_embed"}, + {"visual_blocks_0_8", 0, 8, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_0_8"}, + {"visual_blocks_8_16", 8, 8, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_8_16"}, + {"visual_blocks_16_24", 16, 8, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_16_24"}, + {"visual_blocks_24_32", 24, 8, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_24_32"}, + {"visual_merger", 0, 0, true, false, {visual_patch_tokens, cfg.visual_embed_dim}, "merger"}, + }; + } else if (bundle_layout == "tail4") { + segments = { + {"visual_patch_embed", 0, 0, false, true, {visual_patch_tokens, patch_flat_dim}, "patch_embed"}, + {"visual_blocks_0_8", 0, 8, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_0_8"}, + {"visual_blocks_8_16", 8, 8, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_8_16"}, + {"visual_blocks_16_20", 16, 4, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_16_20"}, + {"visual_blocks_20_24", 20, 4, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_20_24"}, + {"visual_blocks_24_28", 24, 4, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_24_28"}, + {"visual_blocks_28_32", 28, 4, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_28_32"}, + {"visual_merger", 0, 0, true, false, {visual_patch_tokens, cfg.visual_embed_dim}, "merger"}, + }; + } else if (bundle_layout == "tail2") { + segments = { + {"visual_patch_embed", 0, 0, false, true, {visual_patch_tokens, patch_flat_dim}, "patch_embed"}, + {"visual_blocks_0_24", 0, 24, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_0_24"}, + {"visual_blocks_24_26", 24, 2, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_24_26"}, + {"visual_blocks_26_28", 26, 2, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_26_28"}, + {"visual_blocks_28_30", 28, 2, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_28_30"}, + {"visual_blocks_30_32", 30, 2, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_30_32"}, + {"visual_merger", 0, 0, true, false, {visual_patch_tokens, cfg.visual_embed_dim}, "merger"}, + }; + } else if (bundle_layout == "early2") { + segments = { + {"visual_patch_embed", 0, 0, false, true, {visual_patch_tokens, patch_flat_dim}, "patch_embed"}, + {"visual_blocks_0_2", 0, 2, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_0_2"}, + {"visual_blocks_2_4", 2, 2, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_2_4"}, + {"visual_blocks_4_6", 4, 2, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_4_6"}, + {"visual_blocks_6_8", 6, 2, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_6_8"}, + {"visual_blocks_8_16", 8, 8, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_8_16"}, + {"visual_blocks_16_24", 16, 8, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_16_24"}, + {"visual_blocks_24_32", 24, 8, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_24_32"}, + {"visual_merger", 0, 0, true, false, {visual_patch_tokens, cfg.visual_embed_dim}, "merger"}, + }; + } else if (bundle_layout == "block1") { + segments.push_back({"visual_patch_embed", 0, 0, false, true, {visual_patch_tokens, patch_flat_dim}, "patch_embed"}); + for (int32_t i = 0; i < cfg.visual_depth; ++i) { + const auto graph_name = "visual_blocks_" + std::to_string(i) + "_" + std::to_string(i + 1); + const auto ir_name = "blocks_" + std::to_string(i) + "_" + std::to_string(i + 1); + segments.push_back({graph_name, i, 1, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, ir_name}); + } + segments.push_back({"visual_merger", 0, 0, true, false, {visual_patch_tokens, cfg.visual_embed_dim}, "merger"}); + } else { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "--visual_bundle_layout must be 6x8, tail4, tail2, early2 or block1."); + } + + for (const auto& segment : segments) { + fmt::print("\n{:=^72}\n", fmt::format(" Compile {} ", segment.graph_name)); + auto segment_img = mllm::Tensor::empty(segment.input_shape, mllm::kFloat32, mllm::kCPU).alloc(); + auto visual = Qwen2VisionTransformerPretrainedModelAOTRewrite("visual", cfg, segment.start_block, segment.visual_blocks, + segment.skip_merger, segment.skip_patch_embed); + visual.load(params); + auto segment_ir_path = output_ir.get() + "." + segment.ir_name + ".mir"; + auto segment_ir = + traceAndReportVisual(visual, segment_ir_path, segment_img, visual_embedding_sin, visual_embedding_cos).first; + compileVisualSegment(qnn_aot_env, qnn_aot_cfg_files.get(), params, segment_ir, segment.graph_name); + } + } else { + compileVisualSegment(qnn_aot_env, qnn_aot_cfg_files.get(), params, visual_ir, qnn_graph_name.get()); + } + if (output_context_path.isSet()) { + qnn_aot_env.saveContext("context.0", output_context_path.get()); + mllm::print("Visual QNN AOT context saved to " + output_context_path.get()); + } else { + mllm::print("Visual QNN AOT lowering/finalize completed; context was not saved."); + } + } + + mllm::print("Visual trace diagnostic completed."); +}); diff --git a/examples/qwen2vl_qnn_aot/modeling_qwen2vl_qnn_aot.hpp b/examples/qwen2vl_qnn_aot/modeling_qwen2vl_qnn_aot.hpp new file mode 100644 index 000000000..bb2410311 --- /dev/null +++ b/examples/qwen2vl_qnn_aot/modeling_qwen2vl_qnn_aot.hpp @@ -0,0 +1,558 @@ +// Copyright (c) MLLM Team. +// Licensed under the MIT License. + +#pragma once + +#include +#include + +#include "mllm/mllm.hpp" +#include "mllm/nn/Nn.hpp" +#include "mllm/nn/Module.hpp" +#include "mllm/nn/Functional.hpp" +#include "mllm/core/DataTypes.hpp" +#include "mllm/utils/Enumerate.hpp" +#include "mllm/compile/ir/Trace.hpp" +#include "mllm/models/ARGeneration.hpp" +#include "mllm/models/qwen2vl/configuration_qwen2vl.hpp" + +namespace mllm::models::qwen2vl::qnn_aot { + +struct DebugOutputConfig { + bool dump_block_outputs = false; + bool dump_layer0_outputs = false; + bool key_cache_uint16 = false; +}; + +namespace ptq { + +inline Tensor QDQ_CONSTANT(nn::Module* m, Tensor in, const std::string& qdq_name_in_pytorch) { + auto scale_name = qdq_name_in_pytorch + ".scale"; + auto zp_name = qdq_name_in_pytorch + ".zero_point"; + + switch (in.dtype()) { + case kFloat32: + case kUInt16PerTensorAsy: { + auto scale = m->getTopParameterFile()->pull(scale_name); + auto zp = m->getTopParameterFile()->pull(zp_name); + in.attach("scale", scale.impl(), true); + in.attach("zero_point", zp.impl(), true); + break; + } + default: { + MLLM_ERROR_EXIT(ExitCode::kCoreError, "Can't Process dtype={}", nameOfType(in.dtype())); + } + } + return in; +} + +inline Tensor QDQ(nn::Module* m, Tensor in, const std::string& qdq_name_in_pytorch) { + auto scale_name = qdq_name_in_pytorch + ".fake_quant.scale"; + auto zp_name = qdq_name_in_pytorch + ".fake_quant.zero_point"; + if (!m->getModuleName().empty()) { + scale_name = m->getModuleName() + "." + scale_name; + zp_name = m->getModuleName() + "." + zp_name; + } + + switch (in.dtype()) { + case kUInt16PerTensorAsy: + case kFloat32: { + auto scale = m->getTopParameterFile()->pull(scale_name); + auto zp = m->getTopParameterFile()->pull(zp_name); + in.attach("scale", scale.impl(), true); + in.attach("zero_point", zp.impl(), true); + break; + } + default: { + MLLM_ERROR_EXIT(ExitCode::kCoreError, "Can't Process dtype={}", nameOfType(in.dtype())); + } + } + return in; +} + +inline Tensor QDQ_KV(nn::Module* m, Tensor in, const std::string& qdq_name_in_pytorch) { + auto scale_name = m->getModuleName() + "." + qdq_name_in_pytorch + ".fake_quant.scale"; + auto zp_name = m->getModuleName() + "." + qdq_name_in_pytorch + ".fake_quant.zero_point"; + + switch (in.dtype()) { + case kUInt8PerTensorSym: { + auto scale = m->getTopParameterFile()->pull(scale_name); + auto zp = m->getTopParameterFile()->pull(zp_name); + MLLM_RT_ASSERT_EQ(zp.item(), 128); + auto new_zp = Tensor::constant(128, kInt32).setName(zp_name).setMemType(kParamsNormal); + in.attach("scale", scale.impl(), true); + in.attach("zero_point", new_zp.impl(), true); + break; + } + default: { + MLLM_ERROR_EXIT(ExitCode::kCoreError, "Can't Process dtype={}", nameOfType(in.dtype())); + } + } + return in; +} + +inline Tensor QDQ_KV_UInt16Sym(nn::Module* m, Tensor in, const std::string& qdq_name_in_pytorch) { + auto scale_name = m->getModuleName() + "." + qdq_name_in_pytorch + ".fake_quant.scale"; + + switch (in.dtype()) { + case kUInt16PerTensorSym: { + auto scale = m->getTopParameterFile()->pull(scale_name); + in.attach("scale", scale.impl(), true); + break; + } + default: { + MLLM_ERROR_EXIT(ExitCode::kCoreError, "Can't Process dtype={}", nameOfType(in.dtype())); + } + } + return in; +} + +} // namespace ptq + +inline void appendAttentionHeadOutputs(std::vector& outputs, + Tensor x, + const std::string& base_name, + int32_t num_heads) { + for (int32_t h = 0; h < num_heads; ++h) { + outputs.push_back(x.slice({kAll, {h, h + 1}, kAll, kAll}, true).setName(base_name + "_h" + std::to_string(h))); + } +} + +inline Tensor rotateHalf(Tensor x, nn::Module* m, const std::string& qdq_name_in_pytorch) { + auto D = x.size(-1); + auto x1 = x.slice({kAll, kAll, kAll, {kAll, D / 2}}, /*ssa=*/true); + auto x2 = x.slice({kAll, kAll, kAll, {D / 2, kAll}}, /*ssa=*/true); + return nn::functional::concat({ptq::QDQ(m, -x2, qdq_name_in_pytorch), x1}, -1); +} + +using vi32 = std::vector; +#define CONV2D_PROPERTY_NO_BIAS vi32{1, 1}, vi32{1, 1}, vi32{0, 0}, vi32{1, 1}, false, aops::Conv2DOpImplType::kQNN_LPBQ_w4a16o16_G32 + +inline Tensor makeQuantizedBias(nn::Module* m, const std::string& bias_name, const std::string& qdq_name_in_pytorch, + int32_t channels) { + auto prefix = m->getModuleName(); + auto full_bias_name = prefix.empty() ? bias_name : prefix + "." + bias_name; + auto scale_name = prefix.empty() ? qdq_name_in_pytorch + ".fake_quant.scale" + : prefix + "." + qdq_name_in_pytorch + ".fake_quant.scale"; + auto zp_name = prefix.empty() ? qdq_name_in_pytorch + ".fake_quant.zero_point" + : prefix + "." + qdq_name_in_pytorch + ".fake_quant.zero_point"; + + auto params = m->getTopParameterFile(); + auto bias = params->pull(full_bias_name); + auto scale = params->pull(scale_name); + auto zero_point = params->pull(zp_name); + + MLLM_RT_ASSERT_EQ(bias.numel(), channels); + MLLM_RT_ASSERT_EQ(scale.numel(), 1); + MLLM_RT_ASSERT_EQ(zero_point.numel(), 1); + + const auto scale_value = scale.item(); + const auto zero_point_value = zero_point.item(); + auto quant_bias = Tensor::empty({1, 1, 1, channels}, kUInt16).alloc(); + + for (int32_t i = 0; i < channels; ++i) { + const auto quantized = + static_cast(std::lround(bias.ptr()[i] / scale_value)) + zero_point_value; + quant_bias.ptr()[i] = static_cast(std::clamp(quantized, 0, 65535)); + } + + quant_bias = quant_bias.__unsafeSetDType(kUInt16PerTensorAsy); + quant_bias.setName(full_bias_name + ".q_uint16"); + quant_bias.setMemType(kParamsNormal); + quant_bias.attach("scale", scale.impl(), true); + quant_bias.attach("zero_point", zero_point.impl(), true); + return quant_bias; +} + +class Qwen2VLMLP final : public nn::Module { + nn::Conv2D gate_proj_; + nn::Conv2D up_proj_; + nn::Conv2D down_proj_; + nn::SiLU silu_; + int hidden_size_; + int intermediate_size_; + bool dump_layer0_outputs_ = false; + + public: + Qwen2VLMLP() = default; + Qwen2VLMLP(const std::string& name, const Qwen2VLConfig& cfg, bool dump_layer0_outputs = false) : nn::Module(name) { + gate_proj_ = reg("gate_proj", cfg.hidden_size, cfg.intermediate_size, CONV2D_PROPERTY_NO_BIAS); + silu_ = reg("act"); + up_proj_ = reg("up_proj", cfg.hidden_size, cfg.intermediate_size, CONV2D_PROPERTY_NO_BIAS); + down_proj_ = reg("down_proj", cfg.intermediate_size, cfg.hidden_size, CONV2D_PROPERTY_NO_BIAS); + hidden_size_ = cfg.hidden_size; + intermediate_size_ = cfg.intermediate_size; + dump_layer0_outputs_ = dump_layer0_outputs; + } + + std::vector forward(const std::vector& inputs, const std::vector& args) override { + auto x = inputs[0]; + x = ptq::QDQ(this, x, "up_proj_input_qdq"); + x = x.view({1, 1, -1, hidden_size_}, true); + + auto up_result = ptq::QDQ(this, up_proj_(x), "up_proj_output_qdq").view({1, -1, intermediate_size_}, true); + auto gate_result = ptq::QDQ(this, gate_proj_(x), "gate_proj_output_qdq").view({1, -1, intermediate_size_}, true); + auto gate_proj_result = gate_result; + + gate_result = ptq::QDQ(this, gate_result * ptq::QDQ(this, nn::functional::sigmoid(gate_result), "sigmoid_output_qdq"), + "act_output_qdq"); + auto gate_act_result = gate_result; + + auto o = ptq::QDQ(this, gate_result * up_result, "down_proj_input_qdq"); + auto down_proj_input = o; + o = o.view({1, 1, -1, intermediate_size_}, true); + o = down_proj_(o).view({1, -1, hidden_size_}, true); + + if (!dump_layer0_outputs_) { return {o}; } + return {o, + up_result.setName("layer0_mlp_up_proj_out"), + gate_proj_result.setName("layer0_mlp_gate_proj_out"), + gate_act_result.setName("layer0_mlp_gate_act_out"), + down_proj_input.setName("layer0_mlp_down_proj_input")}; + } +}; + +class Qwen2VLAttention final : public nn::Module { + nn::Conv2D q_proj_; + nn::Conv2D k_proj_; + nn::Conv2D v_proj_; + nn::Conv2D o_proj_; + nn::CausalMask mask_; + nn::Softmax softmax_; + + int hidden_size_; + int head_dim_; + int num_attention_heads_; + int num_key_value_heads_; + int num_key_value_groups_; + float scale_; + bool dump_layer0_outputs_ = false; + bool key_cache_uint16_ = false; + + public: + Qwen2VLAttention() = default; + + Qwen2VLAttention(const std::string& name, + const Qwen2VLConfig& cfg, + bool dump_layer0_outputs = false, + bool key_cache_uint16 = false) + : nn::Module(name) { + hidden_size_ = cfg.hidden_size; + num_attention_heads_ = cfg.num_attention_heads; + num_key_value_heads_ = cfg.num_key_value_heads; + head_dim_ = cfg.hidden_size / cfg.num_attention_heads; + num_key_value_groups_ = num_attention_heads_ / num_key_value_heads_; + scale_ = 1.f / sqrtf(static_cast(head_dim_)); + dump_layer0_outputs_ = dump_layer0_outputs; + key_cache_uint16_ = key_cache_uint16; + + q_proj_ = reg("q_proj", hidden_size_, head_dim_ * num_attention_heads_, CONV2D_PROPERTY_NO_BIAS); + k_proj_ = reg("k_proj", hidden_size_, head_dim_ * num_key_value_heads_, CONV2D_PROPERTY_NO_BIAS); + v_proj_ = reg("v_proj", hidden_size_, head_dim_ * num_key_value_heads_, CONV2D_PROPERTY_NO_BIAS); + o_proj_ = reg("o_proj", head_dim_ * num_attention_heads_, hidden_size_, CONV2D_PROPERTY_NO_BIAS); + + mask_ = reg("mask"); + softmax_ = reg("softmax", -1); + } + + std::vector forward(const std::vector& inputs, const std::vector& args) override { + auto hidden_states = inputs[0]; + auto llm_embedding_sin = inputs[1]; + auto llm_embedding_cos = inputs[2]; + auto causal_mask = inputs[3]; + auto past_key = inputs[4]; + auto past_value = inputs[5]; + + hidden_states = ptq::QDQ(this, hidden_states, "q_proj_input_qdq"); + hidden_states = hidden_states.view({1, 1, -1, hidden_size_}, true); + + auto query_states = ptq::QDQ(this, q_proj_(hidden_states), "q_proj_output_qdq"); + auto key_states = ptq::QDQ(this, k_proj_(hidden_states), "k_proj_output_qdq"); + auto value_states = ptq::QDQ(this, v_proj_(hidden_states), "v_cast_to_int16_qdq"); + + query_states = + ptq::QDQ(this, + query_states + + makeQuantizedBias(this, "q_proj.bias", "q_proj_output_qdq", head_dim_ * num_attention_heads_), + "q_proj_output_qdq"); + key_states = + ptq::QDQ(this, + key_states + + makeQuantizedBias(this, "k_proj.bias", "k_proj_output_qdq", head_dim_ * num_key_value_heads_), + "k_proj_output_qdq"); + value_states = + ptq::QDQ(this, + value_states + + makeQuantizedBias(this, "v_proj.bias", "v_cast_to_int16_qdq", head_dim_ * num_key_value_heads_), + "v_cast_to_int16_qdq"); + + auto query_states_flat = query_states.view({1, -1, num_attention_heads_ * head_dim_}, true); + auto key_states_flat = key_states.view({1, -1, num_key_value_heads_ * head_dim_}, true); + + auto value_states_flat = value_states.view({1, -1, num_key_value_heads_ * head_dim_}, true); + + query_states = query_states.view({1, -1, num_attention_heads_, head_dim_}, /*ssa=*/true).transpose(1, 2); + key_states = key_states.view({1, -1, num_key_value_heads_, head_dim_}, /*ssa=*/true).transpose(1, 2); + value_states = value_states_flat.view({1, -1, num_key_value_heads_, head_dim_}, /*ssa=*/true).transpose(1, 2); + + auto cos = llm_embedding_cos.unsqueeze(1, true); + auto sin = llm_embedding_sin.unsqueeze(1, true); + query_states = + ptq::QDQ(this, + ptq::QDQ(this, query_states * cos, "q_rope_mul_0_output_qdq") + + ptq::QDQ(this, rotateHalf(query_states, this, "q_rope_neg_half_qdq") * sin, "q_rope_mul_1_output_qdq"), + "q_rope_add_0_output_qdq"); + key_states = + ptq::QDQ(this, + ptq::QDQ(this, key_states * cos, "k_rope_mul_0_output_qdq") + + ptq::QDQ(this, rotateHalf(key_states, this, "k_rope_neg_half_qdq") * sin, "k_rope_mul_1_output_qdq"), + "k_rope_add_0_output_qdq"); + auto query_states_rope_flat = query_states.transpose(1, 2).view({1, -1, num_attention_heads_ * head_dim_}, true); + auto key_states_rope_flat = key_states.transpose(1, 2).view({1, -1, num_key_value_heads_ * head_dim_}, true); + + if (key_cache_uint16_) { + key_states = key_states.to(kFloat32); + key_states = key_states.to(kUInt16PerTensorSym); + key_states = ptq::QDQ_KV_UInt16Sym(this, key_states, "k_rope_add_0_output_qdq"); + key_states = key_states.transpose(2, 3); + } else { + key_states = key_states.to(kFloat32); + key_states = key_states.to(kUInt8PerTensorSym); + key_states = ptq::QDQ_KV(this, key_states, "k_cast_to_int8_qdq"); + key_states = key_states.transpose(2, 3); + } + + value_states = value_states.to(kFloat32); + value_states = value_states.to(kUInt8PerTensorSym); + value_states = ptq::QDQ_KV(this, value_states, "v_cast_to_int8_qdq"); + auto value_cache_states_flat = + value_states.transpose(1, 2).view({1, -1, num_key_value_heads_ * head_dim_}, true); + + auto kh = nn::functional::concat({past_key, key_states}, -1); + auto vh = nn::functional::concat({past_value, value_states}, 2); + + kh = kh.repeat(num_key_value_groups_, 1); + vh = vh.repeat(num_key_value_groups_, 1); + + auto attn = ptq::QDQ(this, nn::functional::matmul(query_states, kh), "qk_matmul_output_qdq"); + auto qk_matmul_out = attn; + auto scale = ptq::QDQ(this, Tensor::constant(scale_, kFloat32), "scaling_qdq"); + attn = ptq::QDQ(this, attn.mulConstant(scale), "mul_0_output_qdq"); + auto qk_scaled_out = attn; + + auto attn_min = ptq::QDQ(this, attn.min(-1, true), "reduce_min_output_qdq"); + auto minus_value = ptq::QDQ(this, Tensor::constant(-20, kFloat32), "neg_20_qdq"); + auto attn_vv = ptq::QDQ(this, attn_min.addConstant(minus_value), "minus_0_output_qdq"); + auto zero_constant = ptq::QDQ_CONSTANT(this, Tensor::constant(0.f, kFloat32), "constant_zero"); + attn = nn::functional::where(causal_mask.equalConstant(zero_constant), attn, attn_vv); + attn = ptq::QDQ(this, attn, "where_attn_qdq"); + auto qk_masked_out = attn; + attn = ptq::QDQ(this, nn::functional::softmax(attn, -1), "softmax_output_qdq"); + auto softmax_out = attn; + auto attn_value_states = ptq::QDQ(this, nn::functional::matmul(attn, vh), "attn_value_matmul_output_qdq"); + auto y = attn_value_states.transpose(1, 2).view({1, 1, -1, num_attention_heads_ * head_dim_}, /*ssa=*/true); + auto attn_value_states_flat = y.view({1, -1, num_attention_heads_ * head_dim_}, true); + y = o_proj_(y).view({1, -1, hidden_size_}, true); + + if (!dump_layer0_outputs_) { return {y, key_states, value_states}; } + + auto ret = std::vector{y, + key_states, + value_states, + query_states_flat.setName("layer0_q_proj_out_flat"), + key_states_flat.setName("layer0_k_proj_out_flat"), + value_states_flat.setName("layer0_v_proj_out_flat"), + value_cache_states_flat.setName("layer0_v_cache_out_flat"), + query_states_rope_flat.setName("layer0_q_rope_out_flat"), + key_states_rope_flat.setName("layer0_k_rope_out_flat")}; + appendAttentionHeadOutputs(ret, qk_matmul_out, "layer0_qk_matmul_out_flat", num_attention_heads_); + appendAttentionHeadOutputs(ret, qk_scaled_out, "layer0_qk_scaled_out_flat", num_attention_heads_); + appendAttentionHeadOutputs(ret, qk_masked_out, "layer0_qk_masked_out_flat", num_attention_heads_); + appendAttentionHeadOutputs(ret, softmax_out, "layer0_softmax_out_flat", num_attention_heads_); + ret.push_back(attn_value_states_flat.setName("layer0_attn_value_out_flat")); + ret.push_back(y.setName("layer0_o_proj_out")); + return ret; + } + + int layer_idx_; +}; + +class Qwen2VLDecoder final : public nn::Module { + public: + int layer_idx_; + bool dump_layer0_outputs_ = false; + Qwen2VLAttention self_attn_; + Qwen2VLMLP mlp_; + nn::RMSNorm input_layer_norm_; + nn::RMSNorm post_attention_layer_norm_; + + Qwen2VLDecoder() = default; + + Qwen2VLDecoder(const std::string& name, const Qwen2VLConfig& cfg, DebugOutputConfig debug_outputs, int layer_idx) : nn::Module(name) { + layer_idx_ = layer_idx; + dump_layer0_outputs_ = debug_outputs.dump_layer0_outputs && layer_idx == 0; + self_attn_ = reg("self_attn", cfg, dump_layer0_outputs_, debug_outputs.key_cache_uint16); + mlp_ = reg("mlp", cfg, dump_layer0_outputs_); + input_layer_norm_ = reg("input_layernorm", cfg.rms_norm_eps); + post_attention_layer_norm_ = reg("post_attention_layernorm", cfg.rms_norm_eps); + } + + std::vector forward(const std::vector& inputs, const std::vector& args) override { + auto llm_embedding_sin = inputs[1]; + auto llm_embedding_cos = inputs[2]; + auto causal_mask = inputs[3]; + auto past_key = inputs[4]; + auto past_value = inputs[5]; + + auto hidden_states = inputs[0]; + if (layer_idx_ != 0) { hidden_states = ptq::QDQ(this, hidden_states, "input_layernorm_input_qdq"); } + auto residual = hidden_states; + hidden_states = input_layer_norm_(hidden_states); + auto attn_out = self_attn_(hidden_states, llm_embedding_sin, llm_embedding_cos, causal_mask, past_key, past_value); + auto layer0_input_layernorm_out = hidden_states; + hidden_states = attn_out[0]; + hidden_states = ptq::QDQ(this, residual + ptq::QDQ(this, hidden_states, "add_0_lhs_input_qdq"), "add_0_output_qdq"); + residual = hidden_states; + hidden_states = post_attention_layer_norm_(hidden_states); + auto layer0_post_attention_layernorm_out = hidden_states; + auto mlp_out = mlp_(hidden_states); + hidden_states = mlp_out[0]; + auto layer0_mlp_down_proj_out = hidden_states; + hidden_states = residual + ptq::QDQ(this, hidden_states, "add_1_lhs_input_qdq"); + if (!dump_layer0_outputs_) { return {hidden_states, attn_out[1], attn_out[2]}; } + + auto ret = std::vector{hidden_states, attn_out[1], attn_out[2], + layer0_input_layernorm_out.setName("layer0_input_layernorm_out")}; + for (size_t i = 3; i < attn_out.size(); ++i) { ret.push_back(attn_out[i]); } + ret.push_back(layer0_post_attention_layernorm_out.setName("layer0_post_attention_layernorm_out")); + ret.push_back(mlp_out[1]); + ret.push_back(mlp_out[2]); + ret.push_back(mlp_out[3]); + ret.push_back(mlp_out[4]); + ret.push_back(layer0_mlp_down_proj_out.setName("layer0_mlp_down_proj_out")); + ret.push_back(hidden_states.setName("layer0_block_out")); + return ret; + } +}; + +class Qwen2VLText final : public nn::Module { + nn::ModuleListWithIdx decode_blocks_; + nn::RMSNorm norm_; + int32_t num_hidden_layers_; + int32_t hidden_size_; + DebugOutputConfig debug_outputs_; + + public: + Qwen2VLText() = default; + + Qwen2VLText(const std::string& name, const Qwen2VLConfig& cfg, DebugOutputConfig debug_outputs = {}) : nn::Module(name) { + num_hidden_layers_ = cfg.num_hidden_layers; + hidden_size_ = cfg.hidden_size; + debug_outputs_ = debug_outputs; + decode_blocks_ = reg>("layers", cfg.num_hidden_layers, cfg, debug_outputs_); + for (auto [idx, b] : enumerate(decode_blocks_.list())) { b.self_attn_.layer_idx_ = idx; } + norm_ = reg("norm", cfg.rms_norm_eps); + } + + std::vector forward(const std::vector& inputs, const std::vector& args) override { + auto& blocks = decode_blocks_.list(); + auto x = inputs[0]; + auto llm_embedding_sin = inputs[1]; + auto llm_embedding_cos = inputs[2]; + auto causal_mask = inputs[3]; + + std::vector keys; + std::vector values; + std::vector block_outputs; + std::vector layer0_outputs; + for (auto [index, block] : enumerate(blocks)) { + auto pk = inputs[4 + index]; + auto pv = inputs[4 + index + num_hidden_layers_]; + auto out = block(x, llm_embedding_sin, llm_embedding_cos, causal_mask, pk, pv); + x = out[0]; + if (debug_outputs_.dump_block_outputs) { block_outputs.push_back(x.setName("block_out_" + std::to_string(index))); } + if (debug_outputs_.dump_layer0_outputs && index == 0) { + for (size_t i = 3; i < out.size(); ++i) { layer0_outputs.push_back(out[i]); } + } + keys.push_back(out[1]); + values.push_back(out[2]); + } + + x = norm_(ptq::QDQ(this, x, "norm_input_qdq")); + x = x.view({1, 1, -1, hidden_size_}, true); + + auto ret = std::vector{x}; + for (const auto& item : keys) { ret.push_back(item); } + for (const auto& item : values) { ret.push_back(item); } + for (const auto& item : block_outputs) { ret.push_back(item); } + for (const auto& item : layer0_outputs) { ret.push_back(item); } + return ret; + } +}; + +class Qwen2VLForCausalLM : public ARGeneration, public nn::Module { + public: + explicit Qwen2VLForCausalLM(const Qwen2VLConfig& cfg, DebugOutputConfig debug_outputs = {}) + : cfg_(cfg), debug_outputs_(debug_outputs) { + eos_token_id_ = cfg.end_of_text_token_id; + max_length_ = cfg.max_cache_length; + tie_word_embeddings_ = cfg.tie_word_embeddings; + + llm_ = reg("model", cfg, debug_outputs_); + if (cfg.tie_word_embeddings) { + lm_head_ = reg("lm_head", cfg.hidden_size, cfg.vocab_size, CONV2D_PROPERTY_NO_BIAS); + } + } + + IROutput trace(const ARGenerationOutputPast& input, const ARGenerationArgs& args) override { + auto input_embeddings = input.at("input_embeddings"); + auto llm_embedding_sin = input.at("llm_embedding_sin"); + auto llm_embedding_cos = input.at("llm_embedding_cos"); + auto causal_mask = input.at("causal_mask"); + + std::vector kv_caches; + for (int i = 0; i < cfg_.num_hidden_layers; ++i) { + auto past_key_name = "past_key_" + std::to_string(i); + if (!input.count(past_key_name)) { throw std::runtime_error("Missing KV cache: " + past_key_name); } + kv_caches.push_back(input.at(past_key_name)); + } + for (int i = 0; i < cfg_.num_hidden_layers; ++i) { + auto past_value_name = "past_value_" + std::to_string(i); + if (!input.count(past_value_name)) { throw std::runtime_error("Missing KV cache: " + past_value_name); } + kv_caches.push_back(input.at(past_value_name)); + } + + ir::lowlevel::traceStart(); + std::vector llm_inputs = {input_embeddings, llm_embedding_sin, llm_embedding_cos, causal_mask}; + llm_inputs.insert(llm_inputs.end(), kv_caches.begin(), kv_caches.end()); + + auto llm_out = llm_(llm_inputs); + auto logits = llm_out[0]; + const auto base_output_count = 1 + 2 * cfg_.num_hidden_layers; + size_t expected_output_count = base_output_count; + if (debug_outputs_.dump_block_outputs) { expected_output_count += cfg_.num_hidden_layers; } + if (debug_outputs_.dump_layer0_outputs) { expected_output_count += 16 + 4 * cfg_.num_attention_heads; } + if (debug_outputs_.dump_block_outputs || debug_outputs_.dump_layer0_outputs) { + MLLM_RT_ASSERT_EQ(llm_out.size(), expected_output_count); + } + + logits = lm_head_(ptq::QDQ(this, logits, "lm_head_input_qdq")); + logits = ptq::QDQ(this, logits, "lm_head_output_qdq"); + auto llm_ir = ir::lowlevel::traceStop(); + + return {{"model", llm_ir}}; + } + + ARGenerationOutputPast forward(const ARGenerationOutputPast& input, const ARGenerationArgs& args) override { return {}; } + + private: + const Qwen2VLConfig& cfg_; + Qwen2VLText llm_; + nn::Conv2D lm_head_; + bool tie_word_embeddings_; + DebugOutputConfig debug_outputs_; +}; + +#undef CONV2D_PROPERTY_NO_BIAS + +} // namespace mllm::models::qwen2vl::qnn_aot diff --git a/examples/qwen2vl_qnn_aot/visual_aot_helpers.hpp b/examples/qwen2vl_qnn_aot/visual_aot_helpers.hpp new file mode 100644 index 000000000..c3dd69246 --- /dev/null +++ b/examples/qwen2vl_qnn_aot/visual_aot_helpers.hpp @@ -0,0 +1,359 @@ +// Copyright (c) MLLM Team. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace qwen2vl_qnn_aot { + +class PatchEmbedLinear final : public mllm::nn::Module { + int32_t patch_dim_ = 0; + int32_t embed_dim_ = 0; + + mllm::nn::Linear proj_; + + public: + PatchEmbedLinear() = default; + + PatchEmbedLinear(const std::string& name, const mllm::models::qwen2vl::Qwen2VLConfig& cfg) : mllm::nn::Module(name) { + patch_dim_ = cfg.visual_in_chans * cfg.visual_temporal_patch_size * cfg.visual_patch_size * cfg.visual_patch_size; + embed_dim_ = cfg.visual_embed_dim; + proj_ = reg("proj", patch_dim_, embed_dim_, false, mllm::aops::LinearImplTypes::kDefault); + } + + std::vector forward(const std::vector& inputs, + const std::vector& /*args*/) override { + auto hidden_states = inputs[0]; + hidden_states = hidden_states.view({-1, patch_dim_}, true); + hidden_states = proj_(hidden_states).view({-1, embed_dim_}, true); + return {hidden_states}; + } +}; + +class VisionMlpPrimitiveQuickGELU final : public mllm::nn::Module { + int32_t dim_ = 0; + int32_t hidden_dim_ = 0; + + mllm::nn::Linear fc_1_; + mllm::nn::Linear fc_2_; + + public: + VisionMlpPrimitiveQuickGELU() = default; + + VisionMlpPrimitiveQuickGELU(const std::string& name, const mllm::models::qwen2vl::Qwen2VLConfig& cfg) + : mllm::nn::Module(name) { + dim_ = cfg.visual_embed_dim; + hidden_dim_ = cfg.visual_embed_dim * cfg.visual_mlp_ratio; + fc_1_ = reg("fc1", dim_, hidden_dim_, true, cfg.linear_impl_type); + fc_2_ = reg("fc2", hidden_dim_, dim_, true, cfg.linear_impl_type); + } + + std::vector forward(const std::vector& inputs, + const std::vector& /*args*/) override { + auto x = fc_1_(inputs[0]); + x = x * mllm::nn::functional::sigmoid(x * 1.702f); + return {fc_2_(x)}; + } +}; + +class VisionAttentionMaskedAOTRewrite final : public mllm::nn::Module { + int32_t dim_ = 0; + int32_t num_heads_ = 0; + int32_t head_dim_ = 0; + + mllm::nn::Linear qkv_; + mllm::nn::Linear proj_; + mllm::nn::Softmax softmax_; + + public: + VisionAttentionMaskedAOTRewrite() = default; + + VisionAttentionMaskedAOTRewrite(const std::string& name, const mllm::models::qwen2vl::Qwen2VLConfig& cfg) + : mllm::nn::Module(name) { + dim_ = cfg.visual_embed_dim; + num_heads_ = cfg.visual_num_heads; + head_dim_ = dim_ / num_heads_; + + qkv_ = reg("qkv", dim_, dim_ * 3, true, cfg.linear_impl_type); + proj_ = reg("proj", dim_, dim_, true, cfg.linear_impl_type); + softmax_ = reg("softmax", -1); + } + + mllm::Tensor applyVisionRoPEPrimitive(mllm::Tensor x, mllm::Tensor visual_embedding_sin, mllm::Tensor visual_embedding_cos) { + const int32_t half_dim = head_dim_ / 2; + auto x1 = x.slice({mllm::kAll, mllm::kAll, mllm::kAll, {mllm::kAll, half_dim}}, true); + auto x2 = x.slice({mllm::kAll, mllm::kAll, mllm::kAll, {half_dim, mllm::kAll}}, true); + auto sin = visual_embedding_sin; + auto cos = visual_embedding_cos; + if (sin.rank() == 2) { + sin = sin.view({1, -1, 1, half_dim}, true); + cos = cos.view({1, -1, 1, half_dim}, true); + } else { + MLLM_RT_ASSERT_EQ(sin.rank(), 4); + MLLM_RT_ASSERT_EQ(cos.rank(), 4); + } + + auto y1 = x1 * cos + (-(x2 * sin)); + auto y2 = x1 * sin + x2 * cos; + return mllm::nn::functional::concat({y1, y2}, -1); + } + + std::vector forward(const std::vector& inputs, + const std::vector& /*args*/) override { + auto hidden_states = inputs[0]; + auto visual_embedding_sin = inputs[1]; + auto visual_embedding_cos = inputs[2]; + auto attention_mask = inputs.size() > 3 ? inputs[3] : mllm::Tensor::nil(); + + auto qkv_states = qkv_(hidden_states).view({-1, 3, num_heads_, head_dim_}, true); + auto query_states = qkv_states.slice({mllm::kAll, {0, 1}, mllm::kAll, mllm::kAll}, true).transpose(0, 1); + auto key_states = qkv_states.slice({mllm::kAll, {1, 2}, mllm::kAll, mllm::kAll}, true).transpose(0, 1); + auto value_states = qkv_states.slice({mllm::kAll, {2, 3}, mllm::kAll, mllm::kAll}, true).transpose(0, 1); + + query_states = applyVisionRoPEPrimitive(query_states, visual_embedding_sin, visual_embedding_cos); + key_states = applyVisionRoPEPrimitive(key_states, visual_embedding_sin, visual_embedding_cos); + + query_states = query_states.transpose(1, 2); + key_states = key_states.transpose(1, 2); + value_states = value_states.transpose(1, 2); + + auto attn = mllm::nn::functional::matmul(query_states, key_states, false, true) * (1.f / std::sqrt(static_cast(head_dim_))); + if (!attention_mask.isNil()) { attn = attn + attention_mask; } + attn = softmax_(attn); + + auto attn_output = mllm::nn::functional::matmul(attn, value_states); + attn_output = attn_output.transpose(1, 2).view({-1, dim_}, true); + return {proj_(attn_output)}; + } +}; + +class Qwen2VLVisionBlockAOTRewrite final : public mllm::nn::Module { + mllm::nn::LayerNorm norm1_; + mllm::nn::LayerNorm norm2_; + + VisionAttentionMaskedAOTRewrite attn_; + VisionMlpPrimitiveQuickGELU mlp_; + + public: + Qwen2VLVisionBlockAOTRewrite() = default; + + Qwen2VLVisionBlockAOTRewrite(const std::string& name, const mllm::models::qwen2vl::Qwen2VLConfig& cfg) + : mllm::nn::Module(name) { + norm1_ = reg("norm1", std::vector{cfg.visual_embed_dim}, true, true, 1e-6); + norm2_ = reg("norm2", std::vector{cfg.visual_embed_dim}, true, true, 1e-6); + attn_ = reg("attn", cfg); + mlp_ = reg("mlp", cfg); + } + + std::vector forward(const std::vector& inputs, + const std::vector& /*args*/) override { + auto hidden_states = inputs[0]; + auto visual_embedding_sin = inputs[1]; + auto visual_embedding_cos = inputs[2]; + auto attention_mask = inputs.size() > 3 ? inputs[3] : mllm::Tensor::nil(); + + if (attention_mask.isNil()) { + hidden_states = hidden_states + attn_(norm1_(hidden_states), visual_embedding_sin, visual_embedding_cos)[0]; + } else { + hidden_states = + hidden_states + attn_(norm1_(hidden_states), visual_embedding_sin, visual_embedding_cos, attention_mask)[0]; + } + hidden_states = hidden_states + mlp_(norm2_(hidden_states))[0]; + return {hidden_states}; + } +}; + +class Qwen2VisionTransformerPretrainedModelAOTRewrite final : public mllm::nn::Module { + PatchEmbedLinear patch_embed_; + mllm::models::qwen2vl::PatchMerger patch_merger_; + mllm::nn::ModuleList blocks_; + int32_t start_block_ = 0; + int32_t active_blocks_ = -1; + bool skip_merger_ = false; + bool skip_patch_embed_ = false; + + public: + Qwen2VisionTransformerPretrainedModelAOTRewrite() = default; + + Qwen2VisionTransformerPretrainedModelAOTRewrite(const std::string& name, + const mllm::models::qwen2vl::Qwen2VLConfig& cfg, + int32_t start_block = 0, + int32_t active_blocks = -1, + bool skip_merger = false, + bool skip_patch_embed = false) + : mllm::nn::Module(name) { + start_block_ = start_block; + active_blocks_ = active_blocks; + skip_merger_ = skip_merger; + skip_patch_embed_ = skip_patch_embed; + patch_embed_ = reg("patch_embed", cfg); + patch_merger_ = reg("merger", cfg); + blocks_ = reg>("blocks", cfg.visual_depth, cfg); + } + + std::vector forward(const std::vector& inputs, + const std::vector& /*args*/) override { + auto hidden_states = inputs[0]; + auto embedding_sin = inputs[1]; + auto embedding_cos = inputs[2]; + auto attention_mask = inputs.size() > 3 ? inputs[3] : mllm::Tensor::nil(); + + if (!skip_patch_embed_) { hidden_states = patch_embed_(hidden_states)[0]; } + auto num_blocks = active_blocks_ < 0 ? static_cast(blocks_.list().size()) : active_blocks_; + MLLM_RT_ASSERT(start_block_ >= 0); + MLLM_RT_ASSERT(num_blocks >= 0); + MLLM_RT_ASSERT(start_block_ + num_blocks <= static_cast(blocks_.list().size())); + for (int32_t i = 0; i < num_blocks; ++i) { + if (attention_mask.isNil()) { + hidden_states = blocks_.list()[start_block_ + i](hidden_states, embedding_sin, embedding_cos)[0]; + } else { + hidden_states = blocks_.list()[start_block_ + i](hidden_states, embedding_sin, embedding_cos, attention_mask)[0]; + } + } + if (!skip_merger_) { hidden_states = patch_merger_(hidden_states)[0]; } + + return {hidden_states}; + } +}; + +struct VisualSegment { + std::string graph_name; + int32_t start_block = 0; + int32_t visual_blocks = 0; + bool skip_patch_embed = false; + bool skip_merger = false; + std::vector input_shape; + std::string ir_name; +}; + +struct VisualBucketGrid { + int32_t grid_h = 0; + int32_t grid_w = 0; + + [[nodiscard]] int32_t patchTokens() const { return grid_h * grid_w; } +}; + +inline std::vector parseVisualBucketGrids(const std::string& text) { + std::vector buckets; + if (text.empty()) { return buckets; } + + std::stringstream ss(text); + std::string item; + while (std::getline(ss, item, ',')) { + item.erase(std::remove_if(item.begin(), item.end(), [](unsigned char c) { return std::isspace(c); }), item.end()); + if (item.empty()) { continue; } + + auto sep = item.find('x'); + if (sep == std::string::npos) { sep = item.find('X'); } + if (sep == std::string::npos) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "Invalid visual bucket grid '{}', expected HxW.", item); + } + + const int32_t grid_h = std::stoi(item.substr(0, sep)); + const int32_t grid_w = std::stoi(item.substr(sep + 1)); + if (grid_h <= 0 || grid_w <= 0 || grid_h % 2 != 0 || grid_w % 2 != 0) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, + "Invalid visual bucket grid '{}': grid_h/grid_w must be positive even numbers.", + item); + } + buckets.push_back({grid_h, grid_w}); + } + return buckets; +} + +inline std::vector uniqueVisualBucketPatchTokens(const std::vector& buckets) { + std::vector tokens; + std::unordered_set seen; + for (const auto& bucket : buckets) { + const int32_t patch_tokens = bucket.patchTokens(); + if (seen.insert(patch_tokens).second) { tokens.push_back(patch_tokens); } + } + std::sort(tokens.begin(), tokens.end()); + return tokens; +} + +inline std::string visualGraphSuffixForPatchTokens(int32_t patch_tokens) { return "_s" + std::to_string(patch_tokens); } + +inline void reshapePatchEmbedConv3DWeightForLinear(const mllm::ParameterFile::ptr_t& params, + const mllm::models::qwen2vl::Qwen2VLConfig& cfg) { + const std::string weight_name = "visual.patch_embed.proj.weight"; + if (!params->has(weight_name)) { MLLM_ERROR_EXIT(mllm::ExitCode::kIOError, "Missing {}", weight_name); } + + const int32_t patch_dim = cfg.visual_in_chans * cfg.visual_temporal_patch_size * cfg.visual_patch_size * cfg.visual_patch_size; + auto weight = params->pull(weight_name); + MLLM_RT_ASSERT_EQ(weight.numel(), cfg.visual_embed_dim * patch_dim); + + params->remove(weight_name); + params->push(weight_name, weight.view({cfg.visual_embed_dim, patch_dim})); +} + +inline std::vector makeVisualBundleSegments(const std::string& layout, + int32_t visual_depth, + int32_t visual_patch_tokens, + int32_t patch_flat_dim, + const mllm::models::qwen2vl::Qwen2VLConfig& cfg, + const std::string& graph_suffix = "") { + std::vector segments; + if (layout == "single") { + segments = { + {"visual_full" + graph_suffix, 0, visual_depth, false, false, {visual_patch_tokens, patch_flat_dim}, "full" + graph_suffix}, + }; + } else if (layout == "6x8") { + segments = { + {"visual_patch_embed" + graph_suffix, 0, 0, false, true, {visual_patch_tokens, patch_flat_dim}, "patch_embed" + graph_suffix}, + {"visual_blocks_0_8" + graph_suffix, 0, 8, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_0_8" + graph_suffix}, + {"visual_blocks_8_16" + graph_suffix, 8, 8, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_8_16" + graph_suffix}, + {"visual_blocks_16_24" + graph_suffix, 16, 8, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_16_24" + graph_suffix}, + {"visual_blocks_24_32" + graph_suffix, 24, 8, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_24_32" + graph_suffix}, + {"visual_merger" + graph_suffix, 0, 0, true, false, {visual_patch_tokens, cfg.visual_embed_dim}, "merger" + graph_suffix}, + }; + } else if (layout == "tail4") { + segments = { + {"visual_patch_embed" + graph_suffix, 0, 0, false, true, {visual_patch_tokens, patch_flat_dim}, "patch_embed" + graph_suffix}, + {"visual_blocks_0_8" + graph_suffix, 0, 8, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_0_8" + graph_suffix}, + {"visual_blocks_8_16" + graph_suffix, 8, 8, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_8_16" + graph_suffix}, + {"visual_blocks_16_20" + graph_suffix, 16, 4, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_16_20" + graph_suffix}, + {"visual_blocks_20_24" + graph_suffix, 20, 4, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_20_24" + graph_suffix}, + {"visual_blocks_24_28" + graph_suffix, 24, 4, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_24_28" + graph_suffix}, + {"visual_blocks_28_32" + graph_suffix, 28, 4, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_28_32" + graph_suffix}, + {"visual_merger" + graph_suffix, 0, 0, true, false, {visual_patch_tokens, cfg.visual_embed_dim}, "merger" + graph_suffix}, + }; + } else if (layout == "early2") { + segments = { + {"visual_patch_embed" + graph_suffix, 0, 0, false, true, {visual_patch_tokens, patch_flat_dim}, "patch_embed" + graph_suffix}, + {"visual_blocks_0_2" + graph_suffix, 0, 2, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_0_2" + graph_suffix}, + {"visual_blocks_2_4" + graph_suffix, 2, 2, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_2_4" + graph_suffix}, + {"visual_blocks_4_6" + graph_suffix, 4, 2, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_4_6" + graph_suffix}, + {"visual_blocks_6_8" + graph_suffix, 6, 2, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_6_8" + graph_suffix}, + {"visual_blocks_8_16" + graph_suffix, 8, 8, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_8_16" + graph_suffix}, + {"visual_blocks_16_24" + graph_suffix, 16, 8, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_16_24" + graph_suffix}, + {"visual_blocks_24_32" + graph_suffix, 24, 8, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, "blocks_24_32" + graph_suffix}, + {"visual_merger" + graph_suffix, 0, 0, true, false, {visual_patch_tokens, cfg.visual_embed_dim}, "merger" + graph_suffix}, + }; + } else if (layout == "block1") { + segments.push_back({"visual_patch_embed" + graph_suffix, 0, 0, false, true, {visual_patch_tokens, patch_flat_dim}, "patch_embed" + graph_suffix}); + for (int32_t i = 0; i < visual_depth; ++i) { + const auto graph_name = "visual_blocks_" + std::to_string(i) + "_" + std::to_string(i + 1) + graph_suffix; + const auto ir_name = "blocks_" + std::to_string(i) + "_" + std::to_string(i + 1) + graph_suffix; + segments.push_back({graph_name, i, 1, true, true, {visual_patch_tokens, cfg.visual_embed_dim}, ir_name}); + } + segments.push_back({"visual_merger" + graph_suffix, 0, 0, true, false, {visual_patch_tokens, cfg.visual_embed_dim}, "merger" + graph_suffix}); + } else { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "Unsupported visual bundle layout: {}", layout); + } + return segments; +} + +} // namespace qwen2vl_qnn_aot diff --git a/examples/qwen2vl_qnn_aot/visual_aot_run.cpp b/examples/qwen2vl_qnn_aot/visual_aot_run.cpp new file mode 100644 index 000000000..64bcfed19 --- /dev/null +++ b/examples/qwen2vl_qnn_aot/visual_aot_run.cpp @@ -0,0 +1,590 @@ +// Copyright (c) MLLM Team. +// Licensed under the MIT License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +using mllm::Argparse; +using mllm::Tensor; +using mllm::models::qwen2vl::Qwen2VLConfig; +using mllm::models::qwen2vl::Qwen2VLTokenizer; +using mllm::qnn::aot::QnnAOTModule; + +namespace { + +bool g_verbose = false; + +struct TimedResult { + std::string name; + Tensor output; + Tensor compare_output; + double seconds = 0.0; +}; + +struct CompareStats { + int64_t numel = 0; + double cosine = 0.0; + double l2 = 0.0; + double ref_l2 = 0.0; + double l2_rel = 0.0; + double qnn_l2 = 0.0; + double norm_ratio = 0.0; + float max_abs_diff = 0.0f; + float qnn_at_max = 0.0f; + float ref_at_max = 0.0f; + int64_t max_abs_index = -1; + int64_t nonfinite_qnn = 0; + int64_t nonfinite_ref = 0; +}; + +Tensor makeQnnTensor(const std::vector& shape, mllm::DataTypes dtype, const std::string& name) { + if (g_verbose) { + fmt::print("[visual-aot] alloc QNN tensor {} shape=[", name); + for (size_t i = 0; i < shape.size(); ++i) { fmt::print("{}{}", i == 0 ? "" : ", ", shape[i]); } + fmt::print("] dtype={}\n", dtype); + std::cout << std::flush; + } + auto tensor = Tensor::empty(shape, dtype, mllm::kQNN).setName(name).alloc(); + if (g_verbose) { + fmt::print("[visual-aot] alloc QNN tensor {} done, bytes={}, ptr={}\n", name, tensor.bytes(), tensor.ptr()); + std::cout << std::flush; + } + return tensor; +} + +Tensor copyToQnn(const Tensor& cpu_tensor, const std::string& name) { + if (g_verbose) { + fmt::print("[visual-aot] copy {} CPU->QNN begin, bytes={}\n", name, cpu_tensor.bytes()); + std::cout << std::flush; + } + auto qnn_tensor = makeQnnTensor(cpu_tensor.shape(), cpu_tensor.dtype(), name); + if (g_verbose) { + fmt::print("[visual-aot] memcpy {} CPU ptr={} -> QNN ptr={} bytes={}\n", name, cpu_tensor.ptr(), qnn_tensor.ptr(), + cpu_tensor.bytes()); + std::cout << std::flush; + } + std::memcpy(qnn_tensor.ptr(), cpu_tensor.ptr(), cpu_tensor.bytes()); + if (g_verbose) { + fmt::print("[visual-aot] copy {} CPU->QNN done\n", name); + std::cout << std::flush; + } + return qnn_tensor; +} + +TimedResult runVisualGraph(const std::string& graph_name, + Tensor hidden, + Tensor visual_embedding_sin, + Tensor visual_embedding_cos, + Tensor output, + bool snapshot_output) { + fmt::print("[visual-aot] execute {} ...\n", graph_name); + std::cout << std::flush; + + QnnAOTModule module(graph_name); + module.to(mllm::kQNN); + module.setOutputTensors({output}); + + auto inputs = std::vector{hidden, visual_embedding_sin, visual_embedding_cos}; + const auto start = std::chrono::high_resolution_clock::now(); + auto outputs = module(inputs); + const auto end = std::chrono::high_resolution_clock::now(); + + MLLM_RT_ASSERT_EQ(outputs.size(), 1); + fmt::print("[visual-aot] execute {} done\n", graph_name); + std::cout << std::flush; + Tensor compare_output = Tensor::nil(); + if (snapshot_output) { + compare_output = outputs[0].to(mllm::kCPU).clone(); + } + + return { + .name = graph_name, + .output = outputs[0], + .compare_output = compare_output, + .seconds = std::chrono::duration(end - start).count(), + }; +} + +const Tensor& compareTensor(const TimedResult& result) { + return result.compare_output.isNil() ? result.output : result.compare_output; +} + +void printShape(const std::string& name, const Tensor& tensor) { + fmt::print("{} shape=[", name); + const auto& shape = tensor.shape(); + for (size_t i = 0; i < shape.size(); ++i) { fmt::print("{}{}", i == 0 ? "" : ", ", shape[i]); } + fmt::print("] dtype={} device={}\n", tensor.dtype(), tensor.device()); +} + +bool sameShape(const Tensor& lhs, const Tensor& rhs) { return lhs.shape() == rhs.shape(); } + +CompareStats compareFloatTensors(const Tensor& qnn_tensor, const Tensor& ref_tensor) { + MLLM_RT_ASSERT_EQ(qnn_tensor.dtype(), mllm::kFloat32); + MLLM_RT_ASSERT_EQ(ref_tensor.dtype(), mllm::kFloat32); + MLLM_RT_ASSERT(sameShape(qnn_tensor, ref_tensor)); + + CompareStats stats; + stats.numel = qnn_tensor.numel(); + const auto* qnn = qnn_tensor.ptr(); + const auto* ref = ref_tensor.ptr(); + + double dot = 0.0; + double qnn_norm2 = 0.0; + double ref_norm2 = 0.0; + double diff_norm2 = 0.0; + float max_abs = -1.0f; + + for (int64_t i = 0; i < stats.numel; ++i) { + const float q = qnn[i]; + const float r = ref[i]; + if (!std::isfinite(q)) { ++stats.nonfinite_qnn; } + if (!std::isfinite(r)) { ++stats.nonfinite_ref; } + const double qd = static_cast(q); + const double rd = static_cast(r); + const double diff = qd - rd; + dot += qd * rd; + qnn_norm2 += qd * qd; + ref_norm2 += rd * rd; + diff_norm2 += diff * diff; + const float abs_diff = std::abs(q - r); + if (abs_diff > max_abs) { + max_abs = abs_diff; + stats.max_abs_index = i; + stats.qnn_at_max = q; + stats.ref_at_max = r; + } + } + + constexpr double eps = 1e-12; + stats.qnn_l2 = std::sqrt(qnn_norm2); + stats.ref_l2 = std::sqrt(ref_norm2); + stats.l2 = std::sqrt(diff_norm2); + stats.cosine = dot / (stats.qnn_l2 * stats.ref_l2 + eps); + stats.l2_rel = stats.l2 / (stats.ref_l2 + eps); + stats.norm_ratio = stats.qnn_l2 / (stats.ref_l2 + eps); + stats.max_abs_diff = std::max(0.0f, max_abs); + return stats; +} + +void printCompareStats(const CompareStats& stats) { + fmt::print(fg(fmt::color::cyan), "\n{:=^58}\n", " Visual QNN vs Reference "); + fmt::print("numel {:>16}\n", stats.numel); + fmt::print("cosine {:>16.8f}\n", stats.cosine); + fmt::print("L2 diff {:>16.6f}\n", stats.l2); + fmt::print("ref L2 {:>16.6f}\n", stats.ref_l2); + fmt::print("relative L2 {:>16.8f}\n", stats.l2_rel); + fmt::print("norm ratio qnn/ref {:>16.8f}\n", stats.norm_ratio); + fmt::print("max abs diff {:>16.6f} @ {}\n", stats.max_abs_diff, stats.max_abs_index); + fmt::print("qnn/ref at max {:>16.6f} / {:.6f}\n", stats.qnn_at_max, stats.ref_at_max); + fmt::print("nonfinite qnn/ref {:>16} / {}\n", stats.nonfinite_qnn, stats.nonfinite_ref); + fmt::print(fg(fmt::color::cyan), "{:=^58}\n", ""); +} + +Tensor runReferenceVisual(const std::string& model_path, + const std::string& model_version, + const std::string& config_path, + Tensor img, + Tensor visual_embedding_sin, + Tensor visual_embedding_cos) { + fmt::print("[visual-aot] run reference visual model ...\n"); + std::cout << std::flush; + + auto cfg = Qwen2VLConfig(config_path); + mllm::ModelFileVersion file_version = mllm::ModelFileVersion::kV2; + if (model_version == "v1") { file_version = mllm::ModelFileVersion::kV1; } + + auto params = mllm::load(model_path, file_version); + auto visual = mllm::models::qwen2vl::Qwen2VisionTransformerPretrainedModel("visual", cfg); + visual.load(params); + + const auto start = std::chrono::high_resolution_clock::now(); + auto output = visual(img, visual_embedding_sin, visual_embedding_cos)[0]; + const auto end = std::chrono::high_resolution_clock::now(); + fmt::print("[visual-aot] reference visual done, time cost: {:.3f} s\n", + std::chrono::duration(end - start).count()); + printShape("reference_visual_embeddings", output); + std::cout << std::flush; + return output; +} + +std::vector runReferenceVisualBundle(const std::string& model_path, + const std::string& model_version, + const std::string& config_path, + Tensor img, + Tensor visual_embedding_sin, + Tensor visual_embedding_cos, + const std::string& bundle_layout) { + fmt::print("[visual-aot] run reference visual bundle ...\n"); + std::cout << std::flush; + + auto cfg = Qwen2VLConfig(config_path); + mllm::ModelFileVersion file_version = mllm::ModelFileVersion::kV2; + if (model_version == "v1") { file_version = mllm::ModelFileVersion::kV1; } + + auto params = mllm::load(model_path, file_version); + auto patch_embed = mllm::models::qwen2vl::PatchEmbed("visual.patch_embed", cfg); + auto patch_merger = mllm::models::qwen2vl::PatchMerger("visual.merger", cfg); + auto blocks = mllm::nn::ModuleList("visual.blocks", cfg.visual_depth, cfg); + patch_embed.load(params); + patch_merger.load(params); + blocks.load(params); + + std::vector results; + results.reserve(bundle_layout == "tail4" ? 8 : bundle_layout == "early2" ? 9 : bundle_layout == "block1" ? 34 : 6); + + auto push_stage = [&](const std::string& name, const Tensor& output) { + results.push_back({.name = name, .output = output, .seconds = 0.0}); + }; + + auto hidden = patch_embed(img)[0]; + push_stage("visual_patch_embed", hidden); + + auto run_block_range = [&](int32_t begin, int32_t end, const std::string& stage_name) { + for (int32_t i = begin; i < end; ++i) { + hidden = blocks.list()[i](hidden, visual_embedding_sin, visual_embedding_cos)[0]; + } + push_stage(stage_name, hidden); + }; + + if (bundle_layout == "6x8") { + run_block_range(0, 8, "visual_blocks_0_8"); + run_block_range(8, 16, "visual_blocks_8_16"); + run_block_range(16, 24, "visual_blocks_16_24"); + run_block_range(24, 32, "visual_blocks_24_32"); + } else if (bundle_layout == "tail4") { + run_block_range(0, 8, "visual_blocks_0_8"); + run_block_range(8, 16, "visual_blocks_8_16"); + run_block_range(16, 20, "visual_blocks_16_20"); + run_block_range(20, 24, "visual_blocks_20_24"); + run_block_range(24, 28, "visual_blocks_24_28"); + run_block_range(28, 32, "visual_blocks_28_32"); + } else if (bundle_layout == "early2") { + run_block_range(0, 2, "visual_blocks_0_2"); + run_block_range(2, 4, "visual_blocks_2_4"); + run_block_range(4, 6, "visual_blocks_4_6"); + run_block_range(6, 8, "visual_blocks_6_8"); + run_block_range(8, 16, "visual_blocks_8_16"); + run_block_range(16, 24, "visual_blocks_16_24"); + run_block_range(24, 32, "visual_blocks_24_32"); + } else if (bundle_layout == "block1") { + for (int32_t i = 0; i < 32; ++i) { + run_block_range(i, i + 1, "visual_blocks_" + std::to_string(i) + "_" + std::to_string(i + 1)); + } + } else { + std::cerr << "--bundle_layout must be 6x8, tail4, early2 or block1\n"; + return {}; + } + + auto merged = patch_merger(hidden)[0]; + push_stage("visual_merger", merged); + std::cout << std::flush; + return results; +} + +} // namespace + +MLLM_MAIN({ + auto& help = Argparse::add("-h|--help").help("Show help message"); + auto& context_path = Argparse::add("-m|--model").help("Qwen2-VL visual QNN AOT context .bin path").required(true); + auto& tokenizer_path = Argparse::add("-t|--tokenizer").help("tokenizer.json path").required(true); + auto& config_path = Argparse::add("-c|--config").help("Qwen2-VL config path").required(true); + auto& image_path = Argparse::add("-i|--image").help("input image path").required(true); + auto& prompt = Argparse::add("-p|--prompt").help("prompt text").def("describe this picture"); + auto& ref_model_path = + Argparse::add("--ref_model").help("Optional FP32/W4A32 .mllm model used to compare visual output.").def(""); + auto& ref_model_version = + Argparse::add("--ref_model_version").help("reference model file version: v1/v2").def("v2"); + auto& ref_config_path = + Argparse::add("--ref_config").help("Optional reference model config path. Defaults to --config.").def(""); + auto& compare_segments = + Argparse::add("--compare_segments").help("Compare each visual bundle segment against the CPU/KAI reference."); + auto& compare_segments_oracle = Argparse::add("--compare_segments_oracle") + .help("Run each visual QNN segment with the matching FP32 reference segment input."); + auto& bundle_layout = + Argparse::add("--bundle_layout").help("visual bundle layout: 6x8, tail4, early2 or block1").def("6x8"); + auto& max_graphs = Argparse::add("--max_graphs") + .help("Execute only the first N visual graphs for diagnostics. -1 executes the full bundle; 0 only prepares inputs.") + .def(-1); + auto& verbose = Argparse::add("--verbose").help("Print detailed allocation/copy diagnostics."); + + Argparse::parse(argc, argv); + if (help.isSet()) { + Argparse::printHelp(); + return 0; + } + g_verbose = verbose.isSet(); + + mllm::initQnnBackend(context_path.get()); + + auto cfg = Qwen2VLConfig(config_path.get()); + auto tokenizer = Qwen2VLTokenizer(tokenizer_path.get()); + auto inputs = tokenizer.convertMessage({.prompt = prompt.get(), .img_file_path = image_path.get()}); + auto img = inputs.at("img"); + auto grid_thw = inputs.at("grid_thw"); + + auto inv_freq = mllm::models::qwen2vl::makeVisualRoPEInvFreq(cfg.visual_embed_dim / cfg.visual_num_heads, 10000.0); + auto visual_pos_ids = mllm::models::qwen2vl::makeVisualRotaryPosEmbIds(grid_thw, cfg.visual_spatial_merge_size); + auto rotary_pos_emb_full = mllm::models::qwen2vl::makeVisualRotaryPosEmbFull(inv_freq, img.shape()[0]); + auto pos_emb = mllm::models::qwen2vl::makeVisualRotaryPosEmb(rotary_pos_emb_full, visual_pos_ids, grid_thw); + auto [visual_embedding_sin, visual_embedding_cos] = mllm::models::qwen2vl::makeVisualRotarySinCos(pos_emb); + + const int32_t half_dim = cfg.visual_embed_dim / cfg.visual_num_heads / 2; + visual_embedding_sin = visual_embedding_sin.view({1, -1, 1, half_dim}, false); + visual_embedding_cos = visual_embedding_cos.view({1, -1, 1, half_dim}, false); + + const int32_t visual_patch_tokens = img.shape()[0]; + const int32_t patch_dim = img.shape()[1]; + const int32_t merged_tokens = visual_patch_tokens / (cfg.visual_spatial_merge_size * cfg.visual_spatial_merge_size); + + fmt::print("Qwen2-VL visual QNN AOT runner\n"); + printShape("img", img); + printShape("grid_thw", grid_thw); + printShape("visual_embedding_sin", visual_embedding_sin); + printShape("visual_embedding_cos", visual_embedding_cos); + std::cout << std::flush; + + if (g_verbose) { + fmt::print("[visual-aot] copy CPU inputs to QNN shared buffers ...\n"); + std::cout << std::flush; + } + auto img_qnn = copyToQnn(img, "visual_img"); + auto sin_qnn = copyToQnn(visual_embedding_sin, "visual_embedding_sin"); + auto cos_qnn = copyToQnn(visual_embedding_cos, "visual_embedding_cos"); + if (g_verbose) { + fmt::print("[visual-aot] copy CPU inputs to QNN shared buffers done\n"); + std::cout << std::flush; + } + + auto hidden_a = makeQnnTensor({visual_patch_tokens, cfg.visual_embed_dim}, mllm::kFloat32, "visual_hidden_a"); + auto hidden_b = makeQnnTensor({visual_patch_tokens, cfg.visual_embed_dim}, mllm::kFloat32, "visual_hidden_b"); + auto visual_embeddings = makeQnnTensor({merged_tokens, cfg.hidden_size}, mllm::kFloat32, "visual_embeddings"); + + std::vector results; + if (bundle_layout.get() == "6x8") { + results.reserve(6); + const int32_t graph_limit = max_graphs.get() < 0 ? 6 : std::max(0, std::min(max_graphs.get(), 6)); + if (graph_limit >= 1) { + results.push_back(runVisualGraph("visual_patch_embed", img_qnn, sin_qnn, cos_qnn, hidden_a, compare_segments.isSet())); + } + if (graph_limit >= 2) { + results.push_back(runVisualGraph("visual_blocks_0_8", results.back().output, sin_qnn, cos_qnn, hidden_b, + compare_segments.isSet())); + } + if (graph_limit >= 3) { + results.push_back(runVisualGraph("visual_blocks_8_16", results.back().output, sin_qnn, cos_qnn, hidden_a, + compare_segments.isSet())); + } + if (graph_limit >= 4) { + results.push_back(runVisualGraph("visual_blocks_16_24", results.back().output, sin_qnn, cos_qnn, hidden_b, + compare_segments.isSet())); + } + if (graph_limit >= 5) { + results.push_back(runVisualGraph("visual_blocks_24_32", results.back().output, sin_qnn, cos_qnn, hidden_a, + compare_segments.isSet())); + } + if (graph_limit >= 6) { + results.push_back(runVisualGraph("visual_merger", results.back().output, sin_qnn, cos_qnn, visual_embeddings, + compare_segments.isSet())); + } + } else if (bundle_layout.get() == "tail4") { + results.reserve(8); + const int32_t graph_limit = max_graphs.get() < 0 ? 8 : std::max(0, std::min(max_graphs.get(), 8)); + if (graph_limit >= 1) { + results.push_back(runVisualGraph("visual_patch_embed", img_qnn, sin_qnn, cos_qnn, hidden_a, compare_segments.isSet())); + } + if (graph_limit >= 2) { + results.push_back(runVisualGraph("visual_blocks_0_8", results.back().output, sin_qnn, cos_qnn, hidden_b, + compare_segments.isSet())); + } + if (graph_limit >= 3) { + results.push_back(runVisualGraph("visual_blocks_8_16", results.back().output, sin_qnn, cos_qnn, hidden_a, + compare_segments.isSet())); + } + if (graph_limit >= 4) { + results.push_back(runVisualGraph("visual_blocks_16_20", results.back().output, sin_qnn, cos_qnn, hidden_b, + compare_segments.isSet())); + } + if (graph_limit >= 5) { + results.push_back(runVisualGraph("visual_blocks_20_24", results.back().output, sin_qnn, cos_qnn, hidden_a, + compare_segments.isSet())); + } + if (graph_limit >= 6) { + results.push_back(runVisualGraph("visual_blocks_24_28", results.back().output, sin_qnn, cos_qnn, hidden_b, + compare_segments.isSet())); + } + if (graph_limit >= 7) { + results.push_back(runVisualGraph("visual_blocks_28_32", results.back().output, sin_qnn, cos_qnn, hidden_a, + compare_segments.isSet())); + } + if (graph_limit >= 8) { + results.push_back(runVisualGraph("visual_merger", results.back().output, sin_qnn, cos_qnn, visual_embeddings, + compare_segments.isSet())); + } + } else if (bundle_layout.get() == "early2") { + results.reserve(9); + const int32_t graph_limit = max_graphs.get() < 0 ? 9 : std::max(0, std::min(max_graphs.get(), 9)); + if (graph_limit >= 1) { + results.push_back(runVisualGraph("visual_patch_embed", img_qnn, sin_qnn, cos_qnn, hidden_a, compare_segments.isSet())); + } + if (graph_limit >= 2) { + results.push_back(runVisualGraph("visual_blocks_0_2", results.back().output, sin_qnn, cos_qnn, hidden_b, + compare_segments.isSet())); + } + if (graph_limit >= 3) { + results.push_back(runVisualGraph("visual_blocks_2_4", results.back().output, sin_qnn, cos_qnn, hidden_a, + compare_segments.isSet())); + } + if (graph_limit >= 4) { + results.push_back(runVisualGraph("visual_blocks_4_6", results.back().output, sin_qnn, cos_qnn, hidden_b, + compare_segments.isSet())); + } + if (graph_limit >= 5) { + results.push_back(runVisualGraph("visual_blocks_6_8", results.back().output, sin_qnn, cos_qnn, hidden_a, + compare_segments.isSet())); + } + if (graph_limit >= 6) { + results.push_back(runVisualGraph("visual_blocks_8_16", results.back().output, sin_qnn, cos_qnn, hidden_b, + compare_segments.isSet())); + } + if (graph_limit >= 7) { + results.push_back(runVisualGraph("visual_blocks_16_24", results.back().output, sin_qnn, cos_qnn, hidden_a, + compare_segments.isSet())); + } + if (graph_limit >= 8) { + results.push_back(runVisualGraph("visual_blocks_24_32", results.back().output, sin_qnn, cos_qnn, hidden_b, + compare_segments.isSet())); + } + if (graph_limit >= 9) { + results.push_back(runVisualGraph("visual_merger", results.back().output, sin_qnn, cos_qnn, visual_embeddings, + compare_segments.isSet())); + } + } else if (bundle_layout.get() == "block1") { + results.reserve(34); + const int32_t graph_limit = max_graphs.get() < 0 ? 34 : std::max(0, std::min(max_graphs.get(), 34)); + if (graph_limit >= 1) { + results.push_back(runVisualGraph("visual_patch_embed", img_qnn, sin_qnn, cos_qnn, hidden_a, compare_segments.isSet())); + } + for (int32_t i = 0; i < 32 && graph_limit >= i + 2; ++i) { + auto& output = (i % 2 == 0) ? hidden_b : hidden_a; + results.push_back(runVisualGraph("visual_blocks_" + std::to_string(i) + "_" + std::to_string(i + 1), results.back().output, + sin_qnn, cos_qnn, output, compare_segments.isSet())); + } + if (graph_limit >= 34) { + results.push_back(runVisualGraph("visual_merger", results.back().output, sin_qnn, cos_qnn, visual_embeddings, + compare_segments.isSet())); + } + } else { + std::cerr << "--bundle_layout must be 6x8, tail4, early2 or block1\n"; + return 1; + } + + double total_seconds = 0.0; + fmt::print(fg(fmt::color::cyan), "\n{:=^58}\n", " Visual QNN AOT Performance "); + for (const auto& result : results) { + total_seconds += result.seconds; + fmt::print("{:<20} {:>10.3f} s ", result.name, result.seconds); + printShape("output", result.output); + } + fmt::print("Visual total {:>10.3f} s\n", total_seconds); + fmt::print(fg(fmt::color::cyan), "{:=^58}\n", ""); + + if (!ref_model_path.get().empty()) { + const bool bundle_ok = (bundle_layout.get() == "6x8" && results.size() == 6 && results.back().name == "visual_merger") + || (bundle_layout.get() == "tail4" && results.size() == 8 && results.back().name == "visual_merger") + || (bundle_layout.get() == "early2" && results.size() == 9 && results.back().name == "visual_merger") + || (bundle_layout.get() == "block1" && results.size() == 34 && results.back().name == "visual_merger"); + if (!bundle_ok) { + fmt::print("[visual-aot] skip reference comparison: --ref_model requires full bundle execution.\n"); + } else { + const std::string ref_cfg = ref_config_path.get().empty() ? config_path.get() : ref_config_path.get(); + if (compare_segments.isSet() || compare_segments_oracle.isSet()) { + auto ref_segments = runReferenceVisualBundle(ref_model_path.get(), + ref_model_version.get(), + ref_cfg, + img, + visual_embedding_sin, + visual_embedding_cos, + bundle_layout.get()); + if (ref_segments.size() == results.size()) { + if (compare_segments.isSet()) { + fmt::print("\n{:=^72}\n", " Visual Segment vs Reference "); + for (size_t i = 0; i < results.size(); ++i) { + const auto& qnn_stage = results[i]; + const auto& ref_stage = ref_segments[i]; + if (qnn_stage.name != ref_stage.name) { + fmt::print("{:<24} vs {:<24} name mismatch, skip\n", qnn_stage.name, ref_stage.name); + continue; + } + const auto& qnn_compare = compareTensor(qnn_stage); + if (!sameShape(qnn_compare, ref_stage.output)) { + fmt::print("{:<24} shape mismatch, skip\n", qnn_stage.name); + printShape("qnn_stage", qnn_compare); + printShape("ref_stage", ref_stage.output); + continue; + } + auto stats = compareFloatTensors(qnn_compare, ref_stage.output); + fmt::print("{:<24} cosine={:>10.8f} rel_l2={:>10.8f} norm_ratio={:>10.8f}\n", + qnn_stage.name, stats.cosine, stats.l2_rel, stats.norm_ratio); + } + fmt::print("{:=^72}\n", ""); + } + + if (compare_segments_oracle.isSet()) { + fmt::print("\n{:=^84}\n", " Visual Segment Oracle Input vs Reference "); + for (size_t i = 0; i < ref_segments.size(); ++i) { + const auto& ref_stage = ref_segments[i]; + Tensor qnn_input = Tensor::nil(); + if (i == 0) { + qnn_input = img_qnn; + } else { + qnn_input = copyToQnn(ref_segments[i - 1].output, "oracle_input_" + std::to_string(i)); + } + auto qnn_output = + makeQnnTensor(ref_stage.output.shape(), ref_stage.output.dtype(), "oracle_output_" + std::to_string(i)); + auto qnn_stage = runVisualGraph(ref_stage.name, qnn_input, sin_qnn, cos_qnn, qnn_output, true); + const auto& qnn_compare = compareTensor(qnn_stage); + auto stats = compareFloatTensors(qnn_compare, ref_stage.output); + fmt::print("{:<24} cosine={:>10.8f} rel_l2={:>10.8f} norm_ratio={:>10.8f} time={:>8.3f}s\n", + ref_stage.name, stats.cosine, stats.l2_rel, stats.norm_ratio, qnn_stage.seconds); + } + fmt::print("{:=^84}\n", ""); + } + } else { + fmt::print("[visual-aot] skip segment comparison: reference segment count mismatch.\n"); + } + } + + auto ref_visual_embeddings = + runReferenceVisual(ref_model_path.get(), ref_model_version.get(), ref_cfg, img, visual_embedding_sin, visual_embedding_cos); + if (!sameShape(results.back().output, ref_visual_embeddings)) { + fmt::print("[visual-aot] skip comparison: shape mismatch between QNN output and reference output.\n"); + printShape("qnn_visual_embeddings", results.back().output); + printShape("reference_visual_embeddings", ref_visual_embeddings); + } else { + printCompareStats(compareFloatTensors(results.back().output, ref_visual_embeddings)); + } + } + } + + mllm::memoryReport(); + return 0; +}); diff --git a/examples/qwen2vl_qnn_aot/visual_padding_ab_diag.cpp b/examples/qwen2vl_qnn_aot/visual_padding_ab_diag.cpp new file mode 100644 index 000000000..bde42507e --- /dev/null +++ b/examples/qwen2vl_qnn_aot/visual_padding_ab_diag.cpp @@ -0,0 +1,484 @@ +// Copyright (c) MLLM Team. +// Licensed under the MIT License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +using mllm::Argparse; +using mllm::Tensor; +using mllm::models::qwen2vl::Qwen2VLConfig; +using mllm::models::qwen2vl::Qwen2VLTokenizer; +using mllm::qnn::aot::QnnAOTModule; + +namespace { + +struct Grid { + int32_t t = 1; + int32_t h = 0; + int32_t w = 0; + + [[nodiscard]] int32_t patchTokens() const { return t * h * w; } +}; + +struct StageResult { + std::string name; + Tensor output; + double seconds = 0.0; + bool merged = false; +}; + +struct CompareStats { + int64_t numel = 0; + int64_t finite_count = 0; + int64_t nonfinite_test = 0; + int64_t nonfinite_ref = 0; + double cosine = 0.0; + double l2 = 0.0; + double ref_l2 = 0.0; + double rel_l2 = 0.0; + double norm_ratio = 0.0; + float max_abs_diff = 0.0f; + int64_t max_abs_index = -1; +}; + +Grid parseGrid(const std::string& text) { + auto sep = text.find('x'); + if (sep == std::string::npos) { sep = text.find('X'); } + if (sep == std::string::npos) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "Invalid grid '{}', expected HxW.", text); + } + Grid grid{1, std::stoi(text.substr(0, sep)), std::stoi(text.substr(sep + 1))}; + if (grid.h <= 0 || grid.w <= 0 || grid.h % 2 != 0 || grid.w % 2 != 0) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "Invalid grid '{}': H/W must be positive even numbers.", text); + } + return grid; +} + +std::string resolveSuffix(const std::string& value, int32_t patch_tokens) { + if (value == "auto") { return "_s" + std::to_string(patch_tokens); } + if (value == "none") { return ""; } + return value; +} + +Tensor makeGridThwTensor(const Grid& grid) { + auto t = Tensor::empty({1, 3}, mllm::kInt32, mllm::kCPU).alloc(); + t.ptr()[0] = grid.t; + t.ptr()[1] = grid.h; + t.ptr()[2] = grid.w; + return t; +} + +Grid readGrid(Tensor grid_thw) { + const auto* g = grid_thw.ptr(); + return {.t = g[0], .h = g[1], .w = g[2]}; +} + +int64_t visualPatchIndex(int32_t t, int32_t h, int32_t w, int32_t grid_h, int32_t grid_w, int32_t merge_size) { + const int32_t h_blocks = grid_h / merge_size; + const int32_t w_blocks = grid_w / merge_size; + return (((static_cast(t) * h_blocks + h / merge_size) * w_blocks + w / merge_size) * merge_size + h % merge_size) + * merge_size + + w % merge_size; +} + +int64_t visualMergedIndex(int32_t t, int32_t h, int32_t w, int32_t grid_h, int32_t grid_w, int32_t merge_size) { + const int32_t h_blocks = grid_h / merge_size; + const int32_t w_blocks = grid_w / merge_size; + return (static_cast(t) * h_blocks + h) * w_blocks + w; +} + +Tensor padVisualPatchesToBucket(Tensor img, Tensor grid_thw, const Grid& bucket, const Qwen2VLConfig& cfg) { + MLLM_RT_ASSERT_EQ(img.dtype(), mllm::kFloat32); + const auto original = readGrid(grid_thw); + if (bucket.t != original.t) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "Only t=1 visual buckets are supported in this diagnostic."); + } + if (bucket.h < original.h || bucket.w < original.w) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, + "Bucket {}x{} cannot cover original grid {}x{}.", + bucket.h, + bucket.w, + original.h, + original.w); + } + if (bucket.h == original.h && bucket.w == original.w) { return img; } + + const int32_t patch_dim = img.shape()[1]; + auto padded = Tensor::empty({bucket.patchTokens(), patch_dim}, img.dtype(), mllm::kCPU).alloc(); + std::fill(padded.ptr(), padded.ptr() + padded.numel(), 0.0f); + + for (int32_t t = 0; t < original.t; ++t) { + for (int32_t h = 0; h < original.h; ++h) { + for (int32_t w = 0; w < original.w; ++w) { + const auto src_idx = visualPatchIndex(t, h, w, original.h, original.w, cfg.visual_spatial_merge_size); + const auto dst_idx = visualPatchIndex(t, h, w, bucket.h, bucket.w, cfg.visual_spatial_merge_size); + std::memcpy(padded.offsettedPtr({static_cast(dst_idx), 0}), + img.offsettedPtr({static_cast(src_idx), 0}), + static_cast(patch_dim) * sizeof(float)); + } + } + } + return padded; +} + +Tensor makeVisualAttentionMaskForBucket(Tensor original_grid_thw, const Grid& bucket, const Qwen2VLConfig& cfg) { + const auto original = readGrid(original_grid_thw); + if (bucket.t != original.t) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "Only t=1 visual buckets are supported in this diagnostic."); + } + if (bucket.h < original.h || bucket.w < original.w) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, + "Bucket {}x{} cannot cover original grid {}x{}.", + bucket.h, + bucket.w, + original.h, + original.w); + } + + auto mask = Tensor::empty({1, 1, 1, bucket.patchTokens()}, mllm::kFloat32, mllm::kCPU).alloc(); + std::fill(mask.ptr(), mask.ptr() + mask.numel(), -10000.0f); + for (int32_t t = 0; t < original.t; ++t) { + for (int32_t h = 0; h < original.h; ++h) { + for (int32_t w = 0; w < original.w; ++w) { + const auto idx = visualPatchIndex(t, h, w, bucket.h, bucket.w, cfg.visual_spatial_merge_size); + *mask.offsettedPtr({0, 0, 0, static_cast(idx)}) = 0.0f; + } + } + } + return mask; +} + +Tensor makeAllValidVisualAttentionMask(int32_t patch_tokens) { + auto mask = Tensor::empty({1, 1, 1, patch_tokens}, mllm::kFloat32, mllm::kCPU).alloc(); + std::fill(mask.ptr(), mask.ptr() + mask.numel(), 0.0f); + return mask; +} + +Tensor cropPatchTensorFromBucket(Tensor bucket_tensor, Tensor original_grid_thw, const Grid& bucket, const Qwen2VLConfig& cfg) { + const auto original = readGrid(original_grid_thw); + if (bucket.h == original.h && bucket.w == original.w) { return bucket_tensor; } + + const int32_t hidden_size = bucket_tensor.shape()[1]; + auto cropped = Tensor::empty({original.patchTokens(), hidden_size}, bucket_tensor.dtype(), mllm::kCPU).alloc(); + for (int32_t t = 0; t < original.t; ++t) { + for (int32_t h = 0; h < original.h; ++h) { + for (int32_t w = 0; w < original.w; ++w) { + const auto src_idx = visualPatchIndex(t, h, w, bucket.h, bucket.w, cfg.visual_spatial_merge_size); + const auto dst_idx = visualPatchIndex(t, h, w, original.h, original.w, cfg.visual_spatial_merge_size); + std::memcpy(cropped.offsettedPtr({static_cast(dst_idx), 0}), + bucket_tensor.offsettedPtr({static_cast(src_idx), 0}), + static_cast(hidden_size) * sizeof(float)); + } + } + } + return cropped; +} + +Tensor cropMergedTensorFromBucket(Tensor bucket_tensor, Tensor original_grid_thw, const Grid& bucket, const Qwen2VLConfig& cfg) { + const auto original = readGrid(original_grid_thw); + if (bucket.h == original.h && bucket.w == original.w) { return bucket_tensor; } + + const int32_t merge = cfg.visual_spatial_merge_size; + const int32_t hidden_size = bucket_tensor.shape()[1]; + const int32_t original_h = original.h / merge; + const int32_t original_w = original.w / merge; + auto cropped = Tensor::empty({original.t * original_h * original_w, hidden_size}, bucket_tensor.dtype(), mllm::kCPU).alloc(); + + for (int32_t t = 0; t < original.t; ++t) { + for (int32_t h = 0; h < original_h; ++h) { + for (int32_t w = 0; w < original_w; ++w) { + const auto src_idx = visualMergedIndex(t, h, w, bucket.h, bucket.w, merge); + const auto dst_idx = visualMergedIndex(t, h, w, original.h, original.w, merge); + std::memcpy(cropped.offsettedPtr({static_cast(dst_idx), 0}), + bucket_tensor.offsettedPtr({static_cast(src_idx), 0}), + static_cast(hidden_size) * sizeof(float)); + } + } + } + return cropped; +} + +std::pair makeVisualSinCos(const Qwen2VLConfig& cfg, Tensor grid_thw, int32_t patch_tokens) { + auto inv_freq = mllm::models::qwen2vl::makeVisualRoPEInvFreq(cfg.visual_embed_dim / cfg.visual_num_heads, 10000.0); + auto pos_ids = mllm::models::qwen2vl::makeVisualRotaryPosEmbIds(grid_thw, cfg.visual_spatial_merge_size); + auto rotary_pos_emb_full = mllm::models::qwen2vl::makeVisualRotaryPosEmbFull(inv_freq, patch_tokens); + auto pos_emb = mllm::models::qwen2vl::makeVisualRotaryPosEmb(rotary_pos_emb_full, pos_ids, grid_thw); + auto [sin, cos] = mllm::models::qwen2vl::makeVisualRotarySinCos(pos_emb); + const int32_t half_dim = cfg.visual_embed_dim / cfg.visual_num_heads / 2; + return {sin.view({1, -1, 1, half_dim}, false), cos.view({1, -1, 1, half_dim}, false)}; +} + +Tensor makeQnnTensor(const std::vector& shape, mllm::DataTypes dtype, const std::string& name) { + return Tensor::empty(shape, dtype, mllm::kQNN).setName(name).alloc(); +} + +Tensor copyToQnn(const Tensor& cpu_tensor, const std::string& name) { + auto qnn_tensor = makeQnnTensor(cpu_tensor.shape(), cpu_tensor.dtype(), name); + std::memcpy(qnn_tensor.ptr(), cpu_tensor.ptr(), cpu_tensor.bytes()); + return qnn_tensor; +} + +StageResult runVisualGraph(const std::string& graph_name, + Tensor hidden, + Tensor visual_embedding_sin, + Tensor visual_embedding_cos, + Tensor visual_attention_mask, + Tensor output, + bool merged) { + QnnAOTModule module(graph_name); + module.to(mllm::kQNN); + module.setOutputTensors({output}); + auto inputs = visual_attention_mask.isNil() ? std::vector{hidden, visual_embedding_sin, visual_embedding_cos} + : std::vector{hidden, + visual_embedding_sin, + visual_embedding_cos, + visual_attention_mask}; + const auto start = std::chrono::high_resolution_clock::now(); + auto outputs = module(inputs); + const auto end = std::chrono::high_resolution_clock::now(); + MLLM_RT_ASSERT_EQ(outputs.size(), 1); + return { + .name = graph_name, + .output = outputs[0].to(mllm::kCPU).clone(), + .seconds = std::chrono::duration(end - start).count(), + .merged = merged, + }; +} + +std::vector graphNamesForLayout(const std::string& layout) { + if (layout == "6x8") { + return { + "visual_patch_embed", + "visual_blocks_0_8", + "visual_blocks_8_16", + "visual_blocks_16_24", + "visual_blocks_24_32", + "visual_merger", + }; + } + if (layout == "early2") { + return { + "visual_patch_embed", + "visual_blocks_0_2", + "visual_blocks_2_4", + "visual_blocks_4_6", + "visual_blocks_6_8", + "visual_blocks_8_16", + "visual_blocks_16_24", + "visual_blocks_24_32", + "visual_merger", + }; + } + if (layout == "tail4") { + return { + "visual_patch_embed", + "visual_blocks_0_8", + "visual_blocks_8_16", + "visual_blocks_16_20", + "visual_blocks_20_24", + "visual_blocks_24_28", + "visual_blocks_28_32", + "visual_merger", + }; + } + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "--bundle_layout must be 6x8, early2 or tail4."); +} + +std::vector runVisualBundle(Tensor img, + Tensor sin, + Tensor cos, + Tensor attention_mask, + const Qwen2VLConfig& cfg, + const std::string& layout, + const std::string& suffix, + const std::string& tag) { + const int32_t patch_tokens = img.shape()[0]; + const int32_t merged_tokens = patch_tokens / (cfg.visual_spatial_merge_size * cfg.visual_spatial_merge_size); + auto img_qnn = copyToQnn(img, tag + "_img"); + auto sin_qnn = copyToQnn(sin, tag + "_sin"); + auto cos_qnn = copyToQnn(cos, tag + "_cos"); + auto mask_qnn = copyToQnn(attention_mask, tag + "_attention_mask"); + + auto hidden_a = makeQnnTensor({patch_tokens, cfg.visual_embed_dim}, mllm::kFloat32, tag + "_hidden_a"); + auto hidden_b = makeQnnTensor({patch_tokens, cfg.visual_embed_dim}, mllm::kFloat32, tag + "_hidden_b"); + auto merged = makeQnnTensor({merged_tokens, cfg.hidden_size}, mllm::kFloat32, tag + "_merged"); + + std::vector results; + const auto graph_names = graphNamesForLayout(layout); + Tensor current = img_qnn; + for (size_t i = 0; i < graph_names.size(); ++i) { + const bool is_last = i + 1 == graph_names.size(); + Tensor output = is_last ? merged : (i % 2 == 0 ? hidden_a : hidden_b); + const auto graph_name = graph_names[i] + suffix; + fmt::print("[{}] execute {} ...\n", tag, graph_name); + const bool is_block = graph_names[i].find("visual_blocks_") == 0; + current = runVisualGraph(graph_name, current, sin_qnn, cos_qnn, is_block ? mask_qnn : Tensor::nil(), output, is_last).output; + results.push_back({.name = graph_names[i], .output = current, .seconds = 0.0, .merged = is_last}); + if (!is_last) { + current = copyToQnn(results.back().output, tag + "_stage_" + std::to_string(i)); + } + } + return results; +} + +CompareStats compareFloatTensors(const Tensor& test_tensor, const Tensor& ref_tensor) { + MLLM_RT_ASSERT(test_tensor.shape() == ref_tensor.shape()); + MLLM_RT_ASSERT_EQ(test_tensor.dtype(), mllm::kFloat32); + MLLM_RT_ASSERT_EQ(ref_tensor.dtype(), mllm::kFloat32); + + CompareStats stats; + stats.numel = test_tensor.numel(); + const auto* test = test_tensor.ptr(); + const auto* ref = ref_tensor.ptr(); + double dot = 0.0; + double test_norm2 = 0.0; + double ref_norm2 = 0.0; + double diff_norm2 = 0.0; + float max_abs = -1.0f; + + for (int64_t i = 0; i < stats.numel; ++i) { + const float a = test[i]; + const float b = ref[i]; + if (!std::isfinite(a)) { + ++stats.nonfinite_test; + continue; + } + if (!std::isfinite(b)) { + ++stats.nonfinite_ref; + continue; + } + ++stats.finite_count; + const double ad = static_cast(a); + const double bd = static_cast(b); + const double diff = ad - bd; + dot += ad * bd; + test_norm2 += ad * ad; + ref_norm2 += bd * bd; + diff_norm2 += diff * diff; + const float abs_diff = std::abs(a - b); + if (abs_diff > max_abs) { + max_abs = abs_diff; + stats.max_abs_index = i; + } + } + + constexpr double eps = 1e-12; + const double test_l2 = std::sqrt(test_norm2); + stats.ref_l2 = std::sqrt(ref_norm2); + stats.l2 = std::sqrt(diff_norm2); + stats.cosine = dot / (test_l2 * stats.ref_l2 + eps); + stats.rel_l2 = stats.l2 / (stats.ref_l2 + eps); + stats.norm_ratio = test_l2 / (stats.ref_l2 + eps); + stats.max_abs_diff = std::max(0.0f, max_abs); + return stats; +} + +void printShape(const std::string& name, const Tensor& tensor) { + fmt::print("{} dtype={} shape=[", name, mllm::nameOfType(tensor.dtype())); + const auto& shape = tensor.shape(); + for (size_t i = 0; i < shape.size(); ++i) { fmt::print("{}{}", i == 0 ? "" : ", ", shape[i]); } + fmt::print("]\n"); +} + +} // namespace + +MLLM_MAIN({ + auto& help = Argparse::add("-h|--help").help("Show help message"); + auto& context_path = Argparse::add("-m|--model").help("QNN AOT context containing both native and bucket visual graphs.").required(true); + auto& tokenizer_path = Argparse::add("-t|--tokenizer").help("tokenizer.json path").required(true); + auto& config_path = Argparse::add("-c|--config").help("Qwen2-VL visual config path").required(true); + auto& image_path = Argparse::add("-i|--image").help("input image path").required(true); + auto& prompt = Argparse::add("-p|--prompt").help("prompt text").def("describe this picture"); + auto& bucket_grid_arg = Argparse::add("--bucket_grid").help("Padded bucket grid HxW, e.g. 36x26.").required(true); + auto& native_suffix_arg = Argparse::add("--native_suffix") + .help("Native graph suffix. Use auto for _s{native_tokens}, or none for unsuffixed graphs.") + .def("auto"); + auto& padded_suffix_arg = Argparse::add("--padded_suffix") + .help("Padded graph suffix. Use auto for _s{bucket_tokens}, or none for unsuffixed graphs.") + .def("auto"); + auto& bundle_layout = Argparse::add("--bundle_layout").help("visual bundle layout: 6x8, early2 or tail4").def("6x8"); + + Argparse::parse(argc, argv); + if (help.isSet()) { + Argparse::printHelp(); + return 0; + } + + mllm::initQnnBackend(context_path.get()); + + auto cfg = Qwen2VLConfig(config_path.get()); + auto tokenizer = Qwen2VLTokenizer(tokenizer_path.get()); + auto inputs = tokenizer.convertMessage({.prompt = prompt.get(), .img_file_path = image_path.get()}); + auto img = inputs.at("img"); + auto grid_thw = inputs.at("grid_thw"); + const auto original_grid = readGrid(grid_thw); + const auto bucket_grid = parseGrid(bucket_grid_arg.get()); + auto bucket_grid_thw = makeGridThwTensor(bucket_grid); + auto padded_img = padVisualPatchesToBucket(img, grid_thw, bucket_grid, cfg); + + const auto native_suffix = resolveSuffix(native_suffix_arg.get(), img.shape()[0]); + const auto padded_suffix = resolveSuffix(padded_suffix_arg.get(), padded_img.shape()[0]); + + fmt::print("Qwen2-VL visual padding A/B diagnostic\n"); + fmt::print("original grid : {}x{} patches={} suffix='{}'\n", original_grid.h, original_grid.w, img.shape()[0], native_suffix); + fmt::print("bucket grid : {}x{} patches={} suffix='{}'\n", bucket_grid.h, bucket_grid.w, padded_img.shape()[0], padded_suffix); + printShape("img", img); + printShape("padded_img", padded_img); + + auto [native_sin, native_cos] = makeVisualSinCos(cfg, grid_thw, img.shape()[0]); + auto [padded_sin, padded_cos] = makeVisualSinCos(cfg, bucket_grid_thw, padded_img.shape()[0]); + auto native_mask = makeAllValidVisualAttentionMask(img.shape()[0]); + auto padded_mask = makeVisualAttentionMaskForBucket(grid_thw, bucket_grid, cfg); + + auto native_results = runVisualBundle(img, native_sin, native_cos, native_mask, cfg, bundle_layout.get(), native_suffix, "native"); + auto padded_results = + runVisualBundle(padded_img, padded_sin, padded_cos, padded_mask, cfg, bundle_layout.get(), padded_suffix, "padded"); + MLLM_RT_ASSERT_EQ(native_results.size(), padded_results.size()); + + fmt::print(fg(fmt::color::cyan), "\n{:=^96}\n", " Native QNN vs Padded QNN Crop "); + fmt::print("{:<24} {:>12} {:>12} {:>12} {:>12} {:>10} {:>10}\n", + "stage", + "cosine", + "rel_l2", + "norm_ratio", + "max_abs", + "nonfin_t", + "nonfin_r"); + for (size_t i = 0; i < native_results.size(); ++i) { + const auto& native = native_results[i]; + const auto& padded = padded_results[i]; + MLLM_RT_ASSERT(native.name == padded.name); + auto cropped = padded.merged ? cropMergedTensorFromBucket(padded.output, grid_thw, bucket_grid, cfg) + : cropPatchTensorFromBucket(padded.output, grid_thw, bucket_grid, cfg); + auto stats = compareFloatTensors(cropped, native.output); + fmt::print("{:<24} {:>12.8f} {:>12.8f} {:>12.8f} {:>12.6f} {:>10} {:>10}\n", + native.name, + stats.cosine, + stats.rel_l2, + stats.norm_ratio, + stats.max_abs_diff, + stats.nonfinite_test, + stats.nonfinite_ref); + } + fmt::print(fg(fmt::color::cyan), "{:=^96}\n", ""); + + mllm::memoryReport(); + return 0; +}); diff --git a/examples/qwen2vl_qnn_aot/visual_padding_diag.cpp b/examples/qwen2vl_qnn_aot/visual_padding_diag.cpp new file mode 100644 index 000000000..2b0f5f7ee --- /dev/null +++ b/examples/qwen2vl_qnn_aot/visual_padding_diag.cpp @@ -0,0 +1,296 @@ +// Copyright (c) MLLM Team. +// Licensed under the MIT License. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +using mllm::Argparse; +using mllm::Tensor; +using mllm::models::qwen2vl::Qwen2VLConfig; +using mllm::models::qwen2vl::Qwen2VLTokenizer; + +namespace { + +struct CompareStats { + int64_t numel = 0; + int64_t nan_count = 0; + int64_t inf_count = 0; + int64_t finite_count = 0; + double cosine = 0.0; + double l2 = 0.0; + double ref_l2 = 0.0; + double rel_l2 = 0.0; + double norm_ratio = 0.0; + float max_abs_diff = 0.0f; +}; + +struct BucketGrid { + int32_t grid_h = 0; + int32_t grid_w = 0; +}; + +BucketGrid parseBucketGrid(const std::string& text) { + auto sep = text.find('x'); + if (sep == std::string::npos) { sep = text.find('X'); } + if (sep == std::string::npos) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "Invalid --bucket_grid '{}', expected HxW.", text); + } + BucketGrid grid{std::stoi(text.substr(0, sep)), std::stoi(text.substr(sep + 1))}; + if (grid.grid_h <= 0 || grid.grid_w <= 0 || grid.grid_h % 2 != 0 || grid.grid_w % 2 != 0) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "Invalid --bucket_grid '{}': values must be positive even numbers.", text); + } + return grid; +} + +int64_t visualPatchIndex(int32_t t, int32_t h, int32_t w, int32_t grid_h, int32_t grid_w, int32_t merge_size) { + const int32_t h_blocks = grid_h / merge_size; + const int32_t w_blocks = grid_w / merge_size; + return (((static_cast(t) * h_blocks + h / merge_size) * w_blocks + w / merge_size) * merge_size + h % merge_size) + * merge_size + + w % merge_size; +} + +int64_t visualMergedIndex(int32_t t, int32_t h, int32_t w, int32_t grid_h, int32_t grid_w, int32_t merge_size) { + const int32_t h_blocks = grid_h / merge_size; + const int32_t w_blocks = grid_w / merge_size; + return (static_cast(t) * h_blocks + h) * w_blocks + w; +} + +Tensor makeGridThwTensor(int32_t grid_t, int32_t grid_h, int32_t grid_w) { + auto grid = Tensor::empty({1, 3}, mllm::kInt32, mllm::kCPU).alloc(); + grid.ptr()[0] = grid_t; + grid.ptr()[1] = grid_h; + grid.ptr()[2] = grid_w; + return grid; +} + +Tensor padVisualPatchesToBucket(Tensor img, Tensor grid_thw, const BucketGrid& bucket, const Qwen2VLConfig& cfg) { + const auto* grid = grid_thw.ptr(); + const int32_t grid_t = grid[0]; + const int32_t grid_h = grid[1]; + const int32_t grid_w = grid[2]; + if (bucket.grid_h < grid_h || bucket.grid_w < grid_w) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, + "Bucket {}x{} cannot cover original grid {}x{}.", + bucket.grid_h, + bucket.grid_w, + grid_h, + grid_w); + } + if (grid_h == bucket.grid_h && grid_w == bucket.grid_w) { return img; } + + const int32_t patch_dim = img.shape()[1]; + const auto elem_bytes = mllm::bytesOfType(img.dtype()); + auto padded = Tensor::empty({grid_t * bucket.grid_h * bucket.grid_w, patch_dim}, img.dtype(), mllm::kCPU).alloc(); + std::memset(padded.ptr(), 0, padded.bytes()); + + for (int32_t t = 0; t < grid_t; ++t) { + for (int32_t h = 0; h < grid_h; ++h) { + for (int32_t w = 0; w < grid_w; ++w) { + const auto src_idx = visualPatchIndex(t, h, w, grid_h, grid_w, cfg.visual_spatial_merge_size); + const auto dst_idx = visualPatchIndex(t, h, w, bucket.grid_h, bucket.grid_w, cfg.visual_spatial_merge_size); + std::memcpy(padded.offsettedPtr({static_cast(dst_idx), 0}), + img.offsettedPtr({static_cast(src_idx), 0}), + static_cast(patch_dim) * elem_bytes); + } + } + } + return padded; +} + +Tensor cropVisualEmbeddingsFromBucket(Tensor bucket_embeddings, + Tensor original_grid_thw, + Tensor bucket_grid_thw, + const Qwen2VLConfig& cfg) { + const auto* original_grid = original_grid_thw.ptr(); + const auto* bucket_grid = bucket_grid_thw.ptr(); + const int32_t grid_t = original_grid[0]; + const int32_t original_h = original_grid[1]; + const int32_t original_w = original_grid[2]; + const int32_t bucket_h = bucket_grid[1]; + const int32_t bucket_w = bucket_grid[2]; + if (original_h == bucket_h && original_w == bucket_w) { return bucket_embeddings; } + + const int32_t merge = cfg.visual_spatial_merge_size; + const int32_t hidden_size = bucket_embeddings.shape()[1]; + const auto elem_bytes = mllm::bytesOfType(bucket_embeddings.dtype()); + const int32_t original_merged_h = original_h / merge; + const int32_t original_merged_w = original_w / merge; + auto cropped = Tensor::empty({grid_t * original_merged_h * original_merged_w, hidden_size}, bucket_embeddings.dtype(), mllm::kCPU).alloc(); + + for (int32_t t = 0; t < grid_t; ++t) { + for (int32_t h = 0; h < original_merged_h; ++h) { + for (int32_t w = 0; w < original_merged_w; ++w) { + const auto src_idx = visualMergedIndex(t, h, w, bucket_h, bucket_w, merge); + const auto dst_idx = visualMergedIndex(t, h, w, original_h, original_w, merge); + std::memcpy(cropped.offsettedPtr({static_cast(dst_idx), 0}), + bucket_embeddings.offsettedPtr({static_cast(src_idx), 0}), + static_cast(hidden_size) * elem_bytes); + } + } + } + return cropped; +} + +std::pair makeVisualSinCos(const Qwen2VLConfig& cfg, Tensor grid_thw, int32_t patch_tokens) { + auto inv_freq = mllm::models::qwen2vl::makeVisualRoPEInvFreq(cfg.visual_embed_dim / cfg.visual_num_heads, 10000.0); + auto pos_ids = mllm::models::qwen2vl::makeVisualRotaryPosEmbIds(grid_thw, cfg.visual_spatial_merge_size); + auto rotary_pos_emb_full = mllm::models::qwen2vl::makeVisualRotaryPosEmbFull(inv_freq, patch_tokens); + auto pos_emb = mllm::models::qwen2vl::makeVisualRotaryPosEmb(rotary_pos_emb_full, pos_ids, grid_thw); + return mllm::models::qwen2vl::makeVisualRotarySinCos(pos_emb); +} + +Tensor runVisual(const std::string& model_path, + const std::string& model_version, + const Qwen2VLConfig& cfg, + Tensor img, + Tensor sin, + Tensor cos) { + mllm::ModelFileVersion file_version = mllm::ModelFileVersion::kV2; + if (model_version == "v1") { file_version = mllm::ModelFileVersion::kV1; } + auto params = mllm::load(model_path, file_version); + auto visual = mllm::models::qwen2vl::Qwen2VisionTransformerPretrainedModel("visual", cfg); + visual.load(params); + return visual(img, sin, cos)[0]; +} + +CompareStats compareFloatTensors(Tensor test_tensor, Tensor ref_tensor) { + MLLM_RT_ASSERT(test_tensor.shape() == ref_tensor.shape()); + MLLM_RT_ASSERT(test_tensor.dtype() == ref_tensor.dtype()); + CompareStats stats; + stats.numel = test_tensor.numel(); + double dot = 0.0; + double test_norm2 = 0.0; + double ref_norm2 = 0.0; + double diff_norm2 = 0.0; + float max_abs = 0.0f; + + auto accumulate = [&](auto* test, auto* ref) { + for (int64_t i = 0; i < stats.numel; ++i) { + const double a = static_cast(test[i]); + const double b = static_cast(ref[i]); + if (std::isnan(a) || std::isnan(b)) { + ++stats.nan_count; + continue; + } + if (std::isinf(a) || std::isinf(b)) { + ++stats.inf_count; + continue; + } + ++stats.finite_count; + const double diff = a - b; + dot += a * b; + test_norm2 += a * a; + ref_norm2 += b * b; + diff_norm2 += diff * diff; + max_abs = std::max(max_abs, static_cast(std::abs(diff))); + } + }; + + switch (test_tensor.dtype()) { + case mllm::kFloat32: accumulate(test_tensor.ptr(), ref_tensor.ptr()); break; + case mllm::kFloat16: accumulate(test_tensor.ptr(), ref_tensor.ptr()); break; + default: + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "Unsupported comparison dtype: {}", mllm::nameOfType(test_tensor.dtype())); + } + + if (test_norm2 == 0.0 && ref_norm2 == 0.0 && diff_norm2 == 0.0) { + stats.cosine = 1.0; + stats.rel_l2 = 0.0; + stats.norm_ratio = 1.0; + stats.max_abs_diff = 0.0f; + return stats; + } + + constexpr double eps = 1e-12; + const double test_l2 = std::sqrt(test_norm2); + stats.ref_l2 = std::sqrt(ref_norm2); + stats.l2 = std::sqrt(diff_norm2); + stats.cosine = dot / (test_l2 * stats.ref_l2 + eps); + stats.rel_l2 = stats.l2 / (stats.ref_l2 + eps); + stats.norm_ratio = test_l2 / (stats.ref_l2 + eps); + stats.max_abs_diff = max_abs; + return stats; +} + +void printShape(const std::string& name, const Tensor& tensor) { + fmt::print("{} dtype={} shape=[", name, mllm::nameOfType(tensor.dtype())); + for (size_t i = 0; i < tensor.shape().size(); ++i) { fmt::print("{}{}", i == 0 ? "" : ", ", tensor.shape()[i]); } + fmt::print("]\n"); +} + +} // namespace + +MLLM_MAIN({ + auto& help = Argparse::add("-h|--help").help("Show help message"); + auto& model_path = Argparse::add("-m|--model").help("visual-capable .mllm model").required(true); + auto& model_version = Argparse::add("-mv|--model_version").help("model file version: v1/v2").def("v2"); + auto& tokenizer_path = Argparse::add("-t|--tokenizer").help("tokenizer.json path").required(true); + auto& config_path = Argparse::add("-c|--config").help("Qwen2-VL config path").required(true); + auto& image_path = Argparse::add("-i|--image").help("input image path").required(true); + auto& prompt = Argparse::add("-p|--prompt").help("prompt text").def("describe this picture"); + auto& bucket_grid = Argparse::add("--bucket_grid").help("bucket patch grid HxW, e.g. 12x16").required(true); + + Argparse::parse(argc, argv); + if (help.isSet()) { + Argparse::printHelp(); + return 0; + } + + auto cfg = Qwen2VLConfig(config_path.get()); + auto tokenizer = Qwen2VLTokenizer(tokenizer_path.get()); + auto inputs = tokenizer.convertMessage({.prompt = prompt.get(), .img_file_path = image_path.get()}); + auto img = inputs.at("img"); + auto grid_thw = inputs.at("grid_thw"); + const auto* grid = grid_thw.ptr(); + const auto bucket = parseBucketGrid(bucket_grid.get()); + auto bucket_grid_thw = makeGridThwTensor(grid[0], bucket.grid_h, bucket.grid_w); + auto padded_img = padVisualPatchesToBucket(img, grid_thw, bucket, cfg); + + fmt::print("Qwen2-VL visual padding diagnostic\n"); + fmt::print("original grid: t={} h={} w={} patch_tokens={}\n", grid[0], grid[1], grid[2], img.shape()[0]); + fmt::print("bucket grid : t={} h={} w={} patch_tokens={}\n", grid[0], bucket.grid_h, bucket.grid_w, padded_img.shape()[0]); + printShape("img", img); + printShape("padded_img", padded_img); + + auto [sin, cos] = makeVisualSinCos(cfg, grid_thw, img.shape()[0]); + auto [bucket_sin, bucket_cos] = makeVisualSinCos(cfg, bucket_grid_thw, padded_img.shape()[0]); + + auto ref_embeddings = runVisual(model_path.get(), model_version.get(), cfg, img, sin, cos); + auto bucket_embeddings = runVisual(model_path.get(), model_version.get(), cfg, padded_img, bucket_sin, bucket_cos); + auto cropped_embeddings = cropVisualEmbeddingsFromBucket(bucket_embeddings, grid_thw, bucket_grid_thw, cfg); + + printShape("ref_embeddings", ref_embeddings); + printShape("bucket_embeddings", bucket_embeddings); + printShape("cropped_embeddings", cropped_embeddings); + + auto stats = compareFloatTensors(cropped_embeddings, ref_embeddings); + fmt::print(fg(fmt::color::cyan), "\n{:=^58}\n", " Padding Contamination "); + fmt::print("numel {:>16}\n", stats.numel); + fmt::print("finite count {:>16}\n", stats.finite_count); + fmt::print("NaN count {:>16}\n", stats.nan_count); + fmt::print("Inf count {:>16}\n", stats.inf_count); + fmt::print("cosine {:>16.8f}\n", stats.cosine); + fmt::print("relative L2 {:>16.8f}\n", stats.rel_l2); + fmt::print("norm ratio {:>16.8f}\n", stats.norm_ratio); + fmt::print("ref L2 {:>16.8f}\n", stats.ref_l2); + fmt::print("diff L2 {:>16.8f}\n", stats.l2); + fmt::print("max abs diff {:>16.6f}\n", stats.max_abs_diff); + fmt::print(fg(fmt::color::cyan), "{:=^58}\n", ""); + return 0; +}); diff --git a/mllm/backends/cpu/ops/VisionRoPEOp.cpp b/mllm/backends/cpu/ops/VisionRoPEOp.cpp index c10c59c44..c277eb713 100644 --- a/mllm/backends/cpu/ops/VisionRoPEOp.cpp +++ b/mllm/backends/cpu/ops/VisionRoPEOp.cpp @@ -26,16 +26,32 @@ void Qwen2VLVisionRoPEOpImpl::forward(const Tensor& activation, const Tensor& si auto S = activation.shape()[1]; auto H = activation.shape()[2]; auto D = activation.shape()[3]; + auto activation_ptr = activation.ptr(); + auto output_ptr = out.ptr(); + auto sin_ptr = sin.ptr(); + auto cos_ptr = cos.ptr(); + auto half_dim = D / 2; #if defined(MLLM_HOST_ARCH_X86_64) || defined(MLLM_HOST_ARCH_X86) - NYI("Qwen2VLVisionRoPEOpImpl is not implemented for x86"); + for (int b = 0; b < B; ++b) { + for (int s = 0; s < S; ++s) { + for (int h = 0; h < H; ++h) { + auto act_base = activation_ptr + b * S * H * D + s * H * D + h * D; + auto out_base = output_ptr + b * S * H * D + s * H * D + h * D; + auto sin_base = sin_ptr + s * half_dim; + auto cos_base = cos_ptr + s * half_dim; + + for (int d = 0; d < half_dim; ++d) { + const float a = static_cast(act_base[d]); + const float b_val = static_cast(act_base[d + half_dim]); + const float cos_val = static_cast(cos_base[d]); + const float sin_val = static_cast(sin_base[d]); + out_base[d] = static_cast(a * cos_val - b_val * sin_val); + out_base[d + half_dim] = static_cast(a * sin_val + b_val * cos_val); + } + } + } + } #elif defined(MLLM_HOST_ARCH_ARM64) || defined(MLLM_HOST_ARCH_ARM) - auto activation_ptr = activation.ptr(); - auto output_ptr = out.ptr(); - auto sin_ptr = sin.ptr(); - auto cos_ptr = cos.ptr(); - - auto half_dim = D / 2; - for (int b = 0; b < B; ++b) { for (int s = 0; s < S; ++s) { for (int h = 0; h < H; ++h) { @@ -89,7 +105,25 @@ void Qwen2VLVisionRoPEOpImpl::forward(const Tensor& activation, const Tensor& si auto cos_ptr = cos.ptr(); #if defined(MLLM_HOST_ARCH_X86_64) || defined(MLLM_HOST_ARCH_X86) - NYI("Qwen2VLVisionRoPEOpImpl is not implemented for x86"); + for (int b = 0; b < B; ++b) { + for (int s = 0; s < S; ++s) { + for (int h = 0; h < H; ++h) { + auto act_base = activation_ptr + b * S * H * D + s * H * D + h * D; + auto out_base = output_ptr + b * S * H * D + s * H * D + h * D; + auto sin_base = sin_ptr + s * half_dim; + auto cos_base = cos_ptr + s * half_dim; + + for (int d = 0; d < half_dim; ++d) { + const float a_front = act_base[d]; + const float a_back = act_base[d + half_dim]; + const float cos_val = cos_base[d]; + const float sin_val = sin_base[d]; + out_base[d] = a_front * cos_val - a_back * sin_val; + out_base[d + half_dim] = a_front * sin_val + a_back * cos_val; + } + } + } + } #elif defined(MLLM_HOST_ARCH_ARM64) || defined(MLLM_HOST_ARCH_ARM) for (int b = 0; b < B; ++b) { for (int s = 0; s < S; ++s) { diff --git a/mllm/models/ARGeneration.cpp b/mllm/models/ARGeneration.cpp index 001706167..bd27a6785 100644 --- a/mllm/models/ARGeneration.cpp +++ b/mllm/models/ARGeneration.cpp @@ -69,6 +69,7 @@ void ARGenerationChatIterator::step() { } if (step_count_ >= max_length_) { + if (step_count_ > 1) { gen_->decodeEventEndTimePoint(); } finished_ = true; return; } @@ -286,23 +287,50 @@ void ARGeneration::perfSummary() { auto ttft_duration = std::chrono::duration_cast(llm_decode_start_time_ - llm_prefill_start_time_).count(); + int64_t vit_duration = 0; + auto vit_event = custom_event_time_.find("ViT"); + if (vit_event != custom_event_time_.end()) { + vit_duration = + std::chrono::duration_cast(vit_event->second.second - vit_event->second.first).count(); + } + auto llm_prefill_duration = prefill_duration; + if (vit_duration > 0 && prefill_duration > vit_duration) { llm_prefill_duration = prefill_duration - vit_duration; } + double llm_prefill_tokens_per_sec = 0; + if (llm_prefill_duration > 0) { + llm_prefill_tokens_per_sec = (double)ar_prefill_tokens_ / (llm_prefill_duration / 1000000.0); + } + const double prefill_seconds = static_cast(prefill_duration) / 1000000.0; + const double llm_prefill_seconds = static_cast(llm_prefill_duration) / 1000000.0; + const double decode_seconds = static_cast(decode_duration) / 1000000.0; + const double total_seconds = static_cast(total_duration) / 1000000.0; + const double ttft_seconds = static_cast(ttft_duration) / 1000000.0; + const double avg_decode_seconds = avg_decode_duration / 1000000.0; fmt::print(fg(fmt::color::cyan), "\n{:=^50}\n", " Performance Summary "); fmt::print(fg(fmt::color::white), "{:<20}: ", "Total time"); - fmt::print(fg(fmt::color::yellow), "{:>10.2f} μs\n", (double)total_duration); + fmt::print(fg(fmt::color::yellow), "{:>10.3f} s\n", total_seconds); - fmt::print(fg(fmt::color::white), "{:<20}: ", "Prefill time"); - fmt::print(fg(fmt::color::yellow), "{:>10.2f} μs", (double)prefill_duration); + fmt::print(fg(fmt::color::white), "{:<20}: ", "Prefill total time"); + fmt::print(fg(fmt::color::yellow), "{:>10.3f} s", prefill_seconds); if (prefill_tokens_per_sec > 0) { fmt::print(fg(fmt::color::white), " ({:>6.2f} tokens/s)", prefill_tokens_per_sec); } fmt::print("\n"); + if (vit_duration > 0) { + fmt::print(fg(fmt::color::white), "{:<20}: ", "LLM prefill time"); + fmt::print(fg(fmt::color::yellow), "{:>10.3f} s", llm_prefill_seconds); + if (llm_prefill_tokens_per_sec > 0) { + fmt::print(fg(fmt::color::white), " ({:>6.2f} tokens/s)", llm_prefill_tokens_per_sec); + } + fmt::print("\n"); + } + fmt::print(fg(fmt::color::white), "{:<20}: ", "Decode time"); - fmt::print(fg(fmt::color::yellow), "{:>10.2f} μs", (double)decode_duration); + fmt::print(fg(fmt::color::yellow), "{:>10.3f} s", decode_seconds); if (decode_tokens_per_sec > 0) { fmt::print(fg(fmt::color::white), " ({:>6.2f} tokens/s)", decode_tokens_per_sec); } fmt::print("\n"); fmt::print(fg(fmt::color::white), "{:<20}: ", "TTFT"); - fmt::print(fg(fmt::color::magenta), "{:>10.2f} μs\n", (double)ttft_duration); + fmt::print(fg(fmt::color::magenta), "{:>10.3f} s\n", ttft_seconds); fmt::print(fg(fmt::color::white), "{:<20}: ", "Prefill tokens"); fmt::print(fg(fmt::color::green), "{:>10}\n", ar_prefill_tokens_); @@ -312,7 +340,7 @@ void ARGeneration::perfSummary() { if (ar_steps_ > 1) { fmt::print(fg(fmt::color::white), "{:<20}: ", "Avg decode time"); - fmt::print(fg(fmt::color::yellow), "{:>10.2f} μs/token\n", avg_decode_duration); + fmt::print(fg(fmt::color::yellow), "{:>10.3f} s/token\n", avg_decode_seconds); } fmt::print(fg(fmt::color::cyan), "{:=^50}\n", ""); diff --git a/mllm/models/qwen2vl/image_preprocessor_qwen2vl.hpp b/mllm/models/qwen2vl/image_preprocessor_qwen2vl.hpp index d5dc1b8a4..3a20064b2 100644 --- a/mllm/models/qwen2vl/image_preprocessor_qwen2vl.hpp +++ b/mllm/models/qwen2vl/image_preprocessor_qwen2vl.hpp @@ -65,6 +65,20 @@ class Qwen2VLImagePreprocessor { auto old_w = img.w(); auto old_h = img.h(); auto [new_h, new_w] = smartResize(old_h, old_w, 28, min_pixels_, max_pixels_); + return processResizedImage(img, new_h, new_w); + } + + inline std::pair operator()(const std::string& image_path, int32_t grid_h, int32_t grid_w) { + if (grid_h <= 0 || grid_w <= 0 || grid_h % merge_size_ != 0 || grid_w % merge_size_ != 0) { + MLLM_ERROR_EXIT(ExitCode::kIOError, "invalid Qwen2VL image grid override {}x{}", grid_h, grid_w); + } + + auto img = Image::open(image_path); + return processResizedImage(img, grid_h * patch_size_, grid_w * patch_size_); + } + + private: + inline std::pair processResizedImage(Image& img, int32_t new_h, int32_t new_w) { img = img.resize(new_w, new_h); // Process patches @@ -113,7 +127,6 @@ class Qwen2VLImagePreprocessor { return {flatten_patches, grid_thw}; } - private: Tensor OPENAI_CLIP_MEAN; Tensor OPENAI_CLIP_STD; int32_t merge_size_ = 2; diff --git a/mllm/models/qwen2vl/modeling_qwen2vl.hpp b/mllm/models/qwen2vl/modeling_qwen2vl.hpp index df0d247e7..81084a229 100644 --- a/mllm/models/qwen2vl/modeling_qwen2vl.hpp +++ b/mllm/models/qwen2vl/modeling_qwen2vl.hpp @@ -728,11 +728,13 @@ class Qwen2VLForCausalLM : public ARGeneration { auto pos_emb = makeVisualRotaryPosEmb(rotary_pos_emb_full, pos_ids, grid_thw); auto [visual_embedding_sin, visual_embedding_cos] = makeVisualRotarySinCos(pos_emb); + customEventStartTimePoint("ViT"); auto start_time = std::chrono::high_resolution_clock::now(); auto visual_embeddings = visual(img, visual_embedding_sin, visual_embedding_cos)[0]; auto end_time = std::chrono::high_resolution_clock::now(); - auto all_time = std::chrono::duration_cast(end_time - start_time); - print("ViT Processing: done, time cost: {} seconds", all_time.count()); + customEventEndTimePoint("ViT"); + auto all_time = std::chrono::duration(end_time - start_time).count(); + fmt::print("ViT Processing: done, time cost: {:.3f} seconds\n", all_time); // Insert visual embeddings into llm's embedding int32_t vision_pad_token_start = -1; diff --git a/mllm/models/qwen2vl/modeling_qwen2vl_traceable.hpp b/mllm/models/qwen2vl/modeling_qwen2vl_traceable.hpp index 05c009def..7a833d4a8 100644 --- a/mllm/models/qwen2vl/modeling_qwen2vl_traceable.hpp +++ b/mllm/models/qwen2vl/modeling_qwen2vl_traceable.hpp @@ -275,8 +275,8 @@ class PatchEmbed final : public nn::Module { auto hidden_states = inputs[0]; // [batch_size(x), in_channel(3), temporal_patch_size(2), patch_size(14), patch_size(14)] - hidden_states = hidden_states.view({-1, in_chans_, temporal_patch_size_, patch_size_, patch_size_}); - hidden_states = proj_(hidden_states).view({-1, embed_dim_}); + hidden_states = hidden_states.view({-1, in_chans_, temporal_patch_size_, patch_size_, patch_size_}, true); + hidden_states = proj_(hidden_states).view({-1, embed_dim_}, true); return {hidden_states}; } @@ -307,7 +307,7 @@ class PatchMerger final : public nn::Module { } std::vector forward(const std::vector& inputs, const std::vector& args) override { - auto o = ln_q_(inputs[0]).view({-1, hidden_size_}); + auto o = ln_q_(inputs[0]).view({-1, hidden_size_}, true); o = mlp_0_(o); o = mlp_gelu_(o); o = mlp_2_(o); @@ -350,8 +350,6 @@ class VisionAttention final : public nn::Module { nn::Linear qkv_; nn::Linear proj_; nn::Softmax softmax_; - nn::VisionRoPE vision_rope_q_; - nn::VisionRoPE vision_rope_k_; public: VisionAttention() = default; @@ -365,19 +363,25 @@ class VisionAttention final : public nn::Module { qkv_ = reg("qkv", dim_, dim_ * 3, true, cfg.linear_impl_type); proj_ = reg("proj", dim_, dim_, true, cfg.linear_impl_type); softmax_ = reg("softmax", -1); + } + + Tensor applyVisionRoPEPrimitive(Tensor x, Tensor visual_embedding_sin, Tensor visual_embedding_cos) { + const int32_t half_dim = head_dim_ / 2; + auto x1 = x.slice({kAll, kAll, kAll, {kAll, half_dim}}, true); + auto x2 = x.slice({kAll, kAll, kAll, {half_dim, kAll}}, true); + auto sin = visual_embedding_sin; + auto cos = visual_embedding_cos; + if (sin.rank() == 2) { + sin = sin.view({1, -1, 1, half_dim}, true); + cos = cos.view({1, -1, 1, half_dim}, true); + } else { + MLLM_RT_ASSERT_EQ(sin.rank(), 4); + MLLM_RT_ASSERT_EQ(cos.rank(), 4); + } - vision_rope_q_ = reg("vision_rope_q", aops::VisionRoPEOpOptionsType::kQwen2VL, - aops::Qwen2VLRoPEOpOptions{ - .dims = head_dim_, - .spatial_merge_size = cfg.visual_spatial_merge_size, - .theta = 10000.0, - }); - vision_rope_k_ = reg("vision_rope_k", aops::VisionRoPEOpOptionsType::kQwen2VL, - aops::Qwen2VLRoPEOpOptions{ - .dims = head_dim_, - .spatial_merge_size = cfg.visual_spatial_merge_size, - .theta = 10000.0, - }); + auto y1 = x1 * cos + (-(x2 * sin)); + auto y2 = x1 * sin + x2 * cos; + return nn::functional::concat({y1, y2}, -1); } std::vector forward(const std::vector& inputs, const std::vector& args) override { @@ -386,13 +390,15 @@ class VisionAttention final : public nn::Module { auto visual_embedding_sin = inputs[1]; auto visual_embedding_cos = inputs[2]; - auto [query_states, key_states, value_states] = - nn::functional::split<3>(qkv_(hidden_states).view({-1, 3, num_heads_, head_dim_}).permute({1, 0, 2, 3}), 1, 0); + auto qkv_states = qkv_(hidden_states).view({-1, 3, num_heads_, head_dim_}, true); + auto query_states = qkv_states.slice({kAll, {0, 1}, kAll, kAll}, true).transpose(0, 1); + auto key_states = qkv_states.slice({kAll, {1, 2}, kAll, kAll}, true).transpose(0, 1); + auto value_states = qkv_states.slice({kAll, {2, 3}, kAll, kAll}, true).transpose(0, 1); // Input to Vision ROPE must be BSHD format // grid_thw shape is [n, 3], n is always 1 in this case. - query_states = vision_rope_q_(query_states, visual_embedding_sin, visual_embedding_cos); - key_states = vision_rope_k_(key_states, visual_embedding_sin, visual_embedding_cos); + query_states = applyVisionRoPEPrimitive(query_states, visual_embedding_sin, visual_embedding_cos); + key_states = applyVisionRoPEPrimitive(key_states, visual_embedding_sin, visual_embedding_cos); // [B, H, S, D] query_states = query_states.transpose(1, 2); @@ -409,7 +415,7 @@ class VisionAttention final : public nn::Module { auto attn_output = nn::functional::matmul(attn, value_states); // [B=1, H, S, D] -> [B=1, S, H, D] -> [S, H * D] - attn_output = attn_output.transpose(1, 2).view({-1, dim_}); + attn_output = attn_output.transpose(1, 2).view({-1, dim_}, true); attn_output = proj_(attn_output); return { attn_output, @@ -718,11 +724,13 @@ class Qwen2VLForCausalLM : public ARGeneration { auto pos_emb = makeVisualRotaryPosEmb(rotary_pos_emb_full, pos_ids, grid_thw); auto [visual_embedding_sin, visual_embedding_cos] = makeVisualRotarySinCos(pos_emb); + customEventStartTimePoint("ViT"); auto start_time = std::chrono::high_resolution_clock::now(); auto visual_embeddings = visual(img, visual_embedding_sin, visual_embedding_cos)[0]; auto end_time = std::chrono::high_resolution_clock::now(); - auto all_time = std::chrono::duration_cast(end_time - start_time); - print("ViT Processing: done, time cost: {} seconds", all_time.count()); + customEventEndTimePoint("ViT"); + auto all_time = std::chrono::duration(end_time - start_time).count(); + fmt::print("ViT Processing: done, time cost: {:.3f} seconds\n", all_time); // Insert visual embeddings into llm's embedding int32_t vision_pad_token_start = -1; diff --git a/mllm/models/qwen2vl/tokenization_qwen2vl.hpp b/mllm/models/qwen2vl/tokenization_qwen2vl.hpp index 1ff6d034e..bf9606f3f 100644 --- a/mllm/models/qwen2vl/tokenization_qwen2vl.hpp +++ b/mllm/models/qwen2vl/tokenization_qwen2vl.hpp @@ -236,14 +236,16 @@ class Qwen2VLTokenizer final : public mllm::preprocessor::AutoTokenizer { return ret; } - ARGenerationOutputPast convertMessage(const Qwen2VLMessage& message) { + ARGenerationOutputPast convertMessage(const Qwen2VLMessage& message, int32_t image_grid_h = -1, int32_t image_grid_w = -1) { // process prompt auto applied_string = Qwen2VLMessage::message_template; size_t pos = applied_string.find("{{{prompt}}}"); applied_string.replace(pos, 12, message.prompt); // process image - auto [img, grid_thw] = image_preprocessor_(message.img_file_path); + auto [img, grid_thw] = image_grid_h > 0 || image_grid_w > 0 + ? image_preprocessor_(message.img_file_path, image_grid_h, image_grid_w) + : image_preprocessor_(message.img_file_path); // process sequence auto sequence_str = tokenize(applied_string); diff --git a/scripts/qwen2vl_qnn/common.sh b/scripts/qwen2vl_qnn/common.sh new file mode 100755 index 000000000..33823f69c --- /dev/null +++ b/scripts/qwen2vl_qnn/common.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -euo pipefail + +QWEN2VL_QNN_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +QWEN2VL_QNN_REPO_ROOT="$(cd "${QWEN2VL_QNN_SCRIPT_DIR}/../.." && pwd)" + +pick_adb() { + if [[ -n "${ADB:-}" ]]; then + printf '%s\n' "${ADB}" + return + fi + if command -v adb.exe >/dev/null 2>&1; then + command -v adb.exe + return + fi + if command -v adb >/dev/null 2>&1; then + command -v adb + return + fi + echo "Cannot find adb.exe or adb. Set ADB=/path/to/adb first." >&2 + return 1 +} + +remote_quote() { + local value="$1" + printf "'" + printf "%s" "${value}" | sed "s/'/'\\\\''/g" + printf "'" +} + +timestamp() { + date +%Y%m%d_%H%M%S +} + +read_eval_cases() { + local cases_file="$1" + if [[ ! -f "${cases_file}" ]]; then + echo "Cases file not found: ${cases_file}" >&2 + return 1 + fi + sed -e 's/\r$//' "${cases_file}" | awk 'NF && $0 !~ /^#/' +} diff --git a/scripts/qwen2vl_qnn/push_eval_assets.sh b/scripts/qwen2vl_qnn/push_eval_assets.sh new file mode 100755 index 000000000..b19271e73 --- /dev/null +++ b/scripts/qwen2vl_qnn/push_eval_assets.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +ADB_BIN="$(pick_adb)" +CASES_FILE="${CASES_FILE:-${SCRIPT_DIR}/qwen2vl_eval_cases.tsv}" +ASSET_SRC="${ASSET_SRC:-}" +REMOTE_IMAGE_DIR="${REMOTE_IMAGE_DIR:-/data/local/tmp/mllm-qwen2vl/images/eval}" + +if [[ -z "${ASSET_SRC}" ]]; then + echo "Set ASSET_SRC to the local directory that contains the eval images." >&2 + echo "Example: ASSET_SRC=/path/to/images $0" >&2 + exit 1 +fi + +echo "ADB : ${ADB_BIN}" +echo "Cases file : ${CASES_FILE}" +echo "Local assets : ${ASSET_SRC}" +echo "Remote image dir : ${REMOTE_IMAGE_DIR}" + +"${ADB_BIN}" shell "mkdir -p $(remote_quote "${REMOTE_IMAGE_DIR}")" &2 + exit 1 + fi + echo "Pushing ${image_file}" + "${ADB_BIN}" push "${local_image}" "${REMOTE_IMAGE_DIR}/${image_file}" "${OUT_DIR}/run_config.txt" <"${status_file}" + +while IFS='|' read -r case_id image_file prompt; do + remote_image="${REMOTE_IMAGE_DIR}/${image_file}" + log_file="${OUT_DIR}/${case_id}.log" + visual_args="" + if [[ "${VISUAL_QNN}" == "1" ]]; then + visual_args="--visual_qnn --visual_bundle_layout $(remote_quote "${VISUAL_BUNDLE_LAYOUT}")" + if [[ -n "${VISUAL_BUCKET_GRIDS}" ]]; then + visual_args="${visual_args} --visual_bucket_grids $(remote_quote "${VISUAL_BUCKET_GRIDS}")" + fi + else + visual_args="--visual_model $(remote_quote "${VISUAL_MODEL}") --visual_model_version $(remote_quote "${VISUAL_MODEL_VERSION}")" + fi + + echo + echo "==> ${case_id}" + echo "image : ${remote_image}" + echo "prompt: ${prompt}" + echo "log : ${log_file}" + + remote_cmd=" +cd $(remote_quote "${REMOTE_QNN_DIR}") && +export MLLM_QWEN2VL_AOT_DUMP_STATS=$(remote_quote "${DUMP_STATS}") && +export LD_LIBRARY_PATH=\$PWD/lib:${REMOTE_CPU_DIR}/lib:/vendor/lib64:/system/lib64:\$LD_LIBRARY_PATH && +export ADSP_LIBRARY_PATH=\"\$PWD/lib;/vendor/dsp/cdsp;/vendor/lib/rfsa/adsp;/system/lib/rfsa/adsp;/dsp\" && +./bin/mllm-qwen2vl-aot-runner \ + -m $(remote_quote "${CONTEXT}") \ + --qnn_params $(remote_quote "${QNN_PARAMS}") \ + ${visual_args} \ + -t $(remote_quote "${TOKENIZER}") \ + -c $(remote_quote "${QNN_CONFIG}") \ + --visual_config $(remote_quote "${VISUAL_CONFIG}") \ + -i $(remote_quote "${remote_image}") \ + -p $(remote_quote "${prompt}") \ + --ar_len ${AR_LEN} \ + --context_len ${CONTEXT_LEN} \ + --gen_len ${GEN_LEN} \ + --input_embedding_scale ${INPUT_EMBEDDING_SCALE} \ + --input_embedding_zero_point ${INPUT_EMBEDDING_ZERO_POINT} \ + --key_cache_dtype $(remote_quote "${KEY_CACHE_DTYPE}") +" + + if "${ADB_BIN}" shell "${remote_cmd}" &1 | tee "${log_file}"; then + printf "%s\t%s\tOK\t%s\n" "${case_id}" "${image_file}" "${log_file}" >>"${status_file}" + else + printf "%s\t%s\tFAIL\t%s\n" "${case_id}" "${image_file}" "${log_file}" >>"${status_file}" + fi +done < <(read_eval_cases "${CASES_FILE}") + +summary_file="${OUT_DIR}/summary.tsv" +python3 "${QWEN2VL_QNN_REPO_ROOT}/tools/qnn_debug/summarize_qwen2vl_eval_logs.py" \ + --log_dir "${OUT_DIR}" \ + --cases "${CASES_FILE}" \ + --out "${summary_file}" || true + +echo +echo "Logs : ${OUT_DIR}" +echo "Status : ${status_file}" +echo "Summary : ${summary_file}" diff --git a/scripts/qwen2vl_qnn/run_qnn_interactive.sh b/scripts/qwen2vl_qnn/run_qnn_interactive.sh new file mode 100755 index 000000000..ee1a772bc --- /dev/null +++ b/scripts/qwen2vl_qnn/run_qnn_interactive.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${SCRIPT_DIR}/common.sh" + +ADB_BIN="$(pick_adb)" +REMOTE_QNN_DIR="${REMOTE_QNN_DIR:-/data/local/tmp/mllm-qwen2vl-qnn}" +REMOTE_CPU_DIR="${REMOTE_CPU_DIR:-/data/local/tmp/mllm-qwen2vl}" +REMOTE_IMAGE_DIR="${REMOTE_IMAGE_DIR:-${REMOTE_CPU_DIR}/images/eval}" + +LOCAL_RUNNER="${LOCAL_RUNNER:-${QWEN2VL_QNN_REPO_ROOT}/build-android-arm64-v8a-qnn/bin/mllm-qwen2vl-aot-runner}" +PUSH_RUNNER="${PUSH_RUNNER:-1}" + +CONTEXT="${CONTEXT:-models/qwen2vl-2b-sm8650-qnn234-fullqnn-5bucket-baseline-v1.bin}" +QNN_PARAMS="${QNN_PARAMS:-models/qwen2vl-2b-sm8650-qnn234-lpbq-baseline-v1.mllm}" +TOKENIZER="${TOKENIZER:-${REMOTE_CPU_DIR}/tokenizer/tokenizer.json}" +QNN_CONFIG="${QNN_CONFIG:-config/config_2B_qnn_lpbq.json}" +VISUAL_MODEL="${VISUAL_MODEL:-${REMOTE_CPU_DIR}/models/qwen2vl-2b-w4a32-kai.mllm}" +VISUAL_CONFIG="${VISUAL_CONFIG:-${REMOTE_CPU_DIR}/config/config_2B_w32a32.json}" +VISUAL_MODEL_VERSION="${VISUAL_MODEL_VERSION:-v2}" +VISUAL_QNN="${VISUAL_QNN:-1}" +VISUAL_BUNDLE_LAYOUT="${VISUAL_BUNDLE_LAYOUT:-single}" +VISUAL_BUCKET_GRIDS="${VISUAL_BUCKET_GRIDS:-12x16,16x24,24x24,24x32,18x52,52x18,26x36,36x26}" + +AR_LEN="${AR_LEN:-32}" +CONTEXT_LEN="${CONTEXT_LEN:-1024}" +GEN_LEN="${GEN_LEN:-1000}" +PROMPT="${PROMPT:-describe this picture}" +INPUT_EMBEDDING_SCALE="${INPUT_EMBEDDING_SCALE:-0.002563515}" +INPUT_EMBEDDING_ZERO_POINT="${INPUT_EMBEDDING_ZERO_POINT:-15604}" +KEY_CACHE_DTYPE="${KEY_CACHE_DTYPE:-uint8}" +DUMP_STATS="${DUMP_STATS:-1}" +ADB_SHELL_TTY="${ADB_SHELL_TTY:-1}" + +if [[ "${PUSH_RUNNER}" != "0" ]]; then + if [[ ! -f "${LOCAL_RUNNER}" ]]; then + echo "Runner not found: ${LOCAL_RUNNER}" >&2 + echo "Build it first, or set PUSH_RUNNER=0 if the phone already has the correct binary." >&2 + exit 1 + fi + "${ADB_BIN}" shell "mkdir -p $(remote_quote "${REMOTE_QNN_DIR}/bin")" to quit." + +remote_cmd=" +cd $(remote_quote "${REMOTE_QNN_DIR}") && +export MLLM_QWEN2VL_AOT_DUMP_STATS=$(remote_quote "${DUMP_STATS}") && +export LD_LIBRARY_PATH=\$PWD/lib:${REMOTE_CPU_DIR}/lib:/vendor/lib64:/system/lib64:\$LD_LIBRARY_PATH && +export ADSP_LIBRARY_PATH=\"\$PWD/lib;/vendor/dsp/cdsp;/vendor/lib/rfsa/adsp;/system/lib/rfsa/adsp;/dsp\" && +exec ./bin/mllm-qwen2vl-aot-runner \ + -m $(remote_quote "${CONTEXT}") \ + --qnn_params $(remote_quote "${QNN_PARAMS}") \ + ${visual_args} \ + -t $(remote_quote "${TOKENIZER}") \ + -c $(remote_quote "${QNN_CONFIG}") \ + --visual_config $(remote_quote "${VISUAL_CONFIG}") \ + -p $(remote_quote "${PROMPT}") \ + --interactive \ + --ar_len ${AR_LEN} \ + --context_len ${CONTEXT_LEN} \ + --gen_len ${GEN_LEN} \ + --input_embedding_scale ${INPUT_EMBEDDING_SCALE} \ + --input_embedding_zero_point ${INPUT_EMBEDDING_ZERO_POINT} \ + --key_cache_dtype $(remote_quote "${KEY_CACHE_DTYPE}") +" + +if [[ "${ADB_SHELL_TTY}" == "1" ]]; then + "${ADB_BIN}" shell -t "${remote_cmd}" +else + "${ADB_BIN}" shell "${remote_cmd}" +fi From 7cef93bdf596bd4308e5172df11fe5315b4eedaa Mon Sep 17 00:00:00 2001 From: twlddd Date: Thu, 21 May 2026 12:04:55 +0800 Subject: [PATCH 4/7] Update Qwen2-VL QNN example baseline to FP16 visual --- ...n_aot_cfg_2B_lpbq_vprojg16_unsignedpd.json | 57 + .../qwen2vl/qnn_aot_cfg_2B_visual_fp16.json | 30 + examples/qwen2vl_qnn_aot/README.md | 25 +- examples/qwen2vl_qnn_aot/aot_run.cpp | 1021 ++++++++++++++--- examples/qwen2vl_qnn_aot/compile.cpp | 470 +++++++- .../qwen2vl_qnn_aot/visual_aot_helpers.hpp | 52 + mllm/core/aops/ElewiseOps.cpp | 6 + scripts/qwen2vl_qnn/run_qnn_eval_fixed_set.sh | 19 +- scripts/qwen2vl_qnn/run_qnn_interactive.sh | 16 +- 9 files changed, 1481 insertions(+), 215 deletions(-) create mode 100644 examples/qwen2vl/qnn_aot_cfg_2B_lpbq_vprojg16_unsignedpd.json create mode 100644 examples/qwen2vl/qnn_aot_cfg_2B_visual_fp16.json diff --git a/examples/qwen2vl/qnn_aot_cfg_2B_lpbq_vprojg16_unsignedpd.json b/examples/qwen2vl/qnn_aot_cfg_2B_lpbq_vprojg16_unsignedpd.json new file mode 100644 index 000000000..cbb62f1d5 --- /dev/null +++ b/examples/qwen2vl/qnn_aot_cfg_2B_lpbq_vprojg16_unsignedpd.json @@ -0,0 +1,57 @@ +{ + "target_machine": { + "htp_arch": "V75", + "htp_chipset": "SM8650", + "htp_try_best_performance": "HtpBurst", + "htp_security_pd_session": "HtpUnsignedPd", + "htp_vtcm_capability_in_mb": 8 + }, + "graph_on_qnn": [ + "model" + ], + "op_on_qnn": [ + "lm_head" + ], + "split_graph": 1, + "quant_recipe": { + "llm_recipe": true, + "layers": 28, + "builtin_llm_pass": { + "model": "qwen2", + "lm_head": { + "fallback": { + "method": "LPBQ", + "sym": true, + "precision": "w4a16", + "block_size": 32 + } + }, + "linear": { + "fallback": { + "method": "LPBQ", + "sym": true, + "precision": "w4a16", + "block_size": 32 + }, + "model\\.layers\\.[0-9]+\\.self_attn\\.v_proj": { + "method": "LPBQ", + "sym": true, + "precision": "w4a16", + "block_size": 16 + } + }, + "kv_cache": { + "key": { + "method": "per-tensor", + "sym": true, + "precision": "w8a8" + }, + "value": { + "method": "per-tensor", + "sym": true, + "precision": "w8a8" + } + } + } + } +} diff --git a/examples/qwen2vl/qnn_aot_cfg_2B_visual_fp16.json b/examples/qwen2vl/qnn_aot_cfg_2B_visual_fp16.json new file mode 100644 index 000000000..8ef78e133 --- /dev/null +++ b/examples/qwen2vl/qnn_aot_cfg_2B_visual_fp16.json @@ -0,0 +1,30 @@ +{ + "target_machine": { + "htp_arch": "V75", + "htp_chipset": "SM8650", + "htp_try_best_performance": "HtpBurst", + "htp_security_pd_session": "HtpUnsignedPd", + "htp_vtcm_capability_in_mb": 8 + }, + "graph_on_qnn": [ + "visual" + ], + "op_on_qnn": [], + "split_graph": 1, + "quant_recipe": { + "llm_recipe": true, + "layers": 32, + "builtin_llm_pass": { + "model": "qwen2vl_visual", + "linear": { + "fallback": { + "method": "LPBQ", + "sym": true, + "precision": "w4a16", + "block_size": 32, + "allow_raw_float_linear": true + } + } + } + } +} diff --git a/examples/qwen2vl_qnn_aot/README.md b/examples/qwen2vl_qnn_aot/README.md index fe4946344..f86178042 100644 --- a/examples/qwen2vl_qnn_aot/README.md +++ b/examples/qwen2vl_qnn_aot/README.md @@ -9,6 +9,14 @@ This example demonstrates full QNN AOT inference for Qwen2-VL style models: Model weights and compiled QNN context binaries are not stored in this repository. +The recommended Qwen2-VL-2B baseline tested on 2026-05-21 uses: + +- LPBQ / W4A16 LLM weights +- FP16 visual encoder weights +- one combined QNN context with `model.0.s32`, `model.0.s1`, and five visual bucket graphs +- `--visual_bundle_layout single` +- `--visual_io_dtype fp16` + ## Build Build the x86 AOT compiler: @@ -30,16 +38,18 @@ QNN x86 backend to generate a context binary for Android. ```bash ./build-qnn-aot/bin/mllm-qwen2vl-aot-c \ - --model_path path/to/qwen2vl-qnn-lpbq.mllm \ + --model_path path/to/qwen2vl-2b-sm8650-qnn234-lpbq-visualfp16-vprojg16.mllm \ --config examples/qwen2vl/config_2B_qnn_lpbq.json \ - --aot_config examples/qwen2vl/qnn_aot_cfg_2B.json \ - --output_context_name qwen2vl-full-qnn.bin \ + --aot_config examples/qwen2vl/qnn_aot_cfg_2B_lpbq_vprojg16_unsignedpd.json \ + --output_context_name qwen2vl-2b-sm8650-qnn234-fullqnn-5bucket-visualfp16-v1.bin \ --context_len 1024 \ --prefill_len 32 \ --include_visual_bundle \ - --visual_model path/to/qwen2vl-visual.mllm \ + --visual_model path/to/qwen2vl-2b-sm8650-qnn234-lpbq-visualfp16-vprojg16.mllm \ --visual_config examples/qwen2vl/config_2B_qnn_lpbq.json \ - --visual_aot_config examples/qwen2vl/qnn_aot_cfg_2B_visual.json \ + --visual_aot_config examples/qwen2vl/qnn_aot_cfg_2B_visual_fp16.json \ + --visual_bundle_layout single \ + --visual_io_dtype fp16 \ --visual_bucket_grids 12x16,16x24,24x24,24x32,18x52,52x18,26x36,36x26 ``` @@ -56,8 +66,9 @@ session. Set paths through environment variables as needed: ```bash REMOTE_QNN_DIR=/data/local/tmp/mllm-qwen2vl-qnn \ REMOTE_CPU_DIR=/data/local/tmp/mllm-qwen2vl \ -CONTEXT=models/qwen2vl-full-qnn.bin \ -QNN_PARAMS=models/qwen2vl-qnn-lpbq.mllm \ +CONTEXT=models/qwen2vl-2b-sm8650-qnn234-fullqnn-5bucket-visualfp16-v1.bin \ +QNN_PARAMS=models/qwen2vl-2b-sm8650-qnn234-lpbq-visualfp16-vprojg16.mllm \ +VISUAL_IO_DTYPE=fp16 \ scripts/qwen2vl_qnn/run_qnn_interactive.sh ``` diff --git a/examples/qwen2vl_qnn_aot/aot_run.cpp b/examples/qwen2vl_qnn_aot/aot_run.cpp index f9f267488..9d58cf0a3 100644 --- a/examples/qwen2vl_qnn_aot/aot_run.cpp +++ b/examples/qwen2vl_qnn_aot/aot_run.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -46,12 +47,26 @@ namespace { constexpr float kDefaultInputEmbeddingScale = 0.002563515f; constexpr int32_t kDefaultInputEmbeddingZeroPoint = 15604; +constexpr float kDefaultVisualPatchInputScale = 4.4f / 65535.f; +constexpr int32_t kDefaultVisualPatchInputZeroPoint = 32768; +constexpr float kDefaultVisualSinCosScale = 2.0f / 65535.f; +constexpr int32_t kDefaultVisualSinCosZeroPoint = 32768; +constexpr float kDefaultVisualAttentionMaskScale = 10000.0f / 65535.f; +constexpr int32_t kDefaultVisualAttentionMaskZeroPoint = 65535; +constexpr float kDefaultVisualOutputScale = 0.00041101f; +constexpr int32_t kDefaultVisualOutputZeroPoint = 32768; struct QuantParams { float scale = 1.f; int32_t zero_point = 0; }; +enum class VisualIODType { + kUInt16, + kFloat32, + kFloat16, +}; + struct LogitEntry { int32_t token_id = 0; uint16_t raw = 0; @@ -94,6 +109,21 @@ struct VisualResizeOverride { VisualBucketGrid bucket; }; +class Qwen2VLPatchEmbedOnly final : public mllm::nn::Module { + mllm::models::qwen2vl::PatchEmbed patch_embed_; + + public: + Qwen2VLPatchEmbedOnly() = default; + + Qwen2VLPatchEmbedOnly(const std::string& name, const Qwen2VLConfig& cfg) : mllm::nn::Module(name) { + patch_embed_ = reg("patch_embed", cfg); + } + + std::vector forward(const std::vector& inputs, const std::vector& /*args*/) override { + return patch_embed_(inputs[0]); + } +}; + std::string trimInteractiveLine(std::string line) { while (!line.empty() && (line.back() == '\n' || line.back() == '\r' || std::isspace(static_cast(line.back())))) { line.pop_back(); @@ -112,23 +142,136 @@ int32_t readScalarInt32(const mllm::ParameterFile::ptr_t& params, const std::str return params->pull(name).ptr()[0]; } +bool hasQuantParams(const mllm::ParameterFile::ptr_t& params, + const std::string& scale_name, + const std::string& zp_name) { + return params->has(scale_name) && params->has(zp_name); +} + QuantParams readQuantParams(const mllm::ParameterFile::ptr_t& params, const std::string& scale_name, const std::string& zp_name) { return {.scale = readScalarFloat(params, scale_name), .zero_point = readScalarInt32(params, zp_name)}; } +VisualIODType parseVisualIODType(const std::string& dtype) { + if (dtype == "uint16") { return VisualIODType::kUInt16; } + if (dtype == "fp32" || dtype == "float32") { return VisualIODType::kFloat32; } + if (dtype == "fp16" || dtype == "float16") { return VisualIODType::kFloat16; } + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "--visual_io_dtype must be uint16, fp32, or fp16."); +} + +mllm::DataTypes visualIODTypeToDataType(VisualIODType dtype) { + switch (dtype) { + case VisualIODType::kUInt16: return mllm::kUInt16PerTensorAsy; + case VisualIODType::kFloat32: return mllm::kFloat32; + case VisualIODType::kFloat16: return mllm::kFloat16; + } + return mllm::kUInt16PerTensorAsy; +} + +bool isRawFloatVisualIO(VisualIODType dtype) { return dtype == VisualIODType::kFloat32 || dtype == VisualIODType::kFloat16; } + uint16_t quantizeUInt16(float value, const QuantParams& qp) { auto q = static_cast(std::llround(value / qp.scale) + qp.zero_point); q = std::max(0, std::min(65535, q)); return static_cast(q); } +float dequantizeUInt16(uint16_t value, const QuantParams& qp) { + return (static_cast(value) - qp.zero_point) * qp.scale; +} + bool shouldDumpQwen2VLAOTStats() { const char* flag = std::getenv("MLLM_QWEN2VL_AOT_DUMP_STATS"); return flag != nullptr && std::string(flag) != "0"; } +bool shouldProfileQwen2VLAOT() { + const char* flag = std::getenv("MLLM_QWEN2VL_AOT_PROFILE"); + return flag != nullptr && std::string(flag) != "0"; +} + +std::string firstHybridBodyInputQDQName() { return "visual.blocks.0.attn.qkv_input_qdq"; } + +struct ProfileStage { + std::string name; + int64_t total_us = 0; + int64_t max_us = 0; + int64_t count = 0; +}; + +class RequestProfiler { + public: + explicit RequestProfiler(bool enabled = false) : enabled_(enabled) {} + + [[nodiscard]] bool enabled() const { return enabled_; } + + void reset() { + stages_.clear(); + stage_index_.clear(); + } + + void add(const std::string& name, int64_t us) { + if (!enabled_) { return; } + auto it = stage_index_.find(name); + if (it == stage_index_.end()) { + const auto idx = stages_.size(); + stage_index_[name] = idx; + stages_.push_back({.name = name}); + it = stage_index_.find(name); + } + auto& stage = stages_[it->second]; + stage.total_us += us; + stage.max_us = std::max(stage.max_us, us); + stage.count += 1; + } + + void printSummary() const { + if (!enabled_ || stages_.empty()) { return; } + fmt::print(fg(fmt::color::yellow), "\n{:=^72}\n", " Qwen2VL Stage Profile "); + fmt::print("[QWEN2VL_PROFILE] {:<42} {:>8} {:>12} {:>12} {:>12}\n", "stage", "count", "total_us", "avg_us", "max_us"); + for (const auto& stage : stages_) { + const double avg_us = stage.count > 0 ? static_cast(stage.total_us) / static_cast(stage.count) : 0.0; + fmt::print("[QWEN2VL_PROFILE] {:<42} {:>8} {:>12} {:>12.2f} {:>12}\n", + stage.name, + stage.count, + stage.total_us, + avg_us, + stage.max_us); + } + fmt::print(fg(fmt::color::yellow), "{:=^72}\n", ""); + } + + private: + bool enabled_ = false; + std::vector stages_; + std::unordered_map stage_index_; +}; + +class ScopedProfile { + public: + ScopedProfile(RequestProfiler* profiler, std::string name) + : profiler_(profiler), name_(std::move(name)), enabled_(profiler_ != nullptr && profiler_->enabled()) { + if (enabled_) { start_ = std::chrono::high_resolution_clock::now(); } + } + + ~ScopedProfile() { + if (!enabled_) { return; } + const auto end = std::chrono::high_resolution_clock::now(); + profiler_->add(name_, std::chrono::duration_cast(end - start_).count()); + } + + ScopedProfile(const ScopedProfile&) = delete; + ScopedProfile& operator=(const ScopedProfile&) = delete; + + private: + RequestProfiler* profiler_ = nullptr; + std::string name_; + bool enabled_ = false; + std::chrono::high_resolution_clock::time_point start_; +}; + std::vector parseVisualBucketGrids(const std::string& text) { std::vector buckets; if (text.empty()) { return buckets; } @@ -183,6 +326,36 @@ int32_t floorEvenGrid(double value) { return std::max(4, grid); } +std::pair smartResizePixelsForTokenizer(int32_t height, int32_t width) { + constexpr int32_t factor = 28; + constexpr int32_t min_pixels = 56 * 56; + constexpr int32_t max_pixels = 28 * 28 * 256; + + if (std::max(height, width) / static_cast(std::min(height, width)) > 200.0) { + MLLM_ERROR_EXIT(mllm::ExitCode::kIOError, "absolute aspect ratio must be smaller than 200"); + } + + int32_t h_bar = static_cast(std::round(static_cast(height) / factor)) * factor; + int32_t w_bar = static_cast(std::round(static_cast(width) / factor)) * factor; + const int32_t current_pixels = h_bar * w_bar; + + if (current_pixels > max_pixels) { + const double beta = std::sqrt(static_cast(height) * width / max_pixels); + const int32_t new_height = std::max(1, static_cast(std::floor(height / beta))); + const int32_t new_width = std::max(1, static_cast(std::floor(width / beta))); + h_bar = std::max(factor, (new_height / factor) * factor); + w_bar = std::max(factor, (new_width / factor) * factor); + } else if (current_pixels < min_pixels) { + const double beta = std::sqrt(static_cast(min_pixels) / (height * width)); + const int32_t new_height = static_cast(std::ceil(height * beta)); + const int32_t new_width = static_cast(std::ceil(width * beta)); + h_bar = (new_height + factor - 1) / factor * factor; + w_bar = (new_width + factor - 1) / factor * factor; + } + + return {h_bar, w_bar}; +} + int64_t visualPatchIndex(int32_t t, int32_t h, int32_t w, int32_t grid_h, int32_t grid_w, int32_t merge_size) { const int32_t h_blocks = grid_h / merge_size; const int32_t w_blocks = grid_w / merge_size; @@ -221,16 +394,11 @@ VisualBucketGrid selectVisualBucket(Tensor grid_thw, const std::vector& buckets) { +VisualResizeOverride chooseVisualResizeOverrideForGrid(int32_t native_grid_h, + int32_t native_grid_w, + const std::vector& buckets) { VisualResizeOverride override; if (buckets.empty()) { return override; } - - auto image = mllm::Image::open(image_path); - mllm::models::qwen2vl::Qwen2VLImagePreprocessor image_preprocessor(56 * 56, 28 * 28 * 256); - auto [native_h_px, native_w_px] = image_preprocessor.smartResize(image.h(), image.w(), 28, 56 * 56, 28 * 28 * 256); - const int32_t native_grid_h = native_h_px / 14; - const int32_t native_grid_w = native_w_px / 14; if (findCoveringVisualBucketIndex(native_grid_h, native_grid_w, buckets) >= 0) { return override; } int32_t max_patch_tokens = 0; @@ -273,6 +441,19 @@ VisualResizeOverride chooseVisualResizeOverrideForOversizedImage(const std::stri return override; } +VisualResizeOverride chooseVisualResizeOverrideForImage(const std::string& image_path, + const std::vector& buckets) { + int32_t width = 0; + int32_t height = 0; + int32_t channels = 0; + if (!stbi_info(image_path.c_str(), &width, &height, &channels)) { + MLLM_ERROR_EXIT(mllm::ExitCode::kIOError, "Can't get information of image: {}", image_path); + } + + const auto [native_h_px, native_w_px] = smartResizePixelsForTokenizer(height, width); + return chooseVisualResizeOverrideForGrid(native_h_px / 14, native_w_px / 14, buckets); +} + Tensor padVisualPatchesToBucket(Tensor img, Tensor grid_thw, const VisualBucketGrid& bucket, const Qwen2VLConfig& cfg) { const auto* grid = grid_thw.ptr(); const int32_t grid_t = grid[0]; @@ -329,6 +510,7 @@ Tensor cropVisualEmbeddingsFromBucket(Tensor bucket_embeddings, Tensor original_grid_thw, Tensor bucket_grid_thw, const Qwen2VLConfig& cfg) { + MLLM_RT_ASSERT_EQ(bucket_embeddings.dtype(), mllm::kFloat32); const auto* original_grid = original_grid_thw.ptr(); const auto* bucket_grid = bucket_grid_thw.ptr(); const int32_t grid_t = original_grid[0]; @@ -358,6 +540,72 @@ Tensor cropVisualEmbeddingsFromBucket(Tensor bucket_embeddings, return cropped; } +Tensor dequantizeVisualUInt16ToFloat(Tensor tensor, const QuantParams& qp) { + auto cpu_tensor = tensor.device() == mllm::kCPU ? tensor : tensor.to(mllm::kCPU); + if (cpu_tensor.dtype() == mllm::kFloat32) { return cpu_tensor; } + if (cpu_tensor.dtype() == mllm::kFloat16) { return cpu_tensor.to(mllm::kFloat32); } + if (cpu_tensor.dtype() != mllm::kUInt16 && cpu_tensor.dtype() != mllm::kUInt16PerTensorAsy) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, + "Visual QNN output must be Float32, Float16, or UInt16PerTensorAsy, got {}", + nameOfType(cpu_tensor.dtype())); + } + + auto out = Tensor::empty(cpu_tensor.shape(), mllm::kFloat32, mllm::kCPU).alloc(); + const auto* src = cpu_tensor.ptr(); + auto* dst = out.ptr(); + for (int64_t i = 0; i < cpu_tensor.numel(); ++i) { dst[i] = dequantizeUInt16(src[i], qp); } + return out; +} + +void writeFloatTensorBinary(const std::string& path, Tensor tensor); + +template +void dumpVisualSegmentOutput(const std::string& dump_prefix, + const std::string& graph_name, + Tensor tensor, + Fn&& qp_for_graph) { + if (dump_prefix.empty()) { return; } + auto cpu_tensor = tensor.device() == mllm::kCPU ? tensor : tensor.to(mllm::kCPU); + Tensor float_tensor = cpu_tensor; + if (cpu_tensor.dtype() == mllm::kUInt16 || cpu_tensor.dtype() == mllm::kUInt16PerTensorAsy) { + float_tensor = dequantizeVisualUInt16ToFloat(cpu_tensor, qp_for_graph(graph_name)); + } else if (cpu_tensor.dtype() == mllm::kFloat16) { + float_tensor = cpu_tensor.to(mllm::kFloat32); + } else if (cpu_tensor.dtype() != mllm::kFloat32) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, + "Cannot dump visual segment {} with dtype={}", + graph_name, + nameOfType(cpu_tensor.dtype())); + } + writeFloatTensorBinary(dump_prefix + "." + graph_name + ".bin", float_tensor); +} + +void writeFloatTensorBinary(const std::string& path, Tensor tensor) { + if (path.empty()) { return; } + auto cpu_tensor = tensor.device() == mllm::kCPU ? tensor : tensor.to(mllm::kCPU); + MLLM_RT_ASSERT_EQ(cpu_tensor.dtype(), mllm::kFloat32); + + std::ofstream out(path, std::ios::out | std::ios::binary | std::ios::trunc); + if (!out) { + fmt::print("[Qwen2VL AOT Dump] failed to open visual embedding dump path: {}\n", path); + return; + } + + const char magic[8] = {'M', 'L', 'L', 'M', 'F', '3', '2', '\0'}; + const int32_t rank = static_cast(cpu_tensor.rank()); + const int64_t numel = cpu_tensor.numel(); + out.write(magic, sizeof(magic)); + out.write(reinterpret_cast(&rank), sizeof(rank)); + for (auto dim : cpu_tensor.shape()) { out.write(reinterpret_cast(&dim), sizeof(dim)); } + out.write(reinterpret_cast(&numel), sizeof(numel)); + out.write(reinterpret_cast(cpu_tensor.ptr()), numel * sizeof(float)); + fmt::print("[Qwen2VL AOT Dump] wrote visual embeddings to {} shape=[", path); + for (size_t i = 0; i < cpu_tensor.shape().size(); ++i) { + fmt::print("{}{}", i == 0 ? "" : ",", cpu_tensor.shape()[i]); + } + fmt::print("]\n"); +} + Tensor makeQnnTensor(const std::vector& shape, mllm::DataTypes dtype, const std::string& name) { auto tensor = Tensor::empty(shape, dtype, mllm::kQNN).setName(name).alloc(); return tensor; @@ -369,6 +617,56 @@ Tensor copyToQnn(const Tensor& cpu_tensor, const std::string& name) { return qnn_tensor; } +Tensor quantizeFloatToQnnUInt16(Tensor cpu_tensor, const QuantParams& qp, const std::string& name) { + auto qnn_tensor = makeQnnTensor(cpu_tensor.shape(), mllm::kUInt16PerTensorAsy, name); + auto* out = qnn_tensor.ptr(); + const auto* in = cpu_tensor.ptr(); + for (int64_t i = 0; i < cpu_tensor.numel(); ++i) { out[i] = quantizeUInt16(in[i], qp); } + qnn_tensor.attach("scale", Tensor::constant(qp.scale, mllm::kFloat32).impl(), true); + qnn_tensor.attach("zero_point", Tensor::constant(qp.zero_point, mllm::kInt32).impl(), true); + return qnn_tensor; +} + +Tensor copyFloatToQnnDType(Tensor cpu_tensor, const std::string& name, mllm::DataTypes dtype) { + if (cpu_tensor.dtype() == mllm::kFloat32) { + if (dtype == mllm::kFloat32) { return copyToQnn(cpu_tensor, name); } + if (dtype == mllm::kFloat16) { return copyToQnn(cpu_tensor.to(mllm::kFloat16), name); } + } + if (cpu_tensor.dtype() == dtype) { return copyToQnn(cpu_tensor, name); } + if (dtype == mllm::kFloat32 || dtype == mllm::kFloat16) { return copyToQnn(cpu_tensor.to(dtype), name); } + return copyToQnn(cpu_tensor, name); +} + +Tensor copyVisualImageToQnn(Tensor cpu_tensor, const std::string& name, VisualIODType visual_io_dtype) { + if (visual_io_dtype == VisualIODType::kUInt16) { + return quantizeFloatToQnnUInt16(cpu_tensor, + {.scale = kDefaultVisualPatchInputScale, + .zero_point = kDefaultVisualPatchInputZeroPoint}, + name); + } + return copyFloatToQnnDType(cpu_tensor, name, visualIODTypeToDataType(visual_io_dtype)); +} + +Tensor copyVisualSinCosToQnn(Tensor cpu_tensor, const std::string& name, VisualIODType visual_io_dtype) { + if (visual_io_dtype != VisualIODType::kUInt16) { + return copyFloatToQnnDType(cpu_tensor, name, visualIODTypeToDataType(visual_io_dtype)); + } + return quantizeFloatToQnnUInt16(cpu_tensor, + {.scale = kDefaultVisualSinCosScale, + .zero_point = kDefaultVisualSinCosZeroPoint}, + name); +} + +Tensor copyVisualMaskToQnn(Tensor cpu_tensor, const std::string& name, VisualIODType visual_io_dtype) { + if (visual_io_dtype != VisualIODType::kUInt16) { + return copyFloatToQnnDType(cpu_tensor, name, visualIODTypeToDataType(visual_io_dtype)); + } + return quantizeFloatToQnnUInt16(cpu_tensor, + {.scale = kDefaultVisualAttentionMaskScale, + .zero_point = kDefaultVisualAttentionMaskZeroPoint}, + name); +} + QnnAOTModule& getOrCreateCachedModule(QnnAOTModuleCache& cache, const std::string& graph_name) { auto it = cache.find(graph_name); if (it != cache.end()) { return *it->second; } @@ -381,6 +679,7 @@ QnnAOTModule& getOrCreateCachedModule(QnnAOTModuleCache& cache, const std::strin std::vector visualGraphNamesForLayout(const std::string& bundle_layout, const std::string& graph_suffix) { if (bundle_layout == "single") { return {"visual_full" + graph_suffix}; } + if (bundle_layout == "hybrid_single") { return {"visual_body" + graph_suffix}; } if (bundle_layout == "6x8") { return {"visual_patch_embed" + graph_suffix, "visual_blocks_0_8" + graph_suffix, @@ -411,7 +710,37 @@ std::vector visualGraphNamesForLayout(const std::string& bundle_lay "visual_merger" + graph_suffix}; } - MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "--visual_bundle_layout must be single, 6x8, early2 or tail4."); + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "--visual_bundle_layout must be single, hybrid_single, 6x8, early2 or tail4."); +} + +std::string stripVisualGraphSuffix(const std::string& graph_name) { + const auto pos = graph_name.rfind("_s"); + if (pos == std::string::npos || pos + 2 >= graph_name.size()) { return graph_name; } + const auto suffix_is_digits = + std::all_of(graph_name.begin() + static_cast(pos + 2), + graph_name.end(), + [](unsigned char ch) { return std::isdigit(ch); }); + return suffix_is_digits ? graph_name.substr(0, pos) : graph_name; +} + +std::string visualGraphOutputQDQName(const std::string& graph_name, int32_t visual_depth) { + const auto base = stripVisualGraphSuffix(graph_name); + if (base == "visual_full" || base == "visual_body" || base == "visual_merger") { + return "visual.merger.mlp.2_output_qdq"; + } + if (base == "visual_patch_embed") { return "visual.blocks.0.attn.qkv_input_qdq"; } + + constexpr const char* prefix = "visual_blocks_"; + if (base.rfind(prefix, 0) == 0) { + const auto mid = base.find('_', std::strlen(prefix)); + if (mid != std::string::npos) { + const auto end_block = std::stoi(base.substr(mid + 1)); + return end_block < visual_depth ? "visual.blocks." + std::to_string(end_block) + ".attn.qkv_input_qdq" + : "visual.merger.mlp.0_input_qdq"; + } + } + + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "Cannot infer visual output QDQ for graph {}", graph_name); } void initializeVisualQnnModules(const std::string& bundle_layout, @@ -447,7 +776,8 @@ Tensor runQnnGraph1(const std::string& graph_name, Tensor visual_embedding_cos, Tensor visual_attention_mask, Tensor output, - QnnAOTModuleCache* module_cache = nullptr) { + QnnAOTModuleCache* module_cache = nullptr, + RequestProfiler* profiler = nullptr) { std::unique_ptr local_module; QnnAOTModule* module = nullptr; if (module_cache != nullptr) { @@ -464,7 +794,11 @@ Tensor runQnnGraph1(const std::string& graph_name, visual_embedding_sin, visual_embedding_cos, visual_attention_mask}; - auto outputs = (*module)(inputs); + std::vector outputs; + { + ScopedProfile timer(profiler, "visual.qnn_execute." + graph_name); + outputs = (*module)(inputs); + } MLLM_RT_ASSERT_EQ(outputs.size(), 1); return outputs[0]; } @@ -476,59 +810,110 @@ Tensor runVisualQnnBundle(Tensor img, const Qwen2VLConfig& cfg, const std::string& bundle_layout, const std::string& graph_suffix, - QnnAOTModuleCache* module_cache = nullptr) { + VisualIODType visual_io_dtype, + const QuantParams& output_qp, + Tensor patch_embed_hidden = Tensor::nil(), + QuantParams patch_embed_qp = {}, + QnnAOTModuleCache* module_cache = nullptr, + RequestProfiler* profiler = nullptr, + const std::string& segment_dump_prefix = "", + const std::function& segment_output_qp = {}) { const int32_t visual_patch_tokens = img.shape()[0]; const int32_t merged_tokens = visual_patch_tokens / (cfg.visual_spatial_merge_size * cfg.visual_spatial_merge_size); + if (!segment_dump_prefix.empty()) { MLLM_RT_ASSERT(static_cast(segment_output_qp)); } + if (isRawFloatVisualIO(visual_io_dtype) && bundle_layout != "single") { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, + "--visual_io_dtype=fp16/fp32 is currently supported only with --visual_bundle_layout=single."); + } - auto img_qnn = copyToQnn(img, "visual_img"); - auto sin_qnn = copyToQnn(visual_embedding_sin, "visual_embedding_sin"); - auto cos_qnn = copyToQnn(visual_embedding_cos, "visual_embedding_cos"); - auto mask_qnn = copyToQnn(visual_attention_mask, "visual_attention_mask"); + Tensor img_qnn; + Tensor sin_qnn; + Tensor cos_qnn; + Tensor mask_qnn; + { + ScopedProfile timer(profiler, "visual.copy_inputs_to_qnn"); + img_qnn = copyVisualImageToQnn(img, "visual_img", visual_io_dtype); + sin_qnn = copyVisualSinCosToQnn(visual_embedding_sin, "visual_embedding_sin", visual_io_dtype); + cos_qnn = copyVisualSinCosToQnn(visual_embedding_cos, "visual_embedding_cos", visual_io_dtype); + mask_qnn = copyVisualMaskToQnn(visual_attention_mask, "visual_attention_mask", visual_io_dtype); + } - auto hidden_a = makeQnnTensor({visual_patch_tokens, cfg.visual_embed_dim}, mllm::kFloat32, "visual_hidden_a"); - auto hidden_b = makeQnnTensor({visual_patch_tokens, cfg.visual_embed_dim}, mllm::kFloat32, "visual_hidden_b"); - auto visual_embeddings = makeQnnTensor({merged_tokens, cfg.hidden_size}, mllm::kFloat32, "visual_embeddings"); + Tensor hidden_a; + Tensor hidden_b; + Tensor visual_embeddings; + { + ScopedProfile timer(profiler, "visual.alloc_qnn_tensors"); + hidden_a = makeQnnTensor({visual_patch_tokens, cfg.visual_embed_dim}, mllm::kUInt16PerTensorAsy, "visual_hidden_a"); + hidden_b = makeQnnTensor({visual_patch_tokens, cfg.visual_embed_dim}, mllm::kUInt16PerTensorAsy, "visual_hidden_b"); + const auto visual_output_dtype = bundle_layout == "single" ? visualIODTypeToDataType(visual_io_dtype) : mllm::kUInt16PerTensorAsy; + visual_embeddings = makeQnnTensor({merged_tokens, cfg.hidden_size}, visual_output_dtype, "visual_embeddings"); + } + + auto run_and_maybe_dump = [&](const std::string& graph_name, + Tensor graph_input, + Tensor graph_sin, + Tensor graph_cos, + Tensor graph_mask, + Tensor graph_output) { + auto output = runQnnGraph1(graph_name, + graph_input, + graph_sin, + graph_cos, + graph_mask, + graph_output, + module_cache, + profiler); + dumpVisualSegmentOutput(segment_dump_prefix, graph_name, output, segment_output_qp); + return output; + }; Tensor hidden = Tensor::nil(); if (bundle_layout == "single") { - auto output = - runQnnGraph1("visual_full" + graph_suffix, img_qnn, sin_qnn, cos_qnn, mask_qnn, visual_embeddings, module_cache); - return output.to(mllm::kCPU).clone(); + auto output = run_and_maybe_dump("visual_full" + graph_suffix, img_qnn, sin_qnn, cos_qnn, mask_qnn, visual_embeddings); + ScopedProfile timer(profiler, "visual.copy_output_to_cpu"); + return dequantizeVisualUInt16ToFloat(output, output_qp); + } else if (bundle_layout == "hybrid_single") { + if (patch_embed_hidden.isNil()) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "hybrid_single requires CPU patch_embed hidden states."); + } + Tensor patch_hidden_qnn; + { + ScopedProfile timer(profiler, "visual.copy_patch_embed_to_qnn"); + patch_hidden_qnn = quantizeFloatToQnnUInt16(patch_embed_hidden, patch_embed_qp, "visual_patch_embed_hidden"); + } + auto output = run_and_maybe_dump("visual_body" + graph_suffix, patch_hidden_qnn, sin_qnn, cos_qnn, mask_qnn, visual_embeddings); + ScopedProfile timer(profiler, "visual.copy_output_to_cpu"); + return dequantizeVisualUInt16ToFloat(output, output_qp); } else if (bundle_layout == "6x8") { - hidden = runQnnGraph1("visual_patch_embed" + graph_suffix, img_qnn, sin_qnn, cos_qnn, Tensor::nil(), hidden_a, module_cache); - hidden = runQnnGraph1("visual_blocks_0_8" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b, module_cache); - hidden = runQnnGraph1("visual_blocks_8_16" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a, module_cache); - hidden = runQnnGraph1("visual_blocks_16_24" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b, module_cache); - hidden = runQnnGraph1("visual_blocks_24_32" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a, module_cache); + hidden = run_and_maybe_dump("visual_patch_embed" + graph_suffix, img_qnn, sin_qnn, cos_qnn, Tensor::nil(), hidden_a); + hidden = run_and_maybe_dump("visual_blocks_0_8" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b); + hidden = run_and_maybe_dump("visual_blocks_8_16" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a); + hidden = run_and_maybe_dump("visual_blocks_16_24" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b); + hidden = run_and_maybe_dump("visual_blocks_24_32" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a); } else if (bundle_layout == "early2") { - hidden = runQnnGraph1("visual_patch_embed" + graph_suffix, img_qnn, sin_qnn, cos_qnn, Tensor::nil(), hidden_a, module_cache); - hidden = runQnnGraph1("visual_blocks_0_2" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b, module_cache); - hidden = runQnnGraph1("visual_blocks_2_4" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a, module_cache); - hidden = runQnnGraph1("visual_blocks_4_6" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b, module_cache); - hidden = runQnnGraph1("visual_blocks_6_8" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a, module_cache); - hidden = runQnnGraph1("visual_blocks_8_16" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b, module_cache); - hidden = runQnnGraph1("visual_blocks_16_24" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a, module_cache); - hidden = runQnnGraph1("visual_blocks_24_32" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b, module_cache); + hidden = run_and_maybe_dump("visual_patch_embed" + graph_suffix, img_qnn, sin_qnn, cos_qnn, Tensor::nil(), hidden_a); + hidden = run_and_maybe_dump("visual_blocks_0_2" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b); + hidden = run_and_maybe_dump("visual_blocks_2_4" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a); + hidden = run_and_maybe_dump("visual_blocks_4_6" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b); + hidden = run_and_maybe_dump("visual_blocks_6_8" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a); + hidden = run_and_maybe_dump("visual_blocks_8_16" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b); + hidden = run_and_maybe_dump("visual_blocks_16_24" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a); + hidden = run_and_maybe_dump("visual_blocks_24_32" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b); } else if (bundle_layout == "tail4") { - hidden = runQnnGraph1("visual_patch_embed" + graph_suffix, img_qnn, sin_qnn, cos_qnn, Tensor::nil(), hidden_a, module_cache); - hidden = runQnnGraph1("visual_blocks_0_8" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b, module_cache); - hidden = runQnnGraph1("visual_blocks_8_16" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a, module_cache); - hidden = runQnnGraph1("visual_blocks_16_20" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b, module_cache); - hidden = runQnnGraph1("visual_blocks_20_24" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a, module_cache); - hidden = runQnnGraph1("visual_blocks_24_28" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b, module_cache); - hidden = runQnnGraph1("visual_blocks_28_32" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a, module_cache); + hidden = run_and_maybe_dump("visual_patch_embed" + graph_suffix, img_qnn, sin_qnn, cos_qnn, Tensor::nil(), hidden_a); + hidden = run_and_maybe_dump("visual_blocks_0_8" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b); + hidden = run_and_maybe_dump("visual_blocks_8_16" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a); + hidden = run_and_maybe_dump("visual_blocks_16_20" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b); + hidden = run_and_maybe_dump("visual_blocks_20_24" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a); + hidden = run_and_maybe_dump("visual_blocks_24_28" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_b); + hidden = run_and_maybe_dump("visual_blocks_28_32" + graph_suffix, hidden, sin_qnn, cos_qnn, mask_qnn, hidden_a); } else { - MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "--visual_bundle_layout must be single, 6x8, early2 or tail4 for full runner."); + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "--visual_bundle_layout must be single, hybrid_single, 6x8, early2 or tail4 for full runner."); } - auto output = runQnnGraph1("visual_merger" + graph_suffix, - hidden, - sin_qnn, - cos_qnn, - Tensor::nil(), - visual_embeddings, - module_cache); - return output.to(mllm::kCPU).clone(); + auto output = run_and_maybe_dump("visual_merger" + graph_suffix, hidden, sin_qnn, cos_qnn, Tensor::nil(), visual_embeddings); + ScopedProfile timer(profiler, "visual.copy_output_to_cpu"); + return dequantizeVisualUInt16ToFloat(output, output_qp); } int32_t findVisionTokenStart(const Tensor& sequence, int32_t vision_token_id) { @@ -856,11 +1241,18 @@ class Qwen2VLAOTRunner { bool dump_visual_tokens, bool key_cache_uint16, bool visual_qnn, + bool visual_only, std::string visual_bundle_layout, std::vector visual_bucket_grids, + VisualIODType visual_io_dtype, + float visual_output_scale, + int32_t visual_output_zero_point, + float visual_qdq_scale_multiplier, int32_t dump_logits_topk, std::vector dump_token_indices, - std::string dump_path) + std::string dump_path, + std::string visual_embeddings_dump_path, + std::string visual_segments_dump_prefix) : qnn_cfg_(qnn_cfg), visual_cfg_(visual_cfg), qnn_params_(qnn_params), @@ -869,11 +1261,17 @@ class Qwen2VLAOTRunner { dump_visual_tokens_(dump_visual_tokens), key_cache_uint16_(key_cache_uint16), visual_qnn_(visual_qnn), + visual_only_(visual_only), visual_bundle_layout_(std::move(visual_bundle_layout)), visual_bucket_grids_(std::move(visual_bucket_grids)), + visual_io_dtype_(visual_io_dtype), + visual_qdq_scale_multiplier_(visual_qdq_scale_multiplier), dump_logits_topk_(dump_logits_topk), dump_token_indices_(std::move(dump_token_indices)), - dump_path_(std::move(dump_path)) { + dump_path_(std::move(dump_path)), + visual_embeddings_dump_path_(std::move(visual_embeddings_dump_path)), + visual_segments_dump_prefix_(std::move(visual_segments_dump_prefix)), + profiler_(shouldProfileQwen2VLAOT()) { config_.num_layers = qnn_cfg_.num_hidden_layers; config_.num_heads = qnn_cfg_.num_key_value_heads; config_.head_dim = qnn_cfg_.hidden_size / qnn_cfg_.num_attention_heads; @@ -885,6 +1283,11 @@ class Qwen2VLAOTRunner { config_.max_ar_len = std::max(1, ar_len); config_.sliding_window = context_len; + if (visual_qnn_ && isRawFloatVisualIO(visual_io_dtype_) && visual_bundle_layout_ != "single") { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, + "--visual_io_dtype=fp16/fp32 requires --visual_bundle_layout=single for now."); + } + embedding_weight_qp_ = readQuantParams(qnn_params_, "model.embed_tokens.scale", "model.embed_tokens.zero_point"); input_embedding_qp_ = embedding_weight_qp_; if (input_embedding_scale > 0.0f && input_embedding_zero_point >= 0) { @@ -898,6 +1301,48 @@ class Qwen2VLAOTRunner { "model.cos_embedding_input_qdq.fake_quant.zero_point"); logits_qp_ = readQuantParams(qnn_params_, "lm_head_output_qdq.fake_quant.scale", "lm_head_output_qdq.fake_quant.zero_point"); + if (visual_output_scale > 0.0f && visual_output_zero_point >= 0) { + visual_output_qp_ = {.scale = visual_output_scale, .zero_point = visual_output_zero_point}; + fmt::print("[Qwen2VL AOT] override visual output quant params: scale={:.9f}, zero_point={}\n", + visual_output_qp_.scale, visual_output_qp_.zero_point); + } else if (hasQuantParams(qnn_params_, + "visual.merger.mlp.2_output_qdq.fake_quant.scale", + "visual.merger.mlp.2_output_qdq.fake_quant.zero_point")) { + visual_output_qp_ = readQuantParams(qnn_params_, + "visual.merger.mlp.2_output_qdq.fake_quant.scale", + "visual.merger.mlp.2_output_qdq.fake_quant.zero_point"); + } else if (visual_qnn_ && visual_bundle_layout_ == "single") { + visual_output_qp_ = {.scale = kDefaultVisualOutputScale, .zero_point = kDefaultVisualOutputZeroPoint}; + fmt::print("[Qwen2VL AOT] visual output QDQ not found in --qnn_params; using single-graph default: " + "scale={:.9f}, zero_point={}\n", + visual_output_qp_.scale, + visual_output_qp_.zero_point); + } else if (visual_qnn_) { + MLLM_ERROR_EXIT(mllm::ExitCode::kIOError, + "Missing visual output QDQ tensors in --qnn_params. Pass --visual_output_scale and " + "--visual_output_zero_point, or use a parameter file containing visual QDQ tensors."); + } + if (visual_qnn_ && visual_bundle_layout_ == "hybrid_single") { + visual_body_input_qp_ = readQuantParams(qnn_params_, + firstHybridBodyInputQDQName() + ".fake_quant.scale", + firstHybridBodyInputQDQName() + ".fake_quant.zero_point"); + } + if (visual_qdq_scale_multiplier > 0.0f && std::abs(visual_qdq_scale_multiplier - 1.0f) >= 1e-6f) { + if (visual_qnn_ && visual_bundle_layout_ != "hybrid_single") { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, + "--visual_qdq_scale_multiplier is only supported with --visual_bundle_layout=hybrid_single."); + } + visual_body_input_qp_.scale *= visual_qdq_scale_multiplier; + visual_output_qp_.scale *= visual_qdq_scale_multiplier; + fmt::print("[Qwen2VL AOT] scaled visual body input QP by {:.6f}: scale={:.9f}, zero_point={}\n", + visual_qdq_scale_multiplier, + visual_body_input_qp_.scale, + visual_body_input_qp_.zero_point); + fmt::print("[Qwen2VL AOT] scaled visual output QP by {:.6f}: scale={:.9f}, zero_point={}\n", + visual_qdq_scale_multiplier, + visual_output_qp_.scale, + visual_output_qp_.zero_point); + } block_out_qps_.reserve(qnn_cfg_.num_hidden_layers); for (int32_t l = 0; l < qnn_cfg_.num_hidden_layers; ++l) { if (l + 1 < qnn_cfg_.num_hidden_layers) { @@ -911,6 +1356,10 @@ class Qwen2VLAOTRunner { } initLayer0QuantParams(); embedding_weight_ = qnn_params_->pull("model.embed_tokens.weight"); + visual_rope_inv_freq_ = mllm::models::qwen2vl::makeVisualRoPEInvFreq( + visual_cfg_.visual_embed_dim / visual_cfg_.visual_num_heads, 10000.0); + llm_rope_inv_freq_ = mllm::models::qwen2vl::makeMultimodalRoPEInvFreq( + qnn_cfg_.hidden_size / qnn_cfg_.num_attention_heads, qnn_cfg_.rope_theta); } bool load() { @@ -920,24 +1369,32 @@ class Qwen2VLAOTRunner { return false; } - kv_manager_u8_ = std::make_unique>(config_); - kv_manager_u8_->initCache(backend->allocator().get(), config_.ar_len); - if (key_cache_uint16_) { - key_cache_manager_u16_ = std::make_unique>(config_); - key_cache_manager_u16_->initCache(backend->allocator().get(), config_.ar_len); - fmt::print("[Qwen2VL AOT] experimental key cache dtype: uint16, value cache dtype: uint8\n"); - } - prefill_io_.ar_len = config_.ar_len; - decode_io_.ar_len = 1; - prefill_io_.dump_block_outputs = dump_block_outputs_; - decode_io_.dump_block_outputs = dump_block_outputs_; - prefill_io_.dump_layer0_outputs = dump_layer0_outputs_; - decode_io_.dump_layer0_outputs = dump_layer0_outputs_; - initQwen2VLIO(prefill_io_, config_, *kv_manager_u8_, key_cache_manager_u16_.get(), key_cache_uint16_, - qnn_cfg_.hidden_size, qnn_cfg_.intermediate_size); - initQwen2VLIO(decode_io_, config_, *kv_manager_u8_, key_cache_manager_u16_.get(), key_cache_uint16_, - qnn_cfg_.hidden_size, qnn_cfg_.intermediate_size); + if (!visual_only_) { + kv_manager_u8_ = std::make_unique>(config_); + kv_manager_u8_->initCache(backend->allocator().get(), config_.ar_len); + if (key_cache_uint16_) { + key_cache_manager_u16_ = std::make_unique>(config_); + key_cache_manager_u16_->initCache(backend->allocator().get(), config_.ar_len); + fmt::print("[Qwen2VL AOT] experimental key cache dtype: uint16, value cache dtype: uint8\n"); + } + prefill_io_.ar_len = config_.ar_len; + decode_io_.ar_len = 1; + prefill_io_.dump_block_outputs = dump_block_outputs_; + decode_io_.dump_block_outputs = dump_block_outputs_; + prefill_io_.dump_layer0_outputs = dump_layer0_outputs_; + decode_io_.dump_layer0_outputs = dump_layer0_outputs_; + initQwen2VLIO(prefill_io_, config_, *kv_manager_u8_, key_cache_manager_u16_.get(), key_cache_uint16_, + qnn_cfg_.hidden_size, qnn_cfg_.intermediate_size); + initQwen2VLIO(decode_io_, config_, *kv_manager_u8_, key_cache_manager_u16_.get(), key_cache_uint16_, + qnn_cfg_.hidden_size, qnn_cfg_.intermediate_size); + } else { + fmt::print("[Qwen2VL AOT] visual-only mode: skipped LLM QNN module and KV cache initialization.\n"); + } if (visual_qnn_) { initializeVisualQnnModules(visual_bundle_layout_, visual_bucket_grids_, visual_modules_); } + decode_token_tensor_ = Tensor::empty({1, 1}, mllm::kInt64, mllm::kCPU).alloc(); + if (profiler_.enabled()) { + fmt::print("[Qwen2VL AOT] fine-grained stage profiling enabled (MLLM_QWEN2VL_AOT_PROFILE=1)\n"); + } return true; } @@ -949,10 +1406,19 @@ class Qwen2VLAOTRunner { prompt_token_ids_.clear(); vision_token_start_ = -1; vision_token_count_ = 0; + profiler_.reset(); } + RequestProfiler* profiler() { return &profiler_; } + + void printProfileSummary() const { profiler_.printSummary(); } + + void addProfileEvent(const std::string& name, int64_t us) { profiler_.add(name, us); } + PromptFeatures preparePrompt(const mllm::models::ARGenerationOutputPast& inputs, - mllm::models::qwen2vl::Qwen2VisionTransformerPretrainedModel* visual) { + mllm::models::qwen2vl::Qwen2VisionTransformerPretrainedModel* visual, + Qwen2VLPatchEmbedOnly* patch_embed) { + ScopedProfile prepare_timer(&profiler_, "prepare_prompt.total"); PromptFeatures features; features.sequence = inputs.at("sequence"); auto img = inputs.at("img"); @@ -966,15 +1432,23 @@ class Qwen2VLAOTRunner { mllm::print("Image shape is:", img.shape()); auto visual_img = img; auto visual_grid_thw = grid_thw; - auto visual_attention_mask = makeAllValidVisualAttentionMask(img.shape()[0]); + Tensor visual_attention_mask; + { + ScopedProfile timer(&profiler_, "visual.make_initial_mask"); + visual_attention_mask = makeAllValidVisualAttentionMask(img.shape()[0]); + } std::string visual_graph_suffix; if (visual_qnn_ && !visual_bucket_grids_.empty()) { const auto* original_grid = grid_thw.ptr(); - const auto bucket = selectVisualBucket(grid_thw, visual_bucket_grids_); - visual_img = padVisualPatchesToBucket(img, grid_thw, bucket, visual_cfg_); - visual_grid_thw = makeGridThwTensor(original_grid[0], bucket.grid_h, bucket.grid_w); - visual_attention_mask = makeVisualAttentionMaskForBucket(grid_thw, bucket, visual_cfg_); - visual_graph_suffix = visualGraphSuffixForPatchTokens(bucket.patchTokens()); + VisualBucketGrid bucket; + { + ScopedProfile timer(&profiler_, "visual.bucket_select_pad_mask"); + bucket = selectVisualBucket(grid_thw, visual_bucket_grids_); + visual_img = padVisualPatchesToBucket(img, grid_thw, bucket, visual_cfg_); + visual_grid_thw = makeGridThwTensor(original_grid[0], bucket.grid_h, bucket.grid_w); + visual_attention_mask = makeVisualAttentionMaskForBucket(grid_thw, bucket, visual_cfg_); + visual_graph_suffix = visualGraphSuffixForPatchTokens(bucket.patchTokens()); + } fmt::print("[Qwen2VL AOT] visual bucket: original_grid={}x{} patches={} -> bucket_grid={}x{} patches={} graph_suffix={}\n", original_grid[1], original_grid[2], @@ -984,30 +1458,57 @@ class Qwen2VLAOTRunner { visual_img.shape()[0], visual_graph_suffix); } - auto inv_freq = mllm::models::qwen2vl::makeVisualRoPEInvFreq(visual_cfg_.visual_embed_dim / visual_cfg_.visual_num_heads, - 10000.0); - auto visual_pos_ids = mllm::models::qwen2vl::makeVisualRotaryPosEmbIds(visual_grid_thw, visual_cfg_.visual_spatial_merge_size); - auto rotary_pos_emb_full = mllm::models::qwen2vl::makeVisualRotaryPosEmbFull(inv_freq, visual_img.shape()[0]); - auto pos_emb = mllm::models::qwen2vl::makeVisualRotaryPosEmb(rotary_pos_emb_full, visual_pos_ids, visual_grid_thw); - auto [visual_embedding_sin, visual_embedding_cos] = mllm::models::qwen2vl::makeVisualRotarySinCos(pos_emb); + Tensor visual_embedding_sin; + Tensor visual_embedding_cos; + { + ScopedProfile timer(&profiler_, "visual.rope_prepare"); + auto visual_pos_ids = + mllm::models::qwen2vl::makeVisualRotaryPosEmbIds(visual_grid_thw, visual_cfg_.visual_spatial_merge_size); + auto rotary_pos_emb_full = + mllm::models::qwen2vl::makeVisualRotaryPosEmbFull(visual_rope_inv_freq_, visual_img.shape()[0]); + auto pos_emb = mllm::models::qwen2vl::makeVisualRotaryPosEmb(rotary_pos_emb_full, visual_pos_ids, visual_grid_thw); + auto sin_cos = mllm::models::qwen2vl::makeVisualRotarySinCos(pos_emb); + visual_embedding_sin = sin_cos.first; + visual_embedding_cos = sin_cos.second; + } vit_start_ = std::chrono::high_resolution_clock::now(); if (visual_qnn_) { + Tensor patch_embed_hidden = Tensor::nil(); + if (visual_bundle_layout_ == "hybrid_single") { + MLLM_RT_ASSERT(patch_embed != nullptr); + ScopedProfile timer(&profiler_, "visual.cpu_patch_embed"); + patch_embed_hidden = (*patch_embed)(visual_img)[0]; + } const int32_t half_dim = visual_cfg_.visual_embed_dim / visual_cfg_.visual_num_heads / 2; auto visual_sin_4d = visual_embedding_sin.view({1, -1, 1, half_dim}, false); auto visual_cos_4d = visual_embedding_cos.view({1, -1, 1, half_dim}, false); - auto bucket_embeddings = - runVisualQnnBundle(visual_img, - visual_sin_4d, - visual_cos_4d, - visual_attention_mask, - visual_cfg_, - visual_bundle_layout_, - visual_graph_suffix, - &visual_modules_); + Tensor bucket_embeddings; + { + ScopedProfile timer(&profiler_, "visual.qnn_bundle_total"); + bucket_embeddings = runVisualQnnBundle(visual_img, + visual_sin_4d, + visual_cos_4d, + visual_attention_mask, + visual_cfg_, + visual_bundle_layout_, + visual_graph_suffix, + visual_io_dtype_, + visual_output_qp_, + patch_embed_hidden, + visual_body_input_qp_, + &visual_modules_, + &profiler_, + visual_segments_dump_prefix_, + [this](const std::string& graph_name) { return visualSegmentOutputQP(graph_name); }); + } + { + ScopedProfile timer(&profiler_, "visual.crop_embeddings"); features.visual_embeddings = cropVisualEmbeddingsFromBucket(bucket_embeddings, grid_thw, visual_grid_thw, visual_cfg_); + } } else { MLLM_RT_ASSERT(visual != nullptr); + ScopedProfile timer(&profiler_, "visual.cpu_forward"); features.visual_embeddings = (*visual)(img, visual_embedding_sin, visual_embedding_cos)[0]; } vit_end_ = std::chrono::high_resolution_clock::now(); @@ -1015,6 +1516,7 @@ class Qwen2VLAOTRunner { visual_qnn_ ? "QNN AOT" : "CPU/KAI", std::chrono::duration(vit_end_ - vit_start_).count()); if (shouldDumpQwen2VLAOTStats()) { + ScopedProfile timer(&profiler_, "visual.embedding_stats"); const auto* data = features.visual_embeddings.ptr(); const auto numel = features.visual_embeddings.numel(); auto [min_it, max_it] = std::minmax_element(data, data + numel); @@ -1026,27 +1528,33 @@ class Qwen2VLAOTRunner { } fmt::print("[Qwen2VL AOT] visual_embeddings shape=[{}, {}], range=[{:.6f}, {:.6f}], embedding_qrange=[{:.6f}, " "{:.6f}], clipped={}/{} ({:.2f}%)\n", - features.visual_embeddings.shape()[0], features.visual_embeddings.shape()[1], *min_it, *max_it, qmin, qmax, - clipped, numel, numel > 0 ? 100.0 * static_cast(clipped) / static_cast(numel) : 0.0); + features.visual_embeddings.shape()[0], features.visual_embeddings.shape()[1], *min_it, *max_it, qmin, qmax, + clipped, numel, numel > 0 ? 100.0 * static_cast(clipped) / static_cast(numel) : 0.0); + } + if (!visual_embeddings_dump_path_.empty()) { + ScopedProfile timer(&profiler_, "visual.dump_embeddings"); + writeFloatTensorBinary(visual_embeddings_dump_path_, features.visual_embeddings); } - features.vision_token_start = findVisionTokenStart(features.sequence, qnn_cfg_.vision_token_id); - MLLM_RT_ASSERT(features.vision_token_start >= 0); - vision_token_start_ = features.vision_token_start; - vision_token_count_ = features.visual_embeddings.shape()[0]; - - features.position_ids = makePositionIdsPrefill(features.sequence, grid_thw, qnn_cfg_); - auto inv = mllm::models::qwen2vl::makeMultimodalRoPEInvFreq( - qnn_cfg_.hidden_size / qnn_cfg_.num_attention_heads, qnn_cfg_.rope_theta); - auto [sin, cos] = mllm::models::qwen2vl::makeMultimodalPositionEmbedding( - features.position_ids, inv, qnn_cfg_.max_position_embeddings, qnn_cfg_.hidden_size / qnn_cfg_.num_attention_heads, - qnn_cfg_.mrope_section); - features.llm_embedding_sin = sin; - features.llm_embedding_cos = cos; + { + ScopedProfile timer(&profiler_, "llm.position_embedding_prepare"); + features.vision_token_start = findVisionTokenStart(features.sequence, qnn_cfg_.vision_token_id); + MLLM_RT_ASSERT(features.vision_token_start >= 0); + vision_token_start_ = features.vision_token_start; + vision_token_count_ = features.visual_embeddings.shape()[0]; + + features.position_ids = makePositionIdsPrefill(features.sequence, grid_thw, qnn_cfg_); + auto [sin, cos] = mllm::models::qwen2vl::makeMultimodalPositionEmbedding( + features.position_ids, llm_rope_inv_freq_, qnn_cfg_.max_position_embeddings, + qnn_cfg_.hidden_size / qnn_cfg_.num_attention_heads, qnn_cfg_.mrope_section); + features.llm_embedding_sin = sin; + features.llm_embedding_cos = cos; + } return features; } int64_t prefill(const PromptFeatures& features) { + ScopedProfile total_timer(&profiler_, "prefill.total"); const int64_t num_tokens = features.sequence.shape()[1]; const auto dump_tokens = selectDumpTokens(num_tokens); if ((dump_block_outputs_ || dump_layer0_outputs_ || dump_visual_tokens_ || dump_logits_topk_ > 0) && !dump_path_.empty()) { @@ -1055,39 +1563,69 @@ class Qwen2VLAOTRunner { int64_t processed_tokens = 0; int64_t current_pos = 0; - rearrangeCache(config_.ar_len); + { + ScopedProfile timer(&profiler_, "prefill.rearrange_cache"); + rearrangeCache(config_.ar_len); + } - std::vector attention_map(config_.ar_len); - std::iota(attention_map.begin(), attention_map.end(), -1); - kv_manager_u8_->initAttentionMask(prefill_io_.inputs[3].ptr(), attention_map, config_.ar_len, 0, - config_.sliding_window); - prefill_io_.module->setOutputTensors(prefill_io_.outputs); + { + ScopedProfile timer(&profiler_, "prefill.init_attention_mask"); + std::vector attention_map(config_.ar_len); + std::iota(attention_map.begin(), attention_map.end(), -1); + kv_manager_u8_->initAttentionMask(prefill_io_.inputs[3].ptr(), attention_map, config_.ar_len, 0, + config_.sliding_window); + prefill_io_.module->setOutputTensors(prefill_io_.outputs); + } int64_t next_token = 0; while (processed_tokens < num_tokens) { + ScopedProfile chunk_timer(&profiler_, "prefill.chunk_total"); const int32_t chunk = static_cast(std::min(config_.ar_len, num_tokens - processed_tokens)); - fillEmbeddingChunk(features.sequence, embedding_weight_, features.visual_embeddings, features.vision_token_start, - processed_tokens, chunk, prefill_io_.inputs[0], embedding_weight_qp_, input_embedding_qp_, - qnn_cfg_.hidden_size); - fillQuantizedFloatChunk(features.llm_embedding_sin, processed_tokens, chunk, prefill_io_.inputs[1], sin_qp_, - config_.head_dim); - fillQuantizedFloatChunk(features.llm_embedding_cos, processed_tokens, chunk, prefill_io_.inputs[2], cos_qp_, - config_.head_dim); + { + ScopedProfile timer(&profiler_, "prefill.fill_input_embeddings"); + fillEmbeddingChunk(features.sequence, embedding_weight_, features.visual_embeddings, features.vision_token_start, + processed_tokens, chunk, prefill_io_.inputs[0], embedding_weight_qp_, input_embedding_qp_, + qnn_cfg_.hidden_size); + } + { + ScopedProfile timer(&profiler_, "prefill.fill_rope"); + fillQuantizedFloatChunk(features.llm_embedding_sin, processed_tokens, chunk, prefill_io_.inputs[1], sin_qp_, + config_.head_dim); + fillQuantizedFloatChunk(features.llm_embedding_cos, processed_tokens, chunk, prefill_io_.inputs[2], cos_qp_, + config_.head_dim); + } auto module_inputs = prefill_io_.inputs; - prefill_io_.outputs = (*prefill_io_.module)(module_inputs); + { + ScopedProfile timer(&profiler_, "prefill.qnn_execute"); + prefill_io_.outputs = (*prefill_io_.module)(module_inputs); + } if (dump_block_outputs_ || dump_layer0_outputs_ || dump_visual_tokens_) { + ScopedProfile timer(&profiler_, "prefill.dump_debug_outputs"); dumpPrefillDebugOutputs(prefill_io_.inputs, prefill_io_.outputs, chunk, processed_tokens, dump_tokens); } - updateCache(config_.ar_len, current_pos, chunk); - kv_manager_u8_->updateAttentionMask(prefill_io_.inputs[3].ptr(), config_.ar_len, current_pos, chunk, - config_.sliding_window); + { + ScopedProfile timer(&profiler_, "prefill.update_cache"); + updateCache(config_.ar_len, current_pos, chunk); + } + { + ScopedProfile timer(&profiler_, "prefill.update_attention_mask"); + kv_manager_u8_->updateAttentionMask(prefill_io_.inputs[3].ptr(), config_.ar_len, current_pos, chunk, + config_.sliding_window); + } - const int32_t logits_idx = static_cast((processed_tokens + chunk - 1) % config_.ar_len); - next_token = sampleGreedyFromLogitsAt(prefill_io_.outputs[0], logits_idx); - if (dump_logits_topk_ > 0 && processed_tokens + chunk == num_tokens) { - dumpLogitsTopK(prefill_io_.outputs[0], logits_idx, static_cast(processed_tokens + chunk - 1), "prefill"); + const bool is_final_chunk = processed_tokens + chunk == num_tokens; + if (is_final_chunk) { + const int32_t logits_idx = chunk - 1; + { + ScopedProfile timer(&profiler_, "prefill.sample_logits"); + next_token = sampleGreedyFromLogitsAt(prefill_io_.outputs[0], logits_idx); + } + if (dump_logits_topk_ > 0) { + ScopedProfile timer(&profiler_, "prefill.dump_logits_topk"); + dumpLogitsTopK(prefill_io_.outputs[0], logits_idx, static_cast(processed_tokens + chunk - 1), "prefill"); + } } processed_tokens += chunk; @@ -1100,35 +1638,67 @@ class Qwen2VLAOTRunner { } int64_t decodeOne(int64_t token) { - rearrangeCache(1); - auto position_ids = makePositionIdsDecode(prev_position_ids_); - prev_position_ids_ = position_ids; + ScopedProfile step_timer(&profiler_, "decode.step_total"); + { + ScopedProfile timer(&profiler_, "decode.rearrange_cache"); + rearrangeCache(1); + } + Tensor position_ids; + { + ScopedProfile timer(&profiler_, "decode.position_ids"); + position_ids = makePositionIdsDecode(prev_position_ids_); + prev_position_ids_ = position_ids; + } - std::vector attention_map(1); - std::iota(attention_map.begin(), attention_map.end(), -1); - kv_manager_u8_->initAttentionMask(decode_io_.inputs[3].ptr(), attention_map, 1, current_pos_, - config_.sliding_window); + { + ScopedProfile timer(&profiler_, "decode.init_attention_mask"); + std::vector attention_map(1); + std::iota(attention_map.begin(), attention_map.end(), -1); + kv_manager_u8_->initAttentionMask(decode_io_.inputs[3].ptr(), attention_map, 1, current_pos_, + config_.sliding_window); + } - auto token_tensor = Tensor::empty({1, 1}, mllm::kInt64, mllm::kCPU).alloc(); - token_tensor.ptr()[0] = token; + decode_token_tensor_.ptr()[0] = token; auto visual_nil = Tensor::nil(); - fillEmbeddingChunk(token_tensor, embedding_weight_, visual_nil, 0, 0, 1, decode_io_.inputs[0], embedding_weight_qp_, - input_embedding_qp_, qnn_cfg_.hidden_size); + { + ScopedProfile timer(&profiler_, "decode.fill_input_embeddings"); + fillEmbeddingChunk(decode_token_tensor_, + embedding_weight_, + visual_nil, + 0, + 0, + 1, + decode_io_.inputs[0], + embedding_weight_qp_, + input_embedding_qp_, + qnn_cfg_.hidden_size); + } - auto inv = mllm::models::qwen2vl::makeMultimodalRoPEInvFreq( - qnn_cfg_.hidden_size / qnn_cfg_.num_attention_heads, qnn_cfg_.rope_theta); - auto [sin, cos] = mllm::models::qwen2vl::makeMultimodalPositionEmbedding( - position_ids, inv, qnn_cfg_.max_position_embeddings, qnn_cfg_.hidden_size / qnn_cfg_.num_attention_heads, - qnn_cfg_.mrope_section); - fillQuantizedFloatChunk(sin, 0, 1, decode_io_.inputs[1], sin_qp_, config_.head_dim); - fillQuantizedFloatChunk(cos, 0, 1, decode_io_.inputs[2], cos_qp_, config_.head_dim); + { + ScopedProfile timer(&profiler_, "decode.fill_rope"); + auto [sin, cos] = mllm::models::qwen2vl::makeMultimodalPositionEmbedding( + position_ids, llm_rope_inv_freq_, qnn_cfg_.max_position_embeddings, qnn_cfg_.hidden_size / qnn_cfg_.num_attention_heads, + qnn_cfg_.mrope_section); + fillQuantizedFloatChunk(sin, 0, 1, decode_io_.inputs[1], sin_qp_, config_.head_dim); + fillQuantizedFloatChunk(cos, 0, 1, decode_io_.inputs[2], cos_qp_, config_.head_dim); + } decode_io_.module->setOutputTensors(decode_io_.outputs); auto module_inputs = decode_io_.inputs; - decode_io_.outputs = (*decode_io_.module)(module_inputs); + { + ScopedProfile timer(&profiler_, "decode.qnn_execute"); + decode_io_.outputs = (*decode_io_.module)(module_inputs); + } - updateCache(1, current_pos_, 1); - auto next_token = sampleGreedyFromLogitsAt(decode_io_.outputs[0], 0); + { + ScopedProfile timer(&profiler_, "decode.update_cache"); + updateCache(1, current_pos_, 1); + } + int64_t next_token = 0; + { + ScopedProfile timer(&profiler_, "decode.sample_logits"); + next_token = sampleGreedyFromLogitsAt(decode_io_.outputs[0], 0); + } ++current_pos_; return next_token; } @@ -1142,7 +1712,10 @@ class Qwen2VLAOTRunner { prefill_end_ = std::chrono::high_resolution_clock::now(); generated_tokens_ = 1; - emitToken(next_token, tokenizer, token_callback); + { + ScopedProfile timer(&profiler_, "token.emit"); + emitToken(next_token, tokenizer, token_callback); + } decode_start_ = std::chrono::high_resolution_clock::now(); for (int32_t i = 1; i < gen_len; ++i) { @@ -1150,7 +1723,10 @@ class Qwen2VLAOTRunner { if (eos_ids_.count(next_token)) { break; } next_token = decodeOne(next_token); ++generated_tokens_; - emitToken(next_token, tokenizer, token_callback); + { + ScopedProfile timer(&profiler_, "token.emit"); + emitToken(next_token, tokenizer, token_callback); + } if (eos_ids_.count(next_token)) { break; } } decode_end_ = std::chrono::high_resolution_clock::now(); @@ -1217,6 +1793,10 @@ class Qwen2VLAOTRunner { QuantParams sin_qp_; QuantParams cos_qp_; QuantParams logits_qp_; + QuantParams visual_output_qp_; + QuantParams visual_body_input_qp_; + Tensor visual_rope_inv_freq_; + Tensor llm_rope_inv_freq_; std::vector block_out_qps_; std::vector> layer0_output_qps_; QnnAOTConfig config_; @@ -1224,6 +1804,7 @@ class Qwen2VLAOTRunner { std::unique_ptr> key_cache_manager_u16_; RuntimeIO prefill_io_; RuntimeIO decode_io_; + Tensor decode_token_tensor_; QnnAOTModuleCache visual_modules_; Tensor prev_position_ids_; int64_t current_pos_ = 0; @@ -1234,11 +1815,16 @@ class Qwen2VLAOTRunner { bool dump_visual_tokens_ = false; bool key_cache_uint16_ = false; bool visual_qnn_ = false; + bool visual_only_ = false; std::string visual_bundle_layout_ = "6x8"; std::vector visual_bucket_grids_; + VisualIODType visual_io_dtype_ = VisualIODType::kUInt16; + float visual_qdq_scale_multiplier_ = 1.0f; int32_t dump_logits_topk_ = 0; std::vector dump_token_indices_; std::string dump_path_; + std::string visual_embeddings_dump_path_; + std::string visual_segments_dump_prefix_; std::vector prompt_token_ids_; int32_t vision_token_start_ = -1; int32_t vision_token_count_ = 0; @@ -1249,6 +1835,7 @@ class Qwen2VLAOTRunner { std::chrono::high_resolution_clock::time_point prefill_end_; std::chrono::high_resolution_clock::time_point decode_start_; std::chrono::high_resolution_clock::time_point decode_end_; + RequestProfiler profiler_; static float dequantizeUInt16(uint16_t value, const QuantParams& qp) { return (static_cast(value) - qp.zero_point) * qp.scale; @@ -1307,6 +1894,16 @@ class Qwen2VLAOTRunner { fmt::print("\n"); } + QuantParams visualSegmentOutputQP(const std::string& graph_name) const { + auto qp_name = visualGraphOutputQDQName(graph_name, visual_cfg_.visual_depth); + if (qp_name == "visual.merger.mlp.2_output_qdq") { return visual_output_qp_; } + auto qp = readQuantParams(qnn_params_, qp_name + ".fake_quant.scale", qp_name + ".fake_quant.zero_point"); + if (visual_qdq_scale_multiplier_ > 0.0f && std::abs(visual_qdq_scale_multiplier_ - 1.0f) >= 1e-6f) { + qp.scale *= visual_qdq_scale_multiplier_; + } + return qp; + } + void dumpTokenIdLine(std::ofstream& out, const char* label, int32_t begin, int32_t end) const { out << label << "="; for (int32_t i = begin; i < end; ++i) { out << (i == begin ? "" : ",") << prompt_token_ids_[i]; } @@ -1559,7 +2156,7 @@ mllm::models::ARGenerationOutputPast makeRequestInputs(Qwen2VLTokenizer& tokeniz bool visual_qnn, const std::vector& visual_bucket_grids) { const auto resize_override = - visual_qnn ? chooseVisualResizeOverrideForOversizedImage(image_path, visual_bucket_grids) : VisualResizeOverride{}; + visual_qnn ? chooseVisualResizeOverrideForImage(image_path, visual_bucket_grids) : VisualResizeOverride{}; if (resize_override.enabled) { fmt::print("[Qwen2VL AOT] visual oversized fallback: native_grid={}x{} patches={} -> resize_grid={}x{} patches={} " "within bucket_grid={}x{} patches={} graph_suffix={}\n", @@ -1573,38 +2170,54 @@ mllm::models::ARGenerationOutputPast makeRequestInputs(Qwen2VLTokenizer& tokeniz resize_override.bucket.grid_w, resize_override.bucket.patchTokens(), visualGraphSuffixForPatchTokens(resize_override.bucket.patchTokens())); + return tokenizer.convertMessage({.prompt = prompt, .img_file_path = image_path}, + resize_override.resize_grid_h, + resize_override.resize_grid_w); } - return resize_override.enabled - ? tokenizer.convertMessage({.prompt = prompt, .img_file_path = image_path}, - resize_override.resize_grid_h, - resize_override.resize_grid_w) - : tokenizer.convertMessage({.prompt = prompt, .img_file_path = image_path}); + return tokenizer.convertMessage({.prompt = prompt, .img_file_path = image_path}); } void runOneRequest(Qwen2VLAOTRunner& runner, Qwen2VLTokenizer& tokenizer, mllm::models::qwen2vl::Qwen2VisionTransformerPretrainedModel* visual, + Qwen2VLPatchEmbedOnly* patch_embed, const std::string& image_path, const std::string& prompt, bool visual_qnn, const std::vector& visual_bucket_grids, - int32_t gen_len) { - auto inputs = makeRequestInputs(tokenizer, image_path, prompt, visual_qnn, visual_bucket_grids); + int32_t gen_len, + bool visual_only = false) { runner.resetStateForRequest(); - auto features = runner.preparePrompt(inputs, visual); - fmt::print("\nResponse: "); - runner.generate(features, tokenizer, gen_len, [](const std::string& token) { std::cout << token << std::flush; }); - fmt::print("\n"); - runner.perfSummary(); + bool ran_llm = false; + { + ScopedProfile request_timer(runner.profiler(), "request.total"); + auto inputs = [&]() { + ScopedProfile timer(runner.profiler(), "request.make_inputs"); + return makeRequestInputs(tokenizer, image_path, prompt, visual_qnn, visual_bucket_grids); + }(); + auto features = runner.preparePrompt(inputs, visual, patch_embed); + if (visual_only) { + fmt::print("\n[Qwen2VL AOT] visual-only mode: skipped LLM prefill/decode after visual embedding dump.\n"); + } else { + fmt::print("\nResponse: "); + runner.generate(features, tokenizer, gen_len, [](const std::string& token) { std::cout << token << std::flush; }); + fmt::print("\n"); + ran_llm = true; + } + } + if (ran_llm) { runner.perfSummary(); } + runner.printProfileSummary(); } void runInteractiveLoop(Qwen2VLAOTRunner& runner, Qwen2VLTokenizer& tokenizer, mllm::models::qwen2vl::Qwen2VisionTransformerPretrainedModel* visual, + Qwen2VLPatchEmbedOnly* patch_embed, bool visual_qnn, const std::vector& visual_bucket_grids, const std::string& default_prompt, - int32_t gen_len) { + int32_t gen_len, + bool visual_only = false) { fmt::print("\n[Qwen2VL AOT] interactive mode is ready. Enter an image path per request; /exit quits.\n"); fmt::print("[Qwen2VL AOT] default prompt: {}\n", default_prompt); @@ -1626,7 +2239,8 @@ void runInteractiveLoop(Qwen2VLAOTRunner& runner, if (prompt.empty()) { prompt = default_prompt; } fmt::print("[Qwen2VL AOT] running request: image={}, gen_len={}\n", image_path, gen_len); - runOneRequest(runner, tokenizer, visual, image_path, prompt, visual_qnn, visual_bucket_grids, gen_len); + runOneRequest(runner, tokenizer, visual, patch_embed, image_path, prompt, visual_qnn, visual_bucket_grids, gen_len, + visual_only); } fmt::print("\n[Qwen2VL AOT] interactive mode stopped.\n"); @@ -1651,6 +2265,8 @@ MLLM_MAIN({ auto& prompt = Argparse::add("-p|--prompt").help("prompt text").def("Describe this picture."); auto& interactive = Argparse::add("--interactive").help("Run repeated image/prompt requests in one process after model initialization."); + auto& visual_only = + Argparse::add("--visual_only").help("Run only image preprocessing/ViT and optional visual embedding dump."); auto& ar_len = Argparse::add("--ar_len").help("prefill graph chunk length").def(32); auto& context_len = Argparse::add("--context_len").help("QNN context length").def(1024); auto& gen_len = Argparse::add("--gen_len").help("max generated tokens").def(96); @@ -1678,6 +2294,14 @@ MLLM_MAIN({ auto& dump_path = Argparse::add("--dump_path") .help("Path on device for debug vector dump.") .def("/data/local/tmp/qwen2vl_qnn_aot_debug_dump.log"); + auto& dump_visual_embeddings = + Argparse::add("--dump_visual_embeddings") + .help("Optional path to write the full cropped visual embeddings as a binary Float32 tensor.") + .def(""); + auto& dump_visual_segments_prefix = + Argparse::add("--dump_visual_segments_prefix") + .help("Optional prefix to write each visual QNN segment output as Float32 tensor binaries.") + .def(""); auto& key_cache_dtype = Argparse::add("--key_cache_dtype") .help("Key cache dtype for experimental contexts: uint8 or uint16. Value cache remains uint8.") @@ -1685,10 +2309,30 @@ MLLM_MAIN({ auto& visual_qnn = Argparse::add("--visual_qnn").help("Use visual graphs from the loaded combined QNN context instead of CPU/KAI visual."); auto& visual_bundle_layout = - Argparse::add("--visual_bundle_layout").help("visual QNN bundle layout: single, 6x8, early2 or tail4.").def("6x8"); + Argparse::add("--visual_bundle_layout") + .help("visual QNN bundle layout: single, hybrid_single, 6x8, early2 or tail4.") + .def("6x8"); + auto& visual_io_dtype = + Argparse::add("--visual_io_dtype") + .help("visual QNN graph input/output dtype: uint16 for quantized visual, fp32/fp16 for raw float single visual.") + .def("uint16"); auto& visual_bucket_grids = Argparse::add("--visual_bucket_grids") .help("Comma-separated visual QNN patch-grid buckets HxW. Example: 10x16,12x16,26x36.") .def(""); + auto& visual_output_scale = + Argparse::add("--visual_output_scale") + .help("Override visual QNN final output UInt16 scale used when dequantizing visual embeddings. Defaults to " + "the baseline FP32/W32A32 visual single-graph QP when --qnn_params has no visual QDQ tensors.") + .def(-1.0f); + auto& visual_output_zero_point = + Argparse::add("--visual_output_zero_point") + .help("Override visual QNN final output UInt16 zero point used when dequantizing visual embeddings. Defaults " + "to the baseline FP32/W32A32 visual single-graph QP when --qnn_params has no visual QDQ tensors.") + .def(-1); + auto& visual_qdq_scale_multiplier = + Argparse::add("--visual_qdq_scale_multiplier") + .help("Runtime multiplier for visual activation QDQ scales. Must match context compilation for hybrid_single.") + .def(1.0f); Argparse::parse(argc, argv); if (help.isSet()) { @@ -1706,10 +2350,20 @@ MLLM_MAIN({ std::cerr << "--key_cache_dtype must be uint8 or uint16\n"; return 1; } + const bool override_visual_output_qp = visual_output_scale.get() > 0.0f || visual_output_zero_point.get() >= 0; + if (override_visual_output_qp && (visual_output_scale.get() <= 0.0f || visual_output_zero_point.get() < 0)) { + std::cerr << "visual output override requires both --visual_output_scale and --visual_output_zero_point\n"; + return 1; + } if (!interactive.isSet() && !image_path.isSet()) { std::cerr << "-i/--image is required unless --interactive is set\n"; return 1; } + const auto parsed_visual_io_dtype = parseVisualIODType(visual_io_dtype.get()); + if (visual_qnn.isSet() && isRawFloatVisualIO(parsed_visual_io_dtype) && visual_bundle_layout.get() != "single") { + std::cerr << "--visual_io_dtype=fp16/fp32 requires --visual_bundle_layout=single\n"; + return 1; + } mllm::initQnnBackend(context_path.get()); @@ -1719,26 +2373,39 @@ MLLM_MAIN({ auto qnn_params = mllm::load(qnn_params_path.get(), mllm::ModelFileVersion::kV2); std::unique_ptr visual; - if (!visual_qnn.isSet()) { + std::unique_ptr patch_embed; + const bool visual_hybrid_single = visual_qnn.isSet() && visual_bundle_layout.get() == "hybrid_single"; + if (!visual_qnn.isSet() || visual_hybrid_single) { if (!visual_model_path.isSet()) { - std::cerr << "--visual_model is required when --visual_qnn is not set\n"; + std::cerr << "--visual_model is required when --visual_qnn is not set or --visual_bundle_layout=hybrid_single\n"; return 1; } mllm::ModelFileVersion visual_file_version = mllm::ModelFileVersion::kV2; if (visual_model_version.get() == "v1") { visual_file_version = mllm::ModelFileVersion::kV1; } auto visual_params = mllm::load(visual_model_path.get(), visual_file_version); - visual = std::make_unique("visual", visual_cfg); - visual->load(visual_params); + if (visual_hybrid_single) { + patch_embed = std::make_unique("visual", visual_cfg); + patch_embed->load(visual_params); + fmt::print("[Qwen2VL AOT] hybrid_single enabled: CPU/KAI patch_embed from --visual_model, QNN visual_body from context.\n"); + } else { + visual = std::make_unique("visual", visual_cfg); + visual->load(visual_params); + } } auto parsed_visual_bucket_grids = parseVisualBucketGrids(visual_bucket_grids.get()); Qwen2VLAOTRunner runner(qnn_cfg, visual_cfg, qnn_params, ar_len.get(), context_len.get(), input_embedding_scale.get(), input_embedding_zero_point.get(), dump_block_outputs.isSet(), dump_layer0_outputs.isSet(), - dump_visual_tokens.isSet(), key_cache_uint16, visual_qnn.isSet(), visual_bundle_layout.get(), + dump_visual_tokens.isSet(), key_cache_uint16, visual_qnn.isSet(), visual_only.isSet(), + visual_bundle_layout.get(), parsed_visual_bucket_grids, + parsed_visual_io_dtype, + visual_output_scale.get(), visual_output_zero_point.get(), visual_qdq_scale_multiplier.get(), dump_logits_topk.get(), - parseDumpTokenIndices(dump_token_indices.get()), dump_path.get()); + parseDumpTokenIndices(dump_token_indices.get()), dump_path.get(), + dump_visual_embeddings.get(), + dump_visual_segments_prefix.get()); if (!runner.load()) { std::cerr << "Failed to load Qwen2-VL QNN AOT runner\n"; return 1; @@ -1748,19 +2415,23 @@ MLLM_MAIN({ runInteractiveLoop(runner, tokenizer, visual.get(), + patch_embed.get(), visual_qnn.isSet(), parsed_visual_bucket_grids, prompt.get(), - gen_len.get()); + gen_len.get(), + visual_only.isSet()); } else { runOneRequest(runner, tokenizer, visual.get(), + patch_embed.get(), image_path.get(), prompt.get(), visual_qnn.isSet(), parsed_visual_bucket_grids, - gen_len.get()); + gen_len.get(), + visual_only.isSet()); } mllm::memoryReport(); return 0; diff --git a/examples/qwen2vl_qnn_aot/compile.cpp b/examples/qwen2vl_qnn_aot/compile.cpp index fe15eac49..2029c68ad 100644 --- a/examples/qwen2vl_qnn_aot/compile.cpp +++ b/examples/qwen2vl_qnn_aot/compile.cpp @@ -1,6 +1,7 @@ // Copyright (c) MLLM Team. // Licensed under the MIT License. +#include #include #include @@ -21,6 +22,39 @@ namespace { constexpr float kDefaultInputEmbeddingScale = 0.002563515f; constexpr int32_t kDefaultInputEmbeddingZeroPoint = 15604; +constexpr float kDefaultVisualPatchInputScale = 4.4f / 65535.f; +constexpr int32_t kDefaultVisualPatchInputZeroPoint = 32768; +constexpr float kDefaultVisualSinCosScale = 2.0f / 65535.f; +constexpr int32_t kDefaultVisualSinCosZeroPoint = 32768; +constexpr float kDefaultVisualAttentionMaskScale = 10000.0f / 65535.f; +constexpr int32_t kDefaultVisualAttentionMaskZeroPoint = 65535; +const std::string kVisualFinalOutputQDQ = "visual.merger.mlp.2_output_qdq"; + +enum class VisualIODType { + kUInt16, + kFloat32, + kFloat16, +}; + +std::string firstHybridBodyInputQDQName() { return "visual.blocks.0.attn.qkv_input_qdq"; } + +VisualIODType parseVisualIODType(const std::string& dtype) { + if (dtype == "uint16") { return VisualIODType::kUInt16; } + if (dtype == "fp32" || dtype == "float32") { return VisualIODType::kFloat32; } + if (dtype == "fp16" || dtype == "float16") { return VisualIODType::kFloat16; } + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "--visual_io_dtype must be uint16, fp32, or fp16."); +} + +mllm::DataTypes visualFloatDType(VisualIODType dtype) { + switch (dtype) { + case VisualIODType::kFloat32: return mllm::kFloat32; + case VisualIODType::kFloat16: return mllm::kFloat16; + case VisualIODType::kUInt16: + default: return mllm::kFloat32; + } +} + +bool isRawFloatVisualIO(VisualIODType dtype) { return dtype == VisualIODType::kFloat32 || dtype == VisualIODType::kFloat16; } template void addCausalMaskParams(const ParamsT& params) { @@ -30,6 +64,49 @@ void addCausalMaskParams(const ParamsT& params) { params->push("constant_zero.zero_point", mllm::Tensor::constant(65535, mllm::kInt32)); } +template +void replaceParam(const ParamsT& params, const std::string& name, const mllm::Tensor& tensor) { + if (params->has(name)) { params->remove(name); } + params->push(name, tensor); +} + +template +void overrideVisualFinalOutputQDQ(const ParamsT& params, float scale, int32_t zero_point) { + replaceParam(params, kVisualFinalOutputQDQ + ".fake_quant.scale", mllm::Tensor::constant(scale, mllm::kFloat32)); + replaceParam(params, kVisualFinalOutputQDQ + ".fake_quant.zero_point", mllm::Tensor::constant(zero_point, mllm::kInt32)); + fmt::print("[Qwen2VL AOT Compile] override {}: scale={:.9f}, zero_point={}\n", + kVisualFinalOutputQDQ, + scale, + zero_point); +} + +template +void scaleVisualActivationQDQ(const ParamsT& params, float multiplier) { + if (multiplier <= 0.0f || std::abs(multiplier - 1.0f) < 1e-6f) { return; } + + int32_t patched = 0; + std::vector names; + names.reserve(params->dict().size()); + for (auto it = params->begin(); it != params->end(); ++it) { + names.push_back(it->first); + } + + for (const auto& name : names) { + if (name.rfind("visual.", 0) != 0) { continue; } + const std::string scale_suffix = ".fake_quant.scale"; + if (name.size() < scale_suffix.size() || name.compare(name.size() - scale_suffix.size(), scale_suffix.size(), scale_suffix) != 0) { + continue; + } + if (name.find("_input_qdq.") == std::string::npos && name.find("_output_qdq.") == std::string::npos) { continue; } + + mllm::Tensor scale_tensor = params->pull(name); + const auto old_scale = scale_tensor.item(); + replaceParam(params, name, mllm::Tensor::constant(old_scale * multiplier, mllm::kFloat32)); + ++patched; + } + fmt::print("[Qwen2VL AOT Compile] scaled {} visual activation QDQ scale tensor(s) by {:.6f}\n", patched, multiplier); +} + template mllm::Tensor makeUInt16AsymTensor(const std::vector& shape, const std::string& scale_name, @@ -60,6 +137,260 @@ inline mllm::Tensor makeUInt16AsymTensor(const std::vector& shape, floa return tensor; } +mllm::Tensor makeVisualTraceTensor(const std::vector& shape, + VisualIODType visual_io_dtype, + float scale, + int32_t zero_point) { + if (visual_io_dtype == VisualIODType::kUInt16) { return makeUInt16AsymTensor(shape, scale, zero_point); } + return mllm::Tensor::empty(shape, visualFloatDType(visual_io_dtype), mllm::kCPU).alloc(); +} + +class RawPatchEmbedLinear final : public mllm::nn::Module { + int32_t patch_dim_ = 0; + int32_t embed_dim_ = 0; + + mllm::nn::Linear proj_; + + public: + RawPatchEmbedLinear() = default; + + RawPatchEmbedLinear(const std::string& name, const mllm::models::qwen2vl::Qwen2VLConfig& cfg) + : mllm::nn::Module(name) { + patch_dim_ = cfg.visual_in_chans * cfg.visual_temporal_patch_size * cfg.visual_patch_size * cfg.visual_patch_size; + embed_dim_ = cfg.visual_embed_dim; + proj_ = reg("proj", patch_dim_, embed_dim_, false, mllm::aops::LinearImplTypes::kDefault); + } + + std::vector forward(const std::vector& inputs, + const std::vector& /*args*/) override { + auto hidden_states = inputs[0]; + hidden_states = hidden_states.view({-1, patch_dim_}, true); + hidden_states = proj_(hidden_states).view({-1, embed_dim_}, true); + return {hidden_states}; + } +}; + +class RawVisionMlpPrimitiveQuickGELU final : public mllm::nn::Module { + int32_t dim_ = 0; + int32_t hidden_dim_ = 0; + + mllm::nn::Linear fc_1_; + mllm::nn::Linear fc_2_; + + public: + RawVisionMlpPrimitiveQuickGELU() = default; + + RawVisionMlpPrimitiveQuickGELU(const std::string& name, const mllm::models::qwen2vl::Qwen2VLConfig& cfg) + : mllm::nn::Module(name) { + dim_ = cfg.visual_embed_dim; + hidden_dim_ = cfg.visual_embed_dim * cfg.visual_mlp_ratio; + fc_1_ = reg("fc1", dim_, hidden_dim_, true, mllm::aops::LinearImplTypes::kDefault); + fc_2_ = reg("fc2", hidden_dim_, dim_, true, mllm::aops::LinearImplTypes::kDefault); + } + + std::vector forward(const std::vector& inputs, + const std::vector& /*args*/) override { + auto x = fc_1_(inputs[0]); + x = x * mllm::nn::functional::sigmoid(x * 1.702f); + return {fc_2_(x)}; + } +}; + +class RawVisionAttentionMaskedAOTRewrite final : public mllm::nn::Module { + int32_t dim_ = 0; + int32_t num_heads_ = 0; + int32_t head_dim_ = 0; + + mllm::nn::Linear qkv_; + mllm::nn::Linear proj_; + mllm::nn::Softmax softmax_; + + public: + RawVisionAttentionMaskedAOTRewrite() = default; + + RawVisionAttentionMaskedAOTRewrite(const std::string& name, const mllm::models::qwen2vl::Qwen2VLConfig& cfg) + : mllm::nn::Module(name) { + dim_ = cfg.visual_embed_dim; + num_heads_ = cfg.visual_num_heads; + head_dim_ = dim_ / num_heads_; + qkv_ = reg("qkv", dim_, dim_ * 3, true, mllm::aops::LinearImplTypes::kDefault); + proj_ = reg("proj", dim_, dim_, true, mllm::aops::LinearImplTypes::kDefault); + softmax_ = reg("softmax", -1); + } + + mllm::Tensor applyVisionRoPEPrimitive(mllm::Tensor x, mllm::Tensor visual_embedding_sin, mllm::Tensor visual_embedding_cos) { + const int32_t half_dim = head_dim_ / 2; + auto x1 = x.slice({mllm::kAll, mllm::kAll, mllm::kAll, {mllm::kAll, half_dim}}, true); + auto x2 = x.slice({mllm::kAll, mllm::kAll, mllm::kAll, {half_dim, mllm::kAll}}, true); + auto sin = visual_embedding_sin; + auto cos = visual_embedding_cos; + if (sin.rank() == 2) { + sin = sin.view({1, -1, 1, half_dim}, true); + cos = cos.view({1, -1, 1, half_dim}, true); + } else { + MLLM_RT_ASSERT_EQ(sin.rank(), 4); + MLLM_RT_ASSERT_EQ(cos.rank(), 4); + } + auto y1 = x1 * cos + (-(x2 * sin)); + auto y2 = x1 * sin + x2 * cos; + return mllm::nn::functional::concat({y1, y2}, -1); + } + + std::vector forward(const std::vector& inputs, + const std::vector& /*args*/) override { + auto hidden_states = inputs[0]; + auto visual_embedding_sin = inputs[1]; + auto visual_embedding_cos = inputs[2]; + auto attention_mask = inputs.size() > 3 ? inputs[3] : mllm::Tensor::nil(); + + auto qkv_states = qkv_(hidden_states).view({-1, 3, num_heads_, head_dim_}, true); + auto query_states = qkv_states.slice({mllm::kAll, {0, 1}, mllm::kAll, mllm::kAll}, true).transpose(0, 1); + auto key_states = qkv_states.slice({mllm::kAll, {1, 2}, mllm::kAll, mllm::kAll}, true).transpose(0, 1); + auto value_states = qkv_states.slice({mllm::kAll, {2, 3}, mllm::kAll, mllm::kAll}, true).transpose(0, 1); + + query_states = applyVisionRoPEPrimitive(query_states, visual_embedding_sin, visual_embedding_cos); + key_states = applyVisionRoPEPrimitive(key_states, visual_embedding_sin, visual_embedding_cos); + + query_states = query_states.transpose(1, 2); + key_states = key_states.transpose(1, 2); + value_states = value_states.transpose(1, 2); + + auto attn = mllm::nn::functional::matmul(query_states, key_states, false, true) + * (1.f / std::sqrt(static_cast(head_dim_))); + if (!attention_mask.isNil()) { attn = attn + attention_mask; } + attn = softmax_(attn); + + auto attn_output = mllm::nn::functional::matmul(attn, value_states); + attn_output = attn_output.transpose(1, 2).view({-1, dim_}, true); + return {proj_(attn_output)}; + } +}; + +class RawQwen2VLVisionBlockAOTRewrite final : public mllm::nn::Module { + mllm::nn::LayerNorm norm1_; + mllm::nn::LayerNorm norm2_; + + RawVisionAttentionMaskedAOTRewrite attn_; + RawVisionMlpPrimitiveQuickGELU mlp_; + + public: + RawQwen2VLVisionBlockAOTRewrite() = default; + + RawQwen2VLVisionBlockAOTRewrite(const std::string& name, const mllm::models::qwen2vl::Qwen2VLConfig& cfg) + : mllm::nn::Module(name) { + norm1_ = reg("norm1", std::vector{cfg.visual_embed_dim}, true, true, 1e-6); + norm2_ = reg("norm2", std::vector{cfg.visual_embed_dim}, true, true, 1e-6); + attn_ = reg("attn", cfg); + mlp_ = reg("mlp", cfg); + } + + std::vector forward(const std::vector& inputs, + const std::vector& /*args*/) override { + auto hidden_states = inputs[0]; + auto visual_embedding_sin = inputs[1]; + auto visual_embedding_cos = inputs[2]; + auto attention_mask = inputs.size() > 3 ? inputs[3] : mllm::Tensor::nil(); + + auto norm1_out = norm1_(hidden_states); + mllm::Tensor attn_out; + if (attention_mask.isNil()) { + attn_out = attn_(norm1_out, visual_embedding_sin, visual_embedding_cos)[0]; + } else { + attn_out = attn_(norm1_out, visual_embedding_sin, visual_embedding_cos, attention_mask)[0]; + } + hidden_states = hidden_states + attn_out; + hidden_states = hidden_states + mlp_(norm2_(hidden_states))[0]; + return {hidden_states}; + } +}; + +class RawPatchMergerAOTRewrite final : public mllm::nn::Module { + int32_t hidden_size_ = 0; + int32_t spatial_merge_size_ = 0; + int32_t context_dim_ = 0; + + mllm::nn::LayerNorm ln_q_; + mllm::nn::Linear mlp_0_; + mllm::nn::Linear mlp_2_; + mllm::nn::GELU mlp_gelu_; + + public: + RawPatchMergerAOTRewrite() = default; + + RawPatchMergerAOTRewrite(const std::string& name, const mllm::models::qwen2vl::Qwen2VLConfig& cfg) + : mllm::nn::Module(name) { + context_dim_ = cfg.visual_embed_dim; + spatial_merge_size_ = cfg.visual_spatial_merge_size; + hidden_size_ = context_dim_ * spatial_merge_size_ * spatial_merge_size_; + + ln_q_ = reg("ln_q", std::vector{context_dim_}, true, true, 1e-6); + mlp_0_ = reg("mlp.0", hidden_size_, hidden_size_, true, mllm::aops::LinearImplTypes::kDefault); + mlp_gelu_ = reg("mlp.gelu"); + mlp_2_ = reg("mlp.2", hidden_size_, cfg.hidden_size, true, mllm::aops::LinearImplTypes::kDefault); + } + + std::vector forward(const std::vector& inputs, + const std::vector& /*args*/) override { + auto o = ln_q_(inputs[0]).view({-1, hidden_size_}, true); + o = mlp_0_(o); + o = mlp_gelu_(o); + o = mlp_2_(o); + return {o}; + } +}; + +class RawQwen2VisionTransformerPretrainedModelAOTRewrite final : public mllm::nn::Module { + RawPatchEmbedLinear patch_embed_; + RawPatchMergerAOTRewrite patch_merger_; + mllm::nn::ModuleList blocks_; + int32_t start_block_ = 0; + int32_t active_blocks_ = -1; + bool skip_merger_ = false; + bool skip_patch_embed_ = false; + + public: + RawQwen2VisionTransformerPretrainedModelAOTRewrite() = default; + + RawQwen2VisionTransformerPretrainedModelAOTRewrite(const std::string& name, + const mllm::models::qwen2vl::Qwen2VLConfig& cfg, + int32_t start_block = 0, + int32_t active_blocks = -1, + bool skip_merger = false, + bool skip_patch_embed = false) + : mllm::nn::Module(name) { + start_block_ = start_block; + active_blocks_ = active_blocks; + skip_merger_ = skip_merger; + skip_patch_embed_ = skip_patch_embed; + patch_embed_ = reg("patch_embed", cfg); + patch_merger_ = reg("merger", cfg); + blocks_ = reg>("blocks", cfg.visual_depth, cfg); + } + + std::vector forward(const std::vector& inputs, + const std::vector& /*args*/) override { + auto hidden_states = inputs[0]; + auto embedding_sin = inputs[1]; + auto embedding_cos = inputs[2]; + auto attention_mask = inputs.size() > 3 ? inputs[3] : mllm::Tensor::nil(); + + if (!skip_patch_embed_) { hidden_states = patch_embed_(hidden_states)[0]; } + auto num_blocks = active_blocks_ < 0 ? static_cast(blocks_.list().size()) : active_blocks_; + MLLM_RT_ASSERT(start_block_ >= 0); + MLLM_RT_ASSERT(num_blocks >= 0); + MLLM_RT_ASSERT(start_block_ + num_blocks <= static_cast(blocks_.list().size())); + for (int32_t i = 0; i < num_blocks; ++i) { + if (attention_mask.isNil()) { + hidden_states = blocks_.list()[start_block_ + i](hidden_states, embedding_sin, embedding_cos)[0]; + } else { + hidden_states = blocks_.list()[start_block_ + i](hidden_states, embedding_sin, embedding_cos, attention_mask)[0]; + } + } + if (!skip_merger_) { hidden_states = patch_merger_(hidden_states)[0]; } + return {hidden_states}; + } +}; + template std::unordered_map makeTraceInputs(int seq_len, int context_len, @@ -151,12 +482,25 @@ void compileVisualBundleGraphs(mllm::qnn::aot::QnnAOTEnv& qnn_aot_env, int32_t patch_flat_dim, const std::string& bundle_layout, const std::string& visual_ir_prefix, - const std::string& graph_suffix) { + const std::string& graph_suffix, + VisualIODType visual_io_dtype) { + if (isRawFloatVisualIO(visual_io_dtype) && bundle_layout != "single") { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, + "--visual_io_dtype=fp16/fp32 is currently supported only with --visual_bundle_layout=single."); + } const int32_t half_dim = visual_cfg.visual_embed_dim / visual_cfg.visual_num_heads / 2; - auto visual_embedding_sin = mllm::Tensor::empty({1, visual_patch_tokens, 1, half_dim}, mllm::kFloat32, mllm::kCPU).alloc(); - auto visual_embedding_cos = mllm::Tensor::empty({1, visual_patch_tokens, 1, half_dim}, mllm::kFloat32, mllm::kCPU).alloc(); - auto visual_attention_mask = - mllm::Tensor::empty({1, 1, 1, visual_patch_tokens}, mllm::kFloat32, mllm::kCPU).alloc(); + auto visual_embedding_sin = makeVisualTraceTensor({1, visual_patch_tokens, 1, half_dim}, + visual_io_dtype, + kDefaultVisualSinCosScale, + kDefaultVisualSinCosZeroPoint); + auto visual_embedding_cos = makeVisualTraceTensor({1, visual_patch_tokens, 1, half_dim}, + visual_io_dtype, + kDefaultVisualSinCosScale, + kDefaultVisualSinCosZeroPoint); + auto visual_attention_mask = makeVisualTraceTensor({1, 1, 1, visual_patch_tokens}, + visual_io_dtype, + kDefaultVisualAttentionMaskScale, + kDefaultVisualAttentionMaskZeroPoint); const auto segments = qwen2vl_qnn_aot::makeVisualBundleSegments(bundle_layout, visual_cfg.visual_depth, @@ -167,23 +511,65 @@ void compileVisualBundleGraphs(mllm::qnn::aot::QnnAOTEnv& qnn_aot_env, for (const auto& segment : segments) { fmt::print("\n{:=^72}\n", fmt::format(" Compile {} ", segment.graph_name)); - auto segment_img = mllm::Tensor::empty(segment.input_shape, mllm::kFloat32, mllm::kCPU).alloc(); - auto visual = qwen2vl_qnn_aot::Qwen2VisionTransformerPretrainedModelAOTRewrite("visual", - visual_cfg, - segment.start_block, - segment.visual_blocks, - segment.skip_merger, - segment.skip_patch_embed); - visual.load(visual_params); - + const bool quantized_patch_embed = + !segment.skip_patch_embed && visual_params->has("visual.patch_embed.proj.weight") + && visual_params->pull("visual.patch_embed.proj.weight").dtype() == mllm::kInt8; + const auto segment_input_qdq = + qwen2vl_qnn_aot::visualSegmentInputQDQName(segment, visual_cfg.visual_depth); + auto segment_img = [&]() { + if (isRawFloatVisualIO(visual_io_dtype)) { + return mllm::Tensor::empty(segment.input_shape, visualFloatDType(visual_io_dtype), mllm::kCPU).alloc(); + } + return quantized_patch_embed + ? makeUInt16AsymTensor(segment.input_shape, kDefaultVisualPatchInputScale, kDefaultVisualPatchInputZeroPoint) + : (!segment_input_qdq.empty() && visual_params->has(segment_input_qdq + ".fake_quant.scale") + ? makeUInt16AsymTensor(segment.input_shape, + segment_input_qdq + ".fake_quant.scale", + segment_input_qdq + ".fake_quant.zero_point", + visual_params) + : mllm::Tensor::empty(segment.input_shape, mllm::kFloat32, mllm::kCPU).alloc()); + }(); + + mllm::Tensor visual_output; mllm::ir::lowlevel::traceStart(); - auto visual_output = segment.visual_blocks > 0 - ? mllm::ir::lowlevel::traceModule(visual, - segment_img, - visual_embedding_sin, - visual_embedding_cos, - visual_attention_mask)[0] - : mllm::ir::lowlevel::traceModule(visual, segment_img, visual_embedding_sin, visual_embedding_cos)[0]; + if (isRawFloatVisualIO(visual_io_dtype)) { + auto visual = RawQwen2VisionTransformerPretrainedModelAOTRewrite("visual", + visual_cfg, + segment.start_block, + segment.visual_blocks, + segment.skip_merger, + segment.skip_patch_embed); + visual.load(visual_params); + visual_output = segment.visual_blocks > 0 + ? mllm::ir::lowlevel::traceModule(visual, + segment_img, + visual_embedding_sin, + visual_embedding_cos, + visual_attention_mask)[0] + : mllm::ir::lowlevel::traceModule(visual, segment_img, visual_embedding_sin, visual_embedding_cos)[0]; + } else { + auto visual = qwen2vl_qnn_aot::Qwen2VisionTransformerPretrainedModelAOTRewrite("visual", + visual_cfg, + segment.start_block, + segment.visual_blocks, + segment.skip_merger, + segment.skip_patch_embed); + visual.load(visual_params); + visual_output = segment.visual_blocks > 0 + ? mllm::ir::lowlevel::traceModule(visual, + segment_img, + visual_embedding_sin, + visual_embedding_cos, + visual_attention_mask)[0] + : mllm::ir::lowlevel::traceModule(visual, segment_img, visual_embedding_sin, visual_embedding_cos)[0]; + } + const auto segment_output_qdq = + qwen2vl_qnn_aot::visualGraphOutputQDQName(segment.graph_name, visual_cfg.visual_depth); + if (!isRawFloatVisualIO(visual_io_dtype) && !segment_output_qdq.empty() && visual_params->has(segment_output_qdq + ".fake_quant.scale") + && visual_params->has(segment_output_qdq + ".fake_quant.zero_point")) { + visual_output.attach("scale", visual_params->pull(segment_output_qdq + ".fake_quant.scale").impl(), true); + visual_output.attach("zero_point", visual_params->pull(segment_output_qdq + ".fake_quant.zero_point").impl(), true); + } auto visual_ir = mllm::ir::lowlevel::traceStop(); fmt::print("visual segment output shape: [{}, {}]\n", visual_output.shape()[0], visual_output.shape()[1]); @@ -210,7 +596,8 @@ void compileVisualBundleGraphsFromImage(mllm::qnn::aot::QnnAOTEnv& qnn_aot_env, const std::string& image_path, const std::string& prompt, const std::string& bundle_layout, - const std::string& visual_ir_prefix) { + const std::string& visual_ir_prefix, + VisualIODType visual_io_dtype) { if (tokenizer_path.empty() || image_path.empty()) { MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "--include_visual_bundle requires --tokenizer and --image."); } @@ -226,7 +613,8 @@ void compileVisualBundleGraphsFromImage(mllm::qnn::aot::QnnAOTEnv& qnn_aot_env, img.shape()[1], bundle_layout, visual_ir_prefix, - ""); + "", + visual_io_dtype); } } // namespace @@ -274,9 +662,25 @@ MLLM_MAIN({ Argparse::add("--visual_bundle_layout") .def("6x8") .help("Visual bundle layout: single, 6x8, tail4, early2 or block1."); + auto& visual_io_dtype = + Argparse::add("--visual_io_dtype") + .def("uint16") + .help("Visual graph input/output dtype: uint16 for quantized LPBQ visual, fp32/fp16 for raw float single visual graph."); auto& visual_bucket_grids = Argparse::add("--visual_bucket_grids") .def("") .help("Comma-separated visual patch grid buckets HxW. Example: 10x16,12x16,26x36."); + auto& visual_output_scale = + Argparse::add("--visual_output_scale") + .def(-1.0f) + .help("Override visual final output UInt16 scale for visual bundle graph compilation."); + auto& visual_output_zero_point = + Argparse::add("--visual_output_zero_point") + .def(-1) + .help("Override visual final output UInt16 zero point for visual bundle graph compilation."); + auto& visual_qdq_scale_multiplier = + Argparse::add("--visual_qdq_scale_multiplier") + .def(1.0f) + .help("Multiply visual *_input_qdq/*_output_qdq scale tensors before compiling visual graphs."); auto& tokenizer_path = Argparse::add("-t|--tokenizer").help("tokenizer.json path for visual input shape."); auto& image_path = Argparse::add("-i|--image").help("image path for visual input shape."); auto& prompt = Argparse::add("-p|--prompt").help("prompt text used with --image.").def("describe this picture"); @@ -309,6 +713,16 @@ MLLM_MAIN({ if (!key_cache_uint16 && key_cache_dtype.get() != "uint8") { MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "--key_cache_dtype must be uint8 or uint16."); } + const bool override_visual_output_qp = visual_output_scale.get() > 0.0f || visual_output_zero_point.get() >= 0; + if (override_visual_output_qp && (visual_output_scale.get() <= 0.0f || visual_output_zero_point.get() < 0)) { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, + "visual output override requires both --visual_output_scale and --visual_output_zero_point."); + } + const auto parsed_visual_io_dtype = parseVisualIODType(visual_io_dtype.get()); + if (isRawFloatVisualIO(parsed_visual_io_dtype) && visual_bundle_layout.get() != "single") { + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, + "--visual_io_dtype=fp16/fp32 requires --visual_bundle_layout=single for now."); + } auto model_cfg = mllm::models::qwen2vl::Qwen2VLConfig(model_cfg_path.get()); auto params = mllm::load(model_path.get(), mllm::ModelFileVersion::kV2); @@ -358,6 +772,10 @@ MLLM_MAIN({ const auto visual_aot_config = visual_aot_config_path.isSet() ? visual_aot_config_path.get() : qnn_aot_cfg_files.get(); auto visual_cfg = mllm::models::qwen2vl::Qwen2VLConfig(visual_config); auto visual_params = mllm::load(visual_model, mllm::ModelFileVersion::kV2); + if (override_visual_output_qp) { + overrideVisualFinalOutputQDQ(visual_params, visual_output_scale.get(), visual_output_zero_point.get()); + } + scaleVisualActivationQDQ(visual_params, visual_qdq_scale_multiplier.get()); qwen2vl_qnn_aot::reshapePatchEmbedConv3DWeightForLinear(visual_params, visual_cfg); const int32_t patch_flat_dim = @@ -377,7 +795,8 @@ MLLM_MAIN({ patch_flat_dim, visual_bundle_layout.get(), visual_ir_prefix.get(), - qwen2vl_qnn_aot::visualGraphSuffixForPatchTokens(patch_tokens)); + qwen2vl_qnn_aot::visualGraphSuffixForPatchTokens(patch_tokens), + parsed_visual_io_dtype); } } else { compileVisualBundleGraphsFromImage(qnn_aot_env, @@ -388,7 +807,8 @@ MLLM_MAIN({ image_path.get(), prompt.get(), visual_bundle_layout.get(), - visual_ir_prefix.get()); + visual_ir_prefix.get(), + parsed_visual_io_dtype); } } diff --git a/examples/qwen2vl_qnn_aot/visual_aot_helpers.hpp b/examples/qwen2vl_qnn_aot/visual_aot_helpers.hpp index c3dd69246..6d1baebf5 100644 --- a/examples/qwen2vl_qnn_aot/visual_aot_helpers.hpp +++ b/examples/qwen2vl_qnn_aot/visual_aot_helpers.hpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -286,6 +287,46 @@ inline std::vector uniqueVisualBucketPatchTokens(const std::vector= graph_name.size()) { return graph_name; } + const auto suffix_is_digits = + std::all_of(graph_name.begin() + static_cast(pos + 2), + graph_name.end(), + [](unsigned char ch) { return std::isdigit(ch); }); + return suffix_is_digits ? graph_name.substr(0, pos) : graph_name; +} + +inline std::string visualGraphOutputQDQName(const std::string& graph_name, int32_t visual_depth) { + const auto base = stripVisualGraphSuffix(graph_name); + if (base == "visual_full" || base == "visual_body" || base == "visual_merger") { + return "visual.merger.mlp.2_output_qdq"; + } + if (base == "visual_patch_embed") { return "visual.blocks.0.attn.qkv_input_qdq"; } + + constexpr const char* prefix = "visual_blocks_"; + if (base.rfind(prefix, 0) == 0) { + const auto mid = base.find('_', std::strlen(prefix)); + if (mid != std::string::npos) { + const auto end_block = std::stoi(base.substr(mid + 1)); + return end_block < visual_depth ? "visual.blocks." + std::to_string(end_block) + ".attn.qkv_input_qdq" + : "visual.merger.mlp.0_input_qdq"; + } + } + + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, "Cannot infer visual output QDQ for graph {}", graph_name); +} + +inline std::string visualSegmentInputQDQName(const VisualSegment& segment, int32_t visual_depth) { + if (!segment.skip_patch_embed) { return ""; } + if (segment.visual_blocks > 0) { return visualBlockInputQDQName(segment.start_block); } + return segment.start_block >= visual_depth ? "" : "visual.merger.mlp.0_input_qdq"; +} + inline void reshapePatchEmbedConv3DWeightForLinear(const mllm::ParameterFile::ptr_t& params, const mllm::models::qwen2vl::Qwen2VLConfig& cfg) { const std::string weight_name = "visual.patch_embed.proj.weight"; @@ -295,6 +336,13 @@ inline void reshapePatchEmbedConv3DWeightForLinear(const mllm::ParameterFile::pt auto weight = params->pull(weight_name); MLLM_RT_ASSERT_EQ(weight.numel(), cfg.visual_embed_dim * patch_dim); + if (weight.dtype() == mllm::kInt8) { + if (weight.rank() == 4 && weight.size(-2) == patch_dim && weight.size(-1) == cfg.visual_embed_dim) { return; } + MLLM_ERROR_EXIT(mllm::ExitCode::kCoreError, + "{} is already int8 but has unexpected LPBQ layout; expected [1,1,{},{}], got rank={}, last2=[{},{}]", + weight_name, patch_dim, cfg.visual_embed_dim, weight.rank(), weight.size(-2), weight.size(-1)); + } + params->remove(weight_name); params->push(weight_name, weight.view({cfg.visual_embed_dim, patch_dim})); } @@ -310,6 +358,10 @@ inline std::vector makeVisualBundleSegments(const std::string& la segments = { {"visual_full" + graph_suffix, 0, visual_depth, false, false, {visual_patch_tokens, patch_flat_dim}, "full" + graph_suffix}, }; + } else if (layout == "hybrid_single") { + segments = { + {"visual_body" + graph_suffix, 0, visual_depth, true, false, {visual_patch_tokens, cfg.visual_embed_dim}, "body" + graph_suffix}, + }; } else if (layout == "6x8") { segments = { {"visual_patch_embed" + graph_suffix, 0, 0, false, true, {visual_patch_tokens, patch_flat_dim}, "patch_embed" + graph_suffix}, diff --git a/mllm/core/aops/ElewiseOps.cpp b/mllm/core/aops/ElewiseOps.cpp index 1bf8c60dd..9a011b9ab 100644 --- a/mllm/core/aops/ElewiseOps.cpp +++ b/mllm/core/aops/ElewiseOps.cpp @@ -44,6 +44,12 @@ static std::vector broadcastShapes(const std::vector>& sha i_irs[1]->setAttr("constant", ctx->create(inputs[1].toVector())); \ break; \ } \ + case kFloat16: { \ + auto fp16_constant = inputs[1]; \ + auto fp32_constant = fp16_constant.to(kFloat32); \ + i_irs[1]->setAttr("constant", ctx->create(fp32_constant.toVector())); \ + break; \ + } \ case kInt16: { \ i_irs[1]->setAttr("constant", ctx->create(inputs[1].toVector())); \ break; \ diff --git a/scripts/qwen2vl_qnn/run_qnn_eval_fixed_set.sh b/scripts/qwen2vl_qnn/run_qnn_eval_fixed_set.sh index 89d366ca3..4b0d0799d 100755 --- a/scripts/qwen2vl_qnn/run_qnn_eval_fixed_set.sh +++ b/scripts/qwen2vl_qnn/run_qnn_eval_fixed_set.sh @@ -12,8 +12,8 @@ REMOTE_QNN_DIR="${REMOTE_QNN_DIR:-/data/local/tmp/mllm-qwen2vl-qnn}" REMOTE_CPU_DIR="${REMOTE_CPU_DIR:-/data/local/tmp/mllm-qwen2vl}" REMOTE_IMAGE_DIR="${REMOTE_IMAGE_DIR:-${REMOTE_CPU_DIR}/images/eval}" -CONTEXT="${CONTEXT:-models/qwen2vl-2b-sm8650-qnn234-fullqnn-5bucket-baseline-v1.bin}" -QNN_PARAMS="${QNN_PARAMS:-models/qwen2vl-2b-sm8650-qnn234-lpbq-baseline-v1.mllm}" +CONTEXT="${CONTEXT:-models/qwen2vl-2b-sm8650-qnn234-fullqnn-5bucket-visualfp16-v1.bin}" +QNN_PARAMS="${QNN_PARAMS:-models/qwen2vl-2b-sm8650-qnn234-lpbq-visualfp16-vprojg16.mllm}" TOKENIZER="${TOKENIZER:-${REMOTE_CPU_DIR}/tokenizer/tokenizer.json}" QNN_CONFIG="${QNN_CONFIG:-config/config_2B_qnn_lpbq.json}" VISUAL_MODEL="${VISUAL_MODEL:-${REMOTE_CPU_DIR}/models/qwen2vl-2b-w4a32-kai.mllm}" @@ -21,6 +21,7 @@ VISUAL_CONFIG="${VISUAL_CONFIG:-${REMOTE_CPU_DIR}/config/config_2B_w32a32.json}" VISUAL_MODEL_VERSION="${VISUAL_MODEL_VERSION:-v2}" VISUAL_QNN="${VISUAL_QNN:-1}" VISUAL_BUNDLE_LAYOUT="${VISUAL_BUNDLE_LAYOUT:-single}" +VISUAL_IO_DTYPE="${VISUAL_IO_DTYPE:-fp16}" VISUAL_BUCKET_GRIDS="${VISUAL_BUCKET_GRIDS:-12x16,16x24,24x24,24x32,18x52,52x18,26x36,36x26}" AR_LEN="${AR_LEN:-32}" @@ -28,8 +29,10 @@ CONTEXT_LEN="${CONTEXT_LEN:-1024}" GEN_LEN="${GEN_LEN:-120}" INPUT_EMBEDDING_SCALE="${INPUT_EMBEDDING_SCALE:-0.002563515}" INPUT_EMBEDDING_ZERO_POINT="${INPUT_EMBEDDING_ZERO_POINT:-15604}" +VISUAL_OUTPUT_SCALE="${VISUAL_OUTPUT_SCALE:--1}" +VISUAL_OUTPUT_ZERO_POINT="${VISUAL_OUTPUT_ZERO_POINT:--1}" KEY_CACHE_DTYPE="${KEY_CACHE_DTYPE:-uint8}" -DUMP_STATS="${DUMP_STATS:-1}" +DUMP_STATS="${DUMP_STATS:-0}" PUSH_ASSETS="${PUSH_ASSETS:-1}" OUT_ROOT="${OUT_ROOT:-${QWEN2VL_QNN_REPO_ROOT}/logs/qwen2vl_qnn_eval}" RUN_ID="${RUN_ID:-$(timestamp)}" @@ -53,12 +56,15 @@ visual_model=${VISUAL_MODEL} visual_config=${VISUAL_CONFIG} visual_qnn=${VISUAL_QNN} visual_bundle_layout=${VISUAL_BUNDLE_LAYOUT} +visual_io_dtype=${VISUAL_IO_DTYPE} visual_bucket_grids=${VISUAL_BUCKET_GRIDS} ar_len=${AR_LEN} context_len=${CONTEXT_LEN} gen_len=${GEN_LEN} input_embedding_scale=${INPUT_EMBEDDING_SCALE} input_embedding_zero_point=${INPUT_EMBEDDING_ZERO_POINT} +visual_output_scale=${VISUAL_OUTPUT_SCALE} +visual_output_zero_point=${VISUAL_OUTPUT_ZERO_POINT} key_cache_dtype=${KEY_CACHE_DTYPE} EOF @@ -70,10 +76,13 @@ while IFS='|' read -r case_id image_file prompt; do log_file="${OUT_DIR}/${case_id}.log" visual_args="" if [[ "${VISUAL_QNN}" == "1" ]]; then - visual_args="--visual_qnn --visual_bundle_layout $(remote_quote "${VISUAL_BUNDLE_LAYOUT}")" + visual_args="--visual_qnn --visual_bundle_layout $(remote_quote "${VISUAL_BUNDLE_LAYOUT}") --visual_io_dtype $(remote_quote "${VISUAL_IO_DTYPE}")" if [[ -n "${VISUAL_BUCKET_GRIDS}" ]]; then visual_args="${visual_args} --visual_bucket_grids $(remote_quote "${VISUAL_BUCKET_GRIDS}")" fi + if [[ "${VISUAL_BUNDLE_LAYOUT}" == "hybrid_single" ]]; then + visual_args="${visual_args} --visual_model $(remote_quote "${VISUAL_MODEL}") --visual_model_version $(remote_quote "${VISUAL_MODEL_VERSION}")" + fi else visual_args="--visual_model $(remote_quote "${VISUAL_MODEL}") --visual_model_version $(remote_quote "${VISUAL_MODEL_VERSION}")" fi @@ -103,6 +112,8 @@ export ADSP_LIBRARY_PATH=\"\$PWD/lib;/vendor/dsp/cdsp;/vendor/lib/rfsa/adsp;/sys --gen_len ${GEN_LEN} \ --input_embedding_scale ${INPUT_EMBEDDING_SCALE} \ --input_embedding_zero_point ${INPUT_EMBEDDING_ZERO_POINT} \ + --visual_output_scale ${VISUAL_OUTPUT_SCALE} \ + --visual_output_zero_point ${VISUAL_OUTPUT_ZERO_POINT} \ --key_cache_dtype $(remote_quote "${KEY_CACHE_DTYPE}") " diff --git a/scripts/qwen2vl_qnn/run_qnn_interactive.sh b/scripts/qwen2vl_qnn/run_qnn_interactive.sh index ee1a772bc..ece834aa2 100755 --- a/scripts/qwen2vl_qnn/run_qnn_interactive.sh +++ b/scripts/qwen2vl_qnn/run_qnn_interactive.sh @@ -14,8 +14,8 @@ REMOTE_IMAGE_DIR="${REMOTE_IMAGE_DIR:-${REMOTE_CPU_DIR}/images/eval}" LOCAL_RUNNER="${LOCAL_RUNNER:-${QWEN2VL_QNN_REPO_ROOT}/build-android-arm64-v8a-qnn/bin/mllm-qwen2vl-aot-runner}" PUSH_RUNNER="${PUSH_RUNNER:-1}" -CONTEXT="${CONTEXT:-models/qwen2vl-2b-sm8650-qnn234-fullqnn-5bucket-baseline-v1.bin}" -QNN_PARAMS="${QNN_PARAMS:-models/qwen2vl-2b-sm8650-qnn234-lpbq-baseline-v1.mllm}" +CONTEXT="${CONTEXT:-models/qwen2vl-2b-sm8650-qnn234-fullqnn-5bucket-visualfp16-v1.bin}" +QNN_PARAMS="${QNN_PARAMS:-models/qwen2vl-2b-sm8650-qnn234-lpbq-visualfp16-vprojg16.mllm}" TOKENIZER="${TOKENIZER:-${REMOTE_CPU_DIR}/tokenizer/tokenizer.json}" QNN_CONFIG="${QNN_CONFIG:-config/config_2B_qnn_lpbq.json}" VISUAL_MODEL="${VISUAL_MODEL:-${REMOTE_CPU_DIR}/models/qwen2vl-2b-w4a32-kai.mllm}" @@ -23,6 +23,7 @@ VISUAL_CONFIG="${VISUAL_CONFIG:-${REMOTE_CPU_DIR}/config/config_2B_w32a32.json}" VISUAL_MODEL_VERSION="${VISUAL_MODEL_VERSION:-v2}" VISUAL_QNN="${VISUAL_QNN:-1}" VISUAL_BUNDLE_LAYOUT="${VISUAL_BUNDLE_LAYOUT:-single}" +VISUAL_IO_DTYPE="${VISUAL_IO_DTYPE:-fp16}" VISUAL_BUCKET_GRIDS="${VISUAL_BUCKET_GRIDS:-12x16,16x24,24x24,24x32,18x52,52x18,26x36,36x26}" AR_LEN="${AR_LEN:-32}" @@ -31,8 +32,10 @@ GEN_LEN="${GEN_LEN:-1000}" PROMPT="${PROMPT:-describe this picture}" INPUT_EMBEDDING_SCALE="${INPUT_EMBEDDING_SCALE:-0.002563515}" INPUT_EMBEDDING_ZERO_POINT="${INPUT_EMBEDDING_ZERO_POINT:-15604}" +VISUAL_OUTPUT_SCALE="${VISUAL_OUTPUT_SCALE:--1}" +VISUAL_OUTPUT_ZERO_POINT="${VISUAL_OUTPUT_ZERO_POINT:--1}" KEY_CACHE_DTYPE="${KEY_CACHE_DTYPE:-uint8}" -DUMP_STATS="${DUMP_STATS:-1}" +DUMP_STATS="${DUMP_STATS:-0}" ADB_SHELL_TTY="${ADB_SHELL_TTY:-1}" if [[ "${PUSH_RUNNER}" != "0" ]]; then @@ -48,10 +51,13 @@ fi visual_args="" if [[ "${VISUAL_QNN}" == "1" ]]; then - visual_args="--visual_qnn --visual_bundle_layout $(remote_quote "${VISUAL_BUNDLE_LAYOUT}")" + visual_args="--visual_qnn --visual_bundle_layout $(remote_quote "${VISUAL_BUNDLE_LAYOUT}") --visual_io_dtype $(remote_quote "${VISUAL_IO_DTYPE}")" if [[ -n "${VISUAL_BUCKET_GRIDS}" ]]; then visual_args="${visual_args} --visual_bucket_grids $(remote_quote "${VISUAL_BUCKET_GRIDS}")" fi + if [[ "${VISUAL_BUNDLE_LAYOUT}" == "hybrid_single" ]]; then + visual_args="${visual_args} --visual_model $(remote_quote "${VISUAL_MODEL}") --visual_model_version $(remote_quote "${VISUAL_MODEL_VERSION}")" + fi else visual_args="--visual_model $(remote_quote "${VISUAL_MODEL}") --visual_model_version $(remote_quote "${VISUAL_MODEL_VERSION}")" fi @@ -79,6 +85,8 @@ exec ./bin/mllm-qwen2vl-aot-runner \ --gen_len ${GEN_LEN} \ --input_embedding_scale ${INPUT_EMBEDDING_SCALE} \ --input_embedding_zero_point ${INPUT_EMBEDDING_ZERO_POINT} \ + --visual_output_scale ${VISUAL_OUTPUT_SCALE} \ + --visual_output_zero_point ${VISUAL_OUTPUT_ZERO_POINT} \ --key_cache_dtype $(remote_quote "${KEY_CACHE_DTYPE}") " From d3da4c524fc9732ccadc1b433451d4acad39220f Mon Sep 17 00:00:00 2001 From: twlddd Date: Thu, 21 May 2026 12:24:54 +0800 Subject: [PATCH 5/7] Document Qwen2-VL QNN VLM run steps --- examples/qwen2vl_qnn_aot/README.md | 94 +++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/examples/qwen2vl_qnn_aot/README.md b/examples/qwen2vl_qnn_aot/README.md index f86178042..7afd506c6 100644 --- a/examples/qwen2vl_qnn_aot/README.md +++ b/examples/qwen2vl_qnn_aot/README.md @@ -13,7 +13,7 @@ The recommended Qwen2-VL-2B baseline tested on 2026-05-21 uses: - LPBQ / W4A16 LLM weights - FP16 visual encoder weights -- one combined QNN context with `model.0.s32`, `model.0.s1`, and five visual bucket graphs +- one combined QNN context with `model.0.s32`, `model.0.s1`, and five visual size buckets with aspect-ratio variants - `--visual_bundle_layout single` - `--visual_io_dtype fp16` @@ -31,6 +31,98 @@ Build the Android runner: cmake --build build-android-arm64-v8a-qnn --target mllm-qwen2vl-aot-runner -j2 ``` +## Run Qwen2-VL VLM on Android + +Large runtime artifacts are not included in this repository. Download the +prebuilt Qwen2-VL-2B QNN artifacts from ModelScope, or rebuild them with the +compile command below: + +```text +https://www.modelscope.cn/models/twlddd/Qwen2-VL-2B-Instruct-Full-QNN-AOT-for-mllm/summary +``` + +The expected local artifact layout is flat: + +```text +QWEN2VL_ARTIFACT_DIR/ +|-- qwen2vl-2b-sm8650-qnn234-lpbq-visualfp16-vprojg16.mllm +|-- qwen2vl-2b-sm8650-qnn234-fullqnn-5bucket-visualfp16-v1.bin +|-- config_2B_qnn_lpbq.json +`-- tokenizer.json +``` + +Prepare the device directories and push the runner plus model artifacts: + +```bash +REMOTE_QNN_DIR=/data/local/tmp/mllm-qwen2vl-qnn +REMOTE_CPU_DIR=/data/local/tmp/mllm-qwen2vl +ARTIFACT_DIR=/path/to/QWEN2VL_ARTIFACT_DIR + +adb shell "mkdir -p \ + ${REMOTE_QNN_DIR}/bin \ + ${REMOTE_QNN_DIR}/lib \ + ${REMOTE_QNN_DIR}/models \ + ${REMOTE_QNN_DIR}/config \ + ${REMOTE_CPU_DIR}/tokenizer \ + ${REMOTE_CPU_DIR}/images/eval" + +adb push build-android-arm64-v8a-qnn/bin/mllm-qwen2vl-aot-runner \ + ${REMOTE_QNN_DIR}/bin/ +adb shell "chmod 755 ${REMOTE_QNN_DIR}/bin/mllm-qwen2vl-aot-runner" + +adb push "${ARTIFACT_DIR}/qwen2vl-2b-sm8650-qnn234-fullqnn-5bucket-visualfp16-v1.bin" \ + ${REMOTE_QNN_DIR}/models/ +adb push "${ARTIFACT_DIR}/qwen2vl-2b-sm8650-qnn234-lpbq-visualfp16-vprojg16.mllm" \ + ${REMOTE_QNN_DIR}/models/ +adb push "${ARTIFACT_DIR}/config_2B_qnn_lpbq.json" \ + ${REMOTE_QNN_DIR}/config/ +adb push "${ARTIFACT_DIR}/tokenizer.json" \ + ${REMOTE_CPU_DIR}/tokenizer/ +``` + +The device also needs the mllm runtime libraries, the mllm QNN backend library, +the Qualcomm QNN runtime libraries, and the QNN op-package libraries under +`${REMOTE_QNN_DIR}/lib`. Those libraries are built or obtained from the local +mllm/QNN SDK environment and are not shipped in this example. + +Push one test image: + +```bash +adb push /path/to/test.jpg ${REMOTE_CPU_DIR}/images/eval/test.jpg +``` + +Run the interactive VLM session from the repository root: + +```bash +REMOTE_QNN_DIR=/data/local/tmp/mllm-qwen2vl-qnn \ +REMOTE_CPU_DIR=/data/local/tmp/mllm-qwen2vl \ +CONTEXT=models/qwen2vl-2b-sm8650-qnn234-fullqnn-5bucket-visualfp16-v1.bin \ +QNN_PARAMS=models/qwen2vl-2b-sm8650-qnn234-lpbq-visualfp16-vprojg16.mllm \ +QNN_CONFIG=config/config_2B_qnn_lpbq.json \ +TOKENIZER=/data/local/tmp/mllm-qwen2vl/tokenizer/tokenizer.json \ +VISUAL_IO_DTYPE=fp16 \ +scripts/qwen2vl_qnn/run_qnn_interactive.sh +``` + +Then enter the on-device image path and the text prompt: + +```text +Image path> /data/local/tmp/mllm-qwen2vl/images/eval/test.jpg +Prompt> describe this picture +``` + +For fixed-set evaluation, put images under `ASSET_SRC` and use a TSV case file +with `case_id|image_file|prompt` rows: + +```bash +ASSET_SRC=/path/to/eval/images \ +CASES_FILE=scripts/qwen2vl_qnn/qwen2vl_eval_cases_5bucket.tsv \ +REMOTE_QNN_DIR=/data/local/tmp/mllm-qwen2vl-qnn \ +REMOTE_CPU_DIR=/data/local/tmp/mllm-qwen2vl \ +VISUAL_IO_DTYPE=fp16 \ +scripts/qwen2vl_qnn/run_qnn_eval_fixed_set.sh +``` + ## Compile QNN Context Set `QNN_SDK_ROOT` or pass `--qnn_env_path` explicitly. The compiler uses the From 1959dd43fcc91ec506eb46a31c33ebbe6ce0bc78 Mon Sep 17 00:00:00 2001 From: twlddd <153590149+twlddd@users.noreply.github.com> Date: Thu, 21 May 2026 13:04:53 +0800 Subject: [PATCH 6/7] Update DeepSeek-OCR entry with additional link --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7356742cc..4b379dc1d 100644 --- a/README.md +++ b/README.md @@ -78,9 +78,9 @@ The mllm framework integrates seamlessly with popular community frameworks' chec | [Qwen3-0.6B](https://github.com/QwenLM/Qwen3) | [✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/Qwen3-0.6B-w4a32kai) | | ✔️ W8A8 | | [Qwen3-1.7B](https://github.com/QwenLM/Qwen3) | [✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/Qwen3-1.7B-w4a8-i8mm-kai) | [W4A16-SM8650](https://modelscope.cn/models/mllmTeam/Qwen3-1.7B-Qnn-AOT-SM8650/) | | | [Qwen3-4B](https://github.com/QwenLM/Qwen3) | [✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/Qwen3-4B-w4a8-i8mm-kai) | | | -| [DeepSeek-OCR](https://github.com/deepseek-ai/DeepSeek-OCR) | [✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/DeepSeek-OCR-w4a8-i8mm-kai) | | | +| [DeepSeek-OCR](https://github.com/deepseek-ai/DeepSeek-OCR) | [✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/DeepSeek-OCR-w4a8-i8mm-kai) | [W4A16-SM8650](https://www.modelscope.cn/models/twlddd/Qwen2-VL-2B-Instruct-Full-QNN-AOT-for-mllm/) | | | [SmolLM3](https://huggingface.co/blog/smollm3)| [✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/SmolLM3-3B-w4a8-i8mm-kai) | | | -| [Qwen2-VL-2B-Instruct](https://qwenlm.github.io/zh/blog/qwen2-vl/)|[✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/Qwen2-VL-2B-Instruct-w4a32kai) || | +| [Qwen2-VL-2B-Instruct](https://qwenlm.github.io/zh/blog/qwen2-vl/)|[✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/Qwen2-VL-2B-Instruct-w4a32kai) | | | | [Qwen2-VL-7B-Instruct](https://qwenlm.github.io/zh/blog/qwen2-vl/)|[✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/Qwen2-VL-7B-Instruct-w4a32kai)|| | | [Qwen2.5-VL-3B-Instruct](https://qwenlm.github.io/blog/qwen2.5-vl/)|[✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/Qwen2.5-VL-3B-Instruct-w4a32kai)|| | | [Qwen2.5-VL-7B-Instruct](https://qwenlm.github.io/blog/qwen2.5-vl/)|[✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/Qwen2.5-VL-7B-Instruct-w4a32kai)|| | From 4a420c597b30ca4a640d77d7a1297e660e888991 Mon Sep 17 00:00:00 2001 From: twlddd <153590149+twlddd@users.noreply.github.com> Date: Thu, 21 May 2026 13:05:33 +0800 Subject: [PATCH 7/7] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4b379dc1d..3031b452b 100644 --- a/README.md +++ b/README.md @@ -78,9 +78,9 @@ The mllm framework integrates seamlessly with popular community frameworks' chec | [Qwen3-0.6B](https://github.com/QwenLM/Qwen3) | [✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/Qwen3-0.6B-w4a32kai) | | ✔️ W8A8 | | [Qwen3-1.7B](https://github.com/QwenLM/Qwen3) | [✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/Qwen3-1.7B-w4a8-i8mm-kai) | [W4A16-SM8650](https://modelscope.cn/models/mllmTeam/Qwen3-1.7B-Qnn-AOT-SM8650/) | | | [Qwen3-4B](https://github.com/QwenLM/Qwen3) | [✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/Qwen3-4B-w4a8-i8mm-kai) | | | -| [DeepSeek-OCR](https://github.com/deepseek-ai/DeepSeek-OCR) | [✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/DeepSeek-OCR-w4a8-i8mm-kai) | [W4A16-SM8650](https://www.modelscope.cn/models/twlddd/Qwen2-VL-2B-Instruct-Full-QNN-AOT-for-mllm/) | | +| [DeepSeek-OCR](https://github.com/deepseek-ai/DeepSeek-OCR) | [✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/DeepSeek-OCR-w4a8-i8mm-kai) | | | | [SmolLM3](https://huggingface.co/blog/smollm3)| [✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/SmolLM3-3B-w4a8-i8mm-kai) | | | -| [Qwen2-VL-2B-Instruct](https://qwenlm.github.io/zh/blog/qwen2-vl/)|[✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/Qwen2-VL-2B-Instruct-w4a32kai) | | | +| [Qwen2-VL-2B-Instruct](https://qwenlm.github.io/zh/blog/qwen2-vl/)|[✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/Qwen2-VL-2B-Instruct-w4a32kai) | [W4A16-SM8650](https://www.modelscope.cn/models/twlddd/Qwen2-VL-2B-Instruct-Full-QNN-AOT-for-mllm/) | | | [Qwen2-VL-7B-Instruct](https://qwenlm.github.io/zh/blog/qwen2-vl/)|[✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/Qwen2-VL-7B-Instruct-w4a32kai)|| | | [Qwen2.5-VL-3B-Instruct](https://qwenlm.github.io/blog/qwen2.5-vl/)|[✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/Qwen2.5-VL-3B-Instruct-w4a32kai)|| | | [Qwen2.5-VL-7B-Instruct](https://qwenlm.github.io/blog/qwen2.5-vl/)|[✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/Qwen2.5-VL-7B-Instruct-w4a32kai)|| |