Skip to content

qwen35 moe #258

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wine99 wants to merge 1 commit into
base: dev_backend_openvino
Choose a base branch
from
Open

qwen35 moe #258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 27 additions & 2 deletions ggml/src/ggml-openvino/ggml-decoder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -565,7 +565,7 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
if (node->op == GGML_OP_GATED_DELTA_NET) {
model_params.state_size = node->src[0]->ne[0];
}
if (node->op == GGML_OP_SCALE && is_kvcache(node->view_src, nullptr)) {
if (node->op == GGML_OP_SCALE && node->view_src != nullptr && is_kvcache(node->view_src, nullptr)) {
compute_params.cache_rs_reset_len = ggml_nelements(node) / node->view_src->ne[0];
compute_params.cache_rs_reset_idx = node->src[0]->view_offs / node->view_src->ne[0];
}
Expand Down Expand Up @@ -888,6 +888,16 @@ std::map<std::string, std::string> GgmlOvDecoder::get_kv_param_res_names() const
return kv_param_res_names;
}

// MUL_MAT_ID's src[0] is the [k, m, n_expert] expert-weight tensor. It is always a constant per-expert
// weight table -- never a computed activation -- regardless of whether the backend happened to mark its
// buffer as GGML_BACKEND_BUFFER_USAGE_WEIGHTS (test-backend-ops, for example, never sets that usage
// flag, unlike real inference). Without this, non-quantized (F16/F32/BF16) expert weights would fall
// through the check below as "not a weight", get decoded as a Parameter/activation instead of a
// Constant, and crash GatherMatmul's "only constant weights are supported" check.
static bool is_mul_mat_id_expert_weight(const ggml_tensor * node, int src_index) {
return node->op == GGML_OP_MUL_MAT_ID && src_index == 0;
}

std::map<std::string, std::shared_ptr<ov::Node>> GgmlOvDecoder::create_weight_nodes(ggml_cgraph * cgraph, bool naive) {
std::map<std::string, std::shared_ptr<ov::Node>> model_weights;
auto * nodes = cgraph->nodes;
Expand All @@ -906,7 +916,8 @@ std::map<std::string, std::shared_ptr<ov::Node>> GgmlOvDecoder::create_weight_no
}
if (!src->view_src) {
ggml_backend_buffer * buffer = src->buffer;
if (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS || ggml_is_quantized(src->type)) {
if (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS || ggml_is_quantized(src->type) ||
is_mul_mat_id_expert_weight(node, i)) {
if (model_weights.find(src_name) == model_weights.end()) {
auto weight_node = create_weight_node(src, naive);
weight_node->set_friendly_name(src_name);
Expand Down Expand Up @@ -1673,8 +1684,22 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
case GGML_OP_DIAG:
case GGML_OP_TRI:
case GGML_OP_REPEAT:
// Shape-preserving elementwise ops: the dynamic dim is unchanged from src[0].
// DIV/CLAMP are used in the MoE routing-weight normalization
// (sum_rows -> clamp -> div). If they are left untracked here the dynamic
// (token) dim is lost there, the captured prefill token count gets baked into
// the downstream reshapes, and every decoder layer after layer 0 turns static
// (which then triggers the GPU in-place-concat KV-cache corruption).
case GGML_OP_DIV:
case GGML_OP_CLAMP:
m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]];
break;
case GGML_OP_SUM_ROWS:
// SUM_ROWS reduces ggml axis 0 to size 1 and preserves all other axes, so the
// dynamic dim is preserved unless it was axis 0 (then it is summed away).
m_node_dynamic_dims[node] =
(m_node_dynamic_dims[node->src[0]] == 0) ? -1 : m_node_dynamic_dims[node->src[0]];
break;
case GGML_OP_MUL_MAT_ID:
case GGML_OP_SOLVE_TRI:
m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[1]];
Expand Down
20 changes: 16 additions & 4 deletions ggml/src/ggml-openvino/ggml-openvino-extra.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,12 +252,19 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
return layout;
}

// Most quantized weights use the existing 2D extraction path. MXFP4 also
// appears as 3D expert weights for MUL_MAT_ID, so allow that type through.
if (tensor->type != GGML_TYPE_MXFP4 && (tensor->ne[2] != 1 || tensor->ne[3] != 1)) {
// Most quantized weights use the existing 2D extraction path. 3D expert weights for
// MUL_MAT_ID (MoE) are also supported, either as MXFP4 (packed, dedicated branch below) or via the
// generic sizing math below, which is shape-agnostic (based on total element count). Only reject 4D.
if (tensor->ne[3] != 1) {
return layout;
}

// 3D MoE expert weights that are not requantized (see below) always use the exact f16
// zero-point extraction (see extract_quantized_weights), which needs a wider zp slot than
// the packed integer zero point -- must be kept in sync with that function so the buffer
// sizing here matches what process_weight_tensor actually writes.
const bool for_gather_matmul = tensor->ne[2] > 1;

int64_t n_elements = ggml_nelements(tensor);
const size_t alignment = 64; // Good for SIMD

Expand Down Expand Up @@ -387,9 +394,14 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
// Scales: F16 per block, except MXFP4 which stores one E8M0 byte per block.
int64_t n_blocks = n_elements / layout.weights_per_block;
layout.scales_size = n_blocks * (tensor->type == GGML_TYPE_MXFP4 ? sizeof(uint8_t) : sizeof(uint16_t));
// For symmetric quantization, no zp needed (weights stored as signed)
// For symmetric quantization, no zp needed (weights stored as signed). Asymmetric
// for_gather_matmul (3D MoE expert) weights use an exact f16 zero point (see
// extract_quantized_weights/make_int8_weights/make_int4_weights), which needs one f16 per
// block instead of a packed u4/u8 integer zero point.
if (layout.is_symmetric) {
layout.zp_size = 0;
} else if (use_bias || for_gather_matmul) {
layout.zp_size = n_blocks * sizeof(uint16_t);
} else {
layout.zp_size = layout.is_u4 ? ((n_blocks + 1) / 2) : n_blocks;
}
Expand Down
40 changes: 23 additions & 17 deletions ggml/src/ggml-openvino/ggml-openvino.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,9 +235,10 @@ static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer
bool is_weight_buffer = (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
// Full tensor set: offset=0, full size, not a view
bool is_full_tensor_set = (offset == 0 && size == ggml_nbytes(tensor) && tensor->view_src == nullptr);
// 2D tensor (typical weight shape)
// 2D tensor (typical weight shape), or a 3D quantized MoE expert weight (MUL_MAT_ID). Dense 3D
// expert weights are handled later in create_weight_node instead.
bool is_2d = (tensor->ne[2] == 1 && tensor->ne[3] == 1);
bool is_supported_weight_shape = is_2d || tensor->type == GGML_TYPE_MXFP4;
bool is_supported_weight_shape = is_2d || (tensor->ne[3] == 1 && ggml_is_quantized(tensor->type));

if (is_weight_buffer && is_full_tensor_set && is_supported_weight_shape) {
try {
Expand Down Expand Up @@ -460,8 +461,7 @@ static size_t ggml_backend_openvino_buffer_type_get_alloc_size(ggml_backend_buff
GGML_UNUSED(buft);

// For quantized weight tensors, we need extra space for extracted data.
if (ggml_is_quantized(tensor->type) &&
((tensor->ne[2] == 1 && tensor->ne[3] == 1) || tensor->type == GGML_TYPE_MXFP4)) {
if (ggml_is_quantized(tensor->type) && tensor->ne[3] == 1) {
ggml_openvino_extracted_layout layout = ggml_openvino_get_extracted_layout(tensor);
if (layout.total_size > 0) {
// GGML_LOG_DEBUG("%s: tensor %s needs %zu bytes (original %zu, extracted: weights=%zu scales=%zu zp=%zu)\n",
Expand Down Expand Up @@ -890,9 +890,10 @@ static bool mul_mat_id_requires_large_tmp(const ggml_tensor * op) {
return true;
}

// The current OpenVINO translation materializes selected expert weights with
// shape [n_tokens, n_used, rows, k]. Skip cases that would create a very
// large temporary on GPU and let the scheduler fall back instead.
// The MXFP4 MUL_MAT_ID translation (translate_mul_mat_id_mxfp4_packed in mul_mat_id.cpp)
// materializes selected expert weights with shape [n_tokens, n_used, rows, k]. Skip cases that
// would create a very large temporary and let the scheduler fall back instead. Every other weight
// type goes through GatherMatmul, which never materializes this temporary.
size_t tmp_elems = 1;
if (!checked_mul_size(tmp_elems, static_cast<size_t>(ids->ne[1]), tmp_elems) ||
!checked_mul_size(tmp_elems, static_cast<size_t>(ids->ne[0]), tmp_elems) ||
Expand Down Expand Up @@ -1068,11 +1069,16 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
break;
}
case GGML_OP_MUL_MAT_ID: {
// Single-expert (or empty) MUL_MAT_ID is a degenerate shape that stresses GatherMatmul edge
// cases and never occurs in real MoE; let it fall back to CPU.
if (op->src[0] != nullptr && op->src[0]->ne[2] <= 1) {
return true;
}
if (ggml_openvino_get_device_name() == "GPU" && op->src[0] != nullptr && op->src[0]->type == GGML_TYPE_BF16) {
return true;
}
if (mul_mat_id_requires_large_tmp(op) &&
!(op->src[0] != nullptr && op->src[0]->type == GGML_TYPE_MXFP4)) {
// Only MXFP4 materializes a large per-token temporary; all other types use GatherMatmul.
if (op->src[0] != nullptr && op->src[0]->type == GGML_TYPE_MXFP4 && mul_mat_id_requires_large_tmp(op)) {
return true;
}
break;
Expand Down Expand Up @@ -1231,11 +1237,11 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
// GGML_LOG_WARN("OpenVINO backend does not support GLU op %s\n", ggml_glu_op_name(ggml_get_glu_op(op)));
return false;
}
if (has_view_op_input(op)) {
// GGML_LOG_WARN("OpenVINO backend does not support unary op %s with view input\n",
// ggml_glu_op_name(ggml_get_glu_op(op)));
return false;
}
// if (has_view_op_input(op)) {
// // GGML_LOG_WARN("OpenVINO backend does not support unary op %s with view input\n",
// // ggml_glu_op_name(ggml_get_glu_op(op)));
// return false;
// }
if (op->src[1] == nullptr && op->src[0]->ne[0] % 2 != 0) {
// triggers bug in ov gpu
return false;
Expand Down Expand Up @@ -1269,9 +1275,9 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
// GGML_LOG_WARN("OpenVINO backend does not support tensor type %s\n", ggml_type_name(src->type));
return false;
}
const bool is_supported_3d_mxfp4_moe = op->op == GGML_OP_MUL_MAT_ID && i == 0 &&
src->type == GGML_TYPE_MXFP4;
if (ggml_is_quantized(src->type) && src->ne[2] != 1 && !is_supported_3d_mxfp4_moe) {
const bool is_supported_3d_moe_expert =
op->op == GGML_OP_MUL_MAT_ID && i == 0 && (src->type == GGML_TYPE_MXFP4 || src->ne[3] == 1);
if (ggml_is_quantized(src->type) && src->ne[2] != 1 && !is_supported_3d_moe_expert) {
// GGML_LOG_WARN("OpenVINO backend does not support 3D quantized tensors\n");
return false;
}
Expand Down
Loading
Loading