From 9838344b3c1b4b09a564802bc650af144810a779 Mon Sep 17 00:00:00 2001 From: karabambus Date: Sat, 29 Aug 2026 21:41:52 +0200 Subject: [PATCH] This commit fixes 2 bugs. 1. Kernels that using consumer/producer double buffering (matrix engines) - on lunch we call barrier to stop consumer reading counter to early before producer has reset them. 2. During dequantization weights get flushed from one layer of cache to other, and flush_to_l2 cant flush over 16 lines of weights, this rule was broken multiple times. - flush_to_l2_multi flushes n lines instead of 16. Both fixes are verbatim copy (including comments from q8 kernels) --- .../et-kernels/src/mul_mat_Q2_K_matrix_engine.c | 15 +++++++++++++-- .../et-kernels/src/mul_mat_Q3_K_matrix_engine.c | 15 +++++++++++++-- .../et-kernels/src/mul_mat_Q4_0_matrix_engine.c | 15 +++++++++++++-- .../et-kernels/src/mul_mat_Q4_K_matrix_engine.c | 15 +++++++++++++-- .../et-kernels/src/mul_mat_Q5_K_matrix_engine.c | 15 +++++++++++++-- .../et-kernels/src/mul_mat_Q6_K_matrix_engine.c | 15 +++++++++++++-- ggml/src/ggml-et/et-kernels/src/platform.h | 14 ++++++++++++++ 7 files changed, 92 insertions(+), 12 deletions(-) diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K_matrix_engine.c index b2cc6438f677..5951b4e6864c 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K_matrix_engine.c @@ -279,6 +279,11 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // L2-SCP persists across kernel dispatches (only zeroed at boot) and the + // two harts have no implicit ordering at entry. Barrier so the consumer + // sees this reset before its first scp_wait, otherwise on a cold first + // dispatch it reads a stale counter and races ahead of the producer. + et_barrier(ET_BARRIER_MINION); uint32_t wid = 0; for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { @@ -307,7 +312,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { src0_batch, mb, kb0 + i, nb1_0); } FENCE; - flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + flush_to_l2_multi(cache_buf[buf], kbn * BLOCK_K, 64); WAIT_CACHEOPS; wid++; @@ -324,6 +329,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; @@ -424,6 +431,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // See REUSE path: barrier so the consumer observes the reset before it reads. + et_barrier(ET_BARRIER_MINION); uint32_t chunk_id = 0; for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { @@ -447,7 +456,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { dequant_q2_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); FENCE; - flush_to_l2(scp_panel[buf], BLOCK_K, 64); + flush_to_l2_multi(scp_panel[buf], BLOCK_K, 64); WAIT_CACHEOPS; chunk_id++; @@ -463,6 +472,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K_matrix_engine.c index 19bf1b03bb6f..63c863d595e4 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K_matrix_engine.c @@ -281,6 +281,11 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // L2-SCP persists across kernel dispatches (only zeroed at boot) and the + // two harts have no implicit ordering at entry. Barrier so the consumer + // sees this reset before its first scp_wait, otherwise on a cold first + // dispatch it reads a stale counter and races ahead of the producer. + et_barrier(ET_BARRIER_MINION); uint32_t wid = 0; for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { @@ -309,7 +314,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { src0_batch, mb, kb0 + i, nb1_0); } FENCE; - flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + flush_to_l2_multi(cache_buf[buf], kbn * BLOCK_K, 64); WAIT_CACHEOPS; wid++; @@ -326,6 +331,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; @@ -426,6 +433,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // See REUSE path: barrier so the consumer observes the reset before it reads. + et_barrier(ET_BARRIER_MINION); uint32_t chunk_id = 0; for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { @@ -449,7 +458,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { dequant_q3_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); FENCE; - flush_to_l2(scp_panel[buf], BLOCK_K, 64); + flush_to_l2_multi(scp_panel[buf], BLOCK_K, 64); WAIT_CACHEOPS; chunk_id++; @@ -465,6 +474,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c index 7a75ea95e9c8..13de5a4dae37 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c @@ -281,6 +281,11 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // L2-SCP persists across kernel dispatches (only zeroed at boot) and the + // two harts have no implicit ordering at entry. Barrier so the consumer + // sees this reset before its first scp_wait, otherwise on a cold first + // dispatch it reads a stale counter and races ahead of the producer. + et_barrier(ET_BARRIER_MINION); uint32_t wid = 0; for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { @@ -309,7 +314,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { src0_batch, mb, kb0 + i, nb1_0); } FENCE; - flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + flush_to_l2_multi(cache_buf[buf], kbn * BLOCK_K, 64); WAIT_CACHEOPS; wid++; @@ -326,6 +331,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; @@ -426,6 +433,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // See REUSE path: barrier so the consumer observes the reset before it reads. + et_barrier(ET_BARRIER_MINION); uint32_t chunk_id = 0; for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { @@ -449,7 +458,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { dequant_q4_0_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); FENCE; - flush_to_l2(scp_panel[buf], BLOCK_K, 64); + flush_to_l2_multi(scp_panel[buf], BLOCK_K, 64); WAIT_CACHEOPS; chunk_id++; @@ -465,6 +474,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K_matrix_engine.c index f8f5eb4d3350..d5c894929014 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K_matrix_engine.c @@ -339,6 +339,11 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // L2-SCP persists across kernel dispatches (only zeroed at boot) and the + // two harts have no implicit ordering at entry. Barrier so the consumer + // sees this reset before its first scp_wait, otherwise on a cold first + // dispatch it reads a stale counter and races ahead of the producer. + et_barrier(ET_BARRIER_MINION); uint32_t wid = 0; for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { @@ -367,7 +372,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { src0_batch, mb, kb0 + i, nb1_0); } FENCE; - flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + flush_to_l2_multi(cache_buf[buf], kbn * BLOCK_K, 64); WAIT_CACHEOPS; wid++; @@ -384,6 +389,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; @@ -484,6 +491,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // See REUSE path: barrier so the consumer observes the reset before it reads. + et_barrier(ET_BARRIER_MINION); uint32_t chunk_id = 0; for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { @@ -507,7 +516,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { dequant_q4_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); FENCE; - flush_to_l2(scp_panel[buf], BLOCK_K, 64); + flush_to_l2_multi(scp_panel[buf], BLOCK_K, 64); WAIT_CACHEOPS; chunk_id++; @@ -523,6 +532,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K_matrix_engine.c index e78e4102bef2..2b1c25816c97 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K_matrix_engine.c @@ -280,6 +280,11 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // L2-SCP persists across kernel dispatches (only zeroed at boot) and the + // two harts have no implicit ordering at entry. Barrier so the consumer + // sees this reset before its first scp_wait, otherwise on a cold first + // dispatch it reads a stale counter and races ahead of the producer. + et_barrier(ET_BARRIER_MINION); uint32_t wid = 0; for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { @@ -308,7 +313,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { src0_batch, mb, kb0 + i, nb1_0); } FENCE; - flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + flush_to_l2_multi(cache_buf[buf], kbn * BLOCK_K, 64); WAIT_CACHEOPS; wid++; @@ -325,6 +330,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; @@ -425,6 +432,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // See REUSE path: barrier so the consumer observes the reset before it reads. + et_barrier(ET_BARRIER_MINION); uint32_t chunk_id = 0; for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { @@ -448,7 +457,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { dequant_q5_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); FENCE; - flush_to_l2(scp_panel[buf], BLOCK_K, 64); + flush_to_l2_multi(scp_panel[buf], BLOCK_K, 64); WAIT_CACHEOPS; chunk_id++; @@ -464,6 +473,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K_matrix_engine.c index 1345df09453c..761aa5cf658f 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K_matrix_engine.c @@ -283,6 +283,11 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // L2-SCP persists across kernel dispatches (only zeroed at boot) and the + // two harts have no implicit ordering at entry. Barrier so the consumer + // sees this reset before its first scp_wait, otherwise on a cold first + // dispatch it reads a stale counter and races ahead of the producer. + et_barrier(ET_BARRIER_MINION); uint32_t wid = 0; for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { @@ -311,7 +316,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { src0_batch, mb, kb0 + i, nb1_0); } FENCE; - flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + flush_to_l2_multi(cache_buf[buf], kbn * BLOCK_K, 64); WAIT_CACHEOPS; wid++; @@ -328,6 +333,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; @@ -428,6 +435,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // See REUSE path: barrier so the consumer observes the reset before it reads. + et_barrier(ET_BARRIER_MINION); uint32_t chunk_id = 0; for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { @@ -451,7 +460,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { dequant_q6_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); FENCE; - flush_to_l2(scp_panel[buf], BLOCK_K, 64); + flush_to_l2_multi(scp_panel[buf], BLOCK_K, 64); WAIT_CACHEOPS; chunk_id++; @@ -467,6 +476,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; diff --git a/ggml/src/ggml-et/et-kernels/src/platform.h b/ggml/src/ggml-et/et-kernels/src/platform.h index 3ad9cae52487..c49aff91a4b9 100644 --- a/ggml/src/ggml-et/et-kernels/src/platform.h +++ b/ggml/src/ggml-et/et-kernels/src/platform.h @@ -498,6 +498,20 @@ static inline void __attribute__((always_inline)) flush_to_l2(const void * addr, : "x31", "memory"); } +// Flush an arbitrary number of lines to L2, working around the 16-line cap of a +// single FlushVA by issuing multiple flushes. Use this whenever nlines may exceed 16. +static inline void __attribute__((always_inline)) flush_to_l2_multi(const void * addr, uint64_t nlines, uint64_t stride) { + const char * p = (const char *) addr; + while (nlines > 16) { + flush_to_l2(p, 16, stride); + p += 16 * stride; + nlines -= 16; + } + if (nlines) { + flush_to_l2(p, nlines, stride); + } +} + // Evict nlines cache lines at stride apart starting at addr from L1 to L2. // Uses EvictVA (CSR 0x89F). Unlike flush_to_l2, this guarantees the line is // NOT present in L1 after the operation - subsequent loads will miss and go