diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K_matrix_engine.c index b2cc6438f677..5951b4e6864c 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K_matrix_engine.c @@ -279,6 +279,11 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // L2-SCP persists across kernel dispatches (only zeroed at boot) and the + // two harts have no implicit ordering at entry. Barrier so the consumer + // sees this reset before its first scp_wait, otherwise on a cold first + // dispatch it reads a stale counter and races ahead of the producer. + et_barrier(ET_BARRIER_MINION); uint32_t wid = 0; for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { @@ -307,7 +312,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { src0_batch, mb, kb0 + i, nb1_0); } FENCE; - flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + flush_to_l2_multi(cache_buf[buf], kbn * BLOCK_K, 64); WAIT_CACHEOPS; wid++; @@ -324,6 +329,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; @@ -424,6 +431,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // See REUSE path: barrier so the consumer observes the reset before it reads. + et_barrier(ET_BARRIER_MINION); uint32_t chunk_id = 0; for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { @@ -447,7 +456,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { dequant_q2_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); FENCE; - flush_to_l2(scp_panel[buf], BLOCK_K, 64); + flush_to_l2_multi(scp_panel[buf], BLOCK_K, 64); WAIT_CACHEOPS; chunk_id++; @@ -463,6 +472,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K_matrix_engine.c index 19bf1b03bb6f..63c863d595e4 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K_matrix_engine.c @@ -281,6 +281,11 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // L2-SCP persists across kernel dispatches (only zeroed at boot) and the + // two harts have no implicit ordering at entry. Barrier so the consumer + // sees this reset before its first scp_wait, otherwise on a cold first + // dispatch it reads a stale counter and races ahead of the producer. + et_barrier(ET_BARRIER_MINION); uint32_t wid = 0; for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { @@ -309,7 +314,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { src0_batch, mb, kb0 + i, nb1_0); } FENCE; - flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + flush_to_l2_multi(cache_buf[buf], kbn * BLOCK_K, 64); WAIT_CACHEOPS; wid++; @@ -326,6 +331,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; @@ -426,6 +433,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // See REUSE path: barrier so the consumer observes the reset before it reads. + et_barrier(ET_BARRIER_MINION); uint32_t chunk_id = 0; for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { @@ -449,7 +458,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { dequant_q3_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); FENCE; - flush_to_l2(scp_panel[buf], BLOCK_K, 64); + flush_to_l2_multi(scp_panel[buf], BLOCK_K, 64); WAIT_CACHEOPS; chunk_id++; @@ -465,6 +474,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c index 7a75ea95e9c8..13de5a4dae37 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c @@ -281,6 +281,11 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // L2-SCP persists across kernel dispatches (only zeroed at boot) and the + // two harts have no implicit ordering at entry. Barrier so the consumer + // sees this reset before its first scp_wait, otherwise on a cold first + // dispatch it reads a stale counter and races ahead of the producer. + et_barrier(ET_BARRIER_MINION); uint32_t wid = 0; for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { @@ -309,7 +314,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { src0_batch, mb, kb0 + i, nb1_0); } FENCE; - flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + flush_to_l2_multi(cache_buf[buf], kbn * BLOCK_K, 64); WAIT_CACHEOPS; wid++; @@ -326,6 +331,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; @@ -426,6 +433,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // See REUSE path: barrier so the consumer observes the reset before it reads. + et_barrier(ET_BARRIER_MINION); uint32_t chunk_id = 0; for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { @@ -449,7 +458,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { dequant_q4_0_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); FENCE; - flush_to_l2(scp_panel[buf], BLOCK_K, 64); + flush_to_l2_multi(scp_panel[buf], BLOCK_K, 64); WAIT_CACHEOPS; chunk_id++; @@ -465,6 +474,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K_matrix_engine.c index f8f5eb4d3350..d5c894929014 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K_matrix_engine.c @@ -339,6 +339,11 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // L2-SCP persists across kernel dispatches (only zeroed at boot) and the + // two harts have no implicit ordering at entry. Barrier so the consumer + // sees this reset before its first scp_wait, otherwise on a cold first + // dispatch it reads a stale counter and races ahead of the producer. + et_barrier(ET_BARRIER_MINION); uint32_t wid = 0; for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { @@ -367,7 +372,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { src0_batch, mb, kb0 + i, nb1_0); } FENCE; - flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + flush_to_l2_multi(cache_buf[buf], kbn * BLOCK_K, 64); WAIT_CACHEOPS; wid++; @@ -384,6 +389,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; @@ -484,6 +491,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // See REUSE path: barrier so the consumer observes the reset before it reads. + et_barrier(ET_BARRIER_MINION); uint32_t chunk_id = 0; for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { @@ -507,7 +516,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { dequant_q4_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); FENCE; - flush_to_l2(scp_panel[buf], BLOCK_K, 64); + flush_to_l2_multi(scp_panel[buf], BLOCK_K, 64); WAIT_CACHEOPS; chunk_id++; @@ -523,6 +532,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K_matrix_engine.c index e78e4102bef2..2b1c25816c97 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K_matrix_engine.c @@ -280,6 +280,11 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // L2-SCP persists across kernel dispatches (only zeroed at boot) and the + // two harts have no implicit ordering at entry. Barrier so the consumer + // sees this reset before its first scp_wait, otherwise on a cold first + // dispatch it reads a stale counter and races ahead of the producer. + et_barrier(ET_BARRIER_MINION); uint32_t wid = 0; for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { @@ -308,7 +313,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { src0_batch, mb, kb0 + i, nb1_0); } FENCE; - flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + flush_to_l2_multi(cache_buf[buf], kbn * BLOCK_K, 64); WAIT_CACHEOPS; wid++; @@ -325,6 +330,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; @@ -425,6 +432,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // See REUSE path: barrier so the consumer observes the reset before it reads. + et_barrier(ET_BARRIER_MINION); uint32_t chunk_id = 0; for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { @@ -448,7 +457,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { dequant_q5_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); FENCE; - flush_to_l2(scp_panel[buf], BLOCK_K, 64); + flush_to_l2_multi(scp_panel[buf], BLOCK_K, 64); WAIT_CACHEOPS; chunk_id++; @@ -464,6 +473,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; diff --git a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K_matrix_engine.c b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K_matrix_engine.c index 1345df09453c..761aa5cf658f 100644 --- a/ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K_matrix_engine.c +++ b/ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K_matrix_engine.c @@ -283,6 +283,11 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // L2-SCP persists across kernel dispatches (only zeroed at boot) and the + // two harts have no implicit ordering at entry. Barrier so the consumer + // sees this reset before its first scp_wait, otherwise on a cold first + // dispatch it reads a stale counter and races ahead of the producer. + et_barrier(ET_BARRIER_MINION); uint32_t wid = 0; for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) { @@ -311,7 +316,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { src0_batch, mb, kb0 + i, nb1_0); } FENCE; - flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64); + flush_to_l2_multi(cache_buf[buf], kbn * BLOCK_K, 64); WAIT_CACHEOPS; wid++; @@ -328,6 +333,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; @@ -428,6 +435,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { if (is_hart1) { scp_signal(ready_ctr, 0); scp_signal(consumed_ctr, 0); + // See REUSE path: barrier so the consumer observes the reset before it reads. + et_barrier(ET_BARRIER_MINION); uint32_t chunk_id = 0; for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) { @@ -451,7 +460,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { dequant_q6_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0); FENCE; - flush_to_l2(scp_panel[buf], BLOCK_K, 64); + flush_to_l2_multi(scp_panel[buf], BLOCK_K, 64); WAIT_CACHEOPS; chunk_id++; @@ -467,6 +476,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) { ucache_control(1, REP_RATE, CACHEOP_MAX); #endif CLEAR_TENSOR_ERROR; + // Rendezvous with the producer so its counter reset is visible before we read. + et_barrier(ET_BARRIER_MINION); evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS; evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS; diff --git a/ggml/src/ggml-et/et-kernels/src/platform.h b/ggml/src/ggml-et/et-kernels/src/platform.h index 3ad9cae52487..c49aff91a4b9 100644 --- a/ggml/src/ggml-et/et-kernels/src/platform.h +++ b/ggml/src/ggml-et/et-kernels/src/platform.h @@ -498,6 +498,20 @@ static inline void __attribute__((always_inline)) flush_to_l2(const void * addr, : "x31", "memory"); } +// Flush an arbitrary number of lines to L2, working around the 16-line cap of a +// single FlushVA by issuing multiple flushes. Use this whenever nlines may exceed 16. +static inline void __attribute__((always_inline)) flush_to_l2_multi(const void * addr, uint64_t nlines, uint64_t stride) { + const char * p = (const char *) addr; + while (nlines > 16) { + flush_to_l2(p, 16, stride); + p += 16 * stride; + nlines -= 16; + } + if (nlines) { + flush_to_l2(p, nlines, stride); + } +} + // Evict nlines cache lines at stride apart starting at addr from L1 to L2. // Uses EvictVA (CSR 0x89F). Unlike flush_to_l2, this guarantees the line is // NOT present in L1 after the operation - subsequent loads will miss and go