Skip to content

Fix flush/barrier #30

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
karabambus wants to merge 1 commit into
base: et
Choose a base branch
from
Open

Fix flush/barrier #30

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions ggml/src/ggml-et/et-kernels/src/mul_mat_Q2_K_matrix_engine.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,11 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
if (is_hart1) {
scp_signal(ready_ctr, 0);
scp_signal(consumed_ctr, 0);
// L2-SCP persists across kernel dispatches (only zeroed at boot) and the
// two harts have no implicit ordering at entry. Barrier so the consumer
// sees this reset before its first scp_wait, otherwise on a cold first
// dispatch it reads a stale counter and races ahead of the producer.
et_barrier(ET_BARRIER_MINION);
uint32_t wid = 0;

for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) {
Expand Down Expand Up @@ -307,7 +312,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
src0_batch, mb, kb0 + i, nb1_0);
}
FENCE;
flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64);
flush_to_l2_multi(cache_buf[buf], kbn * BLOCK_K, 64);
WAIT_CACHEOPS;

wid++;
Expand All @@ -324,6 +329,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
ucache_control(1, REP_RATE, CACHEOP_MAX);
#endif
CLEAR_TENSOR_ERROR;
// Rendezvous with the producer so its counter reset is visible before we read.
et_barrier(ET_BARRIER_MINION);
evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS;
evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS;

Expand Down Expand Up @@ -424,6 +431,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
if (is_hart1) {
scp_signal(ready_ctr, 0);
scp_signal(consumed_ctr, 0);
// See REUSE path: barrier so the consumer observes the reset before it reads.
et_barrier(ET_BARRIER_MINION);
uint32_t chunk_id = 0;

for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) {
Expand All @@ -447,7 +456,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
dequant_q2_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0);

FENCE;
flush_to_l2(scp_panel[buf], BLOCK_K, 64);
flush_to_l2_multi(scp_panel[buf], BLOCK_K, 64);
WAIT_CACHEOPS;

chunk_id++;
Expand All @@ -463,6 +472,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
ucache_control(1, REP_RATE, CACHEOP_MAX);
#endif
CLEAR_TENSOR_ERROR;
// Rendezvous with the producer so its counter reset is visible before we read.
et_barrier(ET_BARRIER_MINION);
evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS;
evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS;

Expand Down
15 changes: 13 additions & 2 deletions ggml/src/ggml-et/et-kernels/src/mul_mat_Q3_K_matrix_engine.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,11 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
if (is_hart1) {
scp_signal(ready_ctr, 0);
scp_signal(consumed_ctr, 0);
// L2-SCP persists across kernel dispatches (only zeroed at boot) and the
// two harts have no implicit ordering at entry. Barrier so the consumer
// sees this reset before its first scp_wait, otherwise on a cold first
// dispatch it reads a stale counter and races ahead of the producer.
et_barrier(ET_BARRIER_MINION);
uint32_t wid = 0;

for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) {
Expand Down Expand Up @@ -309,7 +314,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
src0_batch, mb, kb0 + i, nb1_0);
}
FENCE;
flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64);
flush_to_l2_multi(cache_buf[buf], kbn * BLOCK_K, 64);
WAIT_CACHEOPS;

wid++;
Expand All @@ -326,6 +331,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
ucache_control(1, REP_RATE, CACHEOP_MAX);
#endif
CLEAR_TENSOR_ERROR;
// Rendezvous with the producer so its counter reset is visible before we read.
et_barrier(ET_BARRIER_MINION);
evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS;
evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS;

Expand Down Expand Up @@ -426,6 +433,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
if (is_hart1) {
scp_signal(ready_ctr, 0);
scp_signal(consumed_ctr, 0);
// See REUSE path: barrier so the consumer observes the reset before it reads.
et_barrier(ET_BARRIER_MINION);
uint32_t chunk_id = 0;

for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) {
Expand All @@ -449,7 +458,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
dequant_q3_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0);

FENCE;
flush_to_l2(scp_panel[buf], BLOCK_K, 64);
flush_to_l2_multi(scp_panel[buf], BLOCK_K, 64);
WAIT_CACHEOPS;

chunk_id++;
Expand All @@ -465,6 +474,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
ucache_control(1, REP_RATE, CACHEOP_MAX);
#endif
CLEAR_TENSOR_ERROR;
// Rendezvous with the producer so its counter reset is visible before we read.
et_barrier(ET_BARRIER_MINION);
evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS;
evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS;

Expand Down
15 changes: 13 additions & 2 deletions ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,11 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
if (is_hart1) {
scp_signal(ready_ctr, 0);
scp_signal(consumed_ctr, 0);
// L2-SCP persists across kernel dispatches (only zeroed at boot) and the
// two harts have no implicit ordering at entry. Barrier so the consumer
// sees this reset before its first scp_wait, otherwise on a cold first
// dispatch it reads a stale counter and races ahead of the producer.
et_barrier(ET_BARRIER_MINION);
uint32_t wid = 0;

for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) {
Expand Down Expand Up @@ -309,7 +314,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
src0_batch, mb, kb0 + i, nb1_0);
}
FENCE;
flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64);
flush_to_l2_multi(cache_buf[buf], kbn * BLOCK_K, 64);
WAIT_CACHEOPS;

wid++;
Expand All @@ -326,6 +331,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
ucache_control(1, REP_RATE, CACHEOP_MAX);
#endif
CLEAR_TENSOR_ERROR;
// Rendezvous with the producer so its counter reset is visible before we read.
et_barrier(ET_BARRIER_MINION);
evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS;
evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS;

Expand Down Expand Up @@ -426,6 +433,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
if (is_hart1) {
scp_signal(ready_ctr, 0);
scp_signal(consumed_ctr, 0);
// See REUSE path: barrier so the consumer observes the reset before it reads.
et_barrier(ET_BARRIER_MINION);
uint32_t chunk_id = 0;

for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) {
Expand All @@ -449,7 +458,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
dequant_q4_0_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0);

FENCE;
flush_to_l2(scp_panel[buf], BLOCK_K, 64);
flush_to_l2_multi(scp_panel[buf], BLOCK_K, 64);
WAIT_CACHEOPS;

chunk_id++;
Expand All @@ -465,6 +474,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
ucache_control(1, REP_RATE, CACHEOP_MAX);
#endif
CLEAR_TENSOR_ERROR;
// Rendezvous with the producer so its counter reset is visible before we read.
et_barrier(ET_BARRIER_MINION);
evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS;
evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS;

Expand Down
15 changes: 13 additions & 2 deletions ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_K_matrix_engine.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,11 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
if (is_hart1) {
scp_signal(ready_ctr, 0);
scp_signal(consumed_ctr, 0);
// L2-SCP persists across kernel dispatches (only zeroed at boot) and the
// two harts have no implicit ordering at entry. Barrier so the consumer
// sees this reset before its first scp_wait, otherwise on a cold first
// dispatch it reads a stale counter and races ahead of the producer.
et_barrier(ET_BARRIER_MINION);
uint32_t wid = 0;

for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) {
Expand Down Expand Up @@ -367,7 +372,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
src0_batch, mb, kb0 + i, nb1_0);
}
FENCE;
flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64);
flush_to_l2_multi(cache_buf[buf], kbn * BLOCK_K, 64);
WAIT_CACHEOPS;

wid++;
Expand All @@ -384,6 +389,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
ucache_control(1, REP_RATE, CACHEOP_MAX);
#endif
CLEAR_TENSOR_ERROR;
// Rendezvous with the producer so its counter reset is visible before we read.
et_barrier(ET_BARRIER_MINION);
evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS;
evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS;

Expand Down Expand Up @@ -484,6 +491,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
if (is_hart1) {
scp_signal(ready_ctr, 0);
scp_signal(consumed_ctr, 0);
// See REUSE path: barrier so the consumer observes the reset before it reads.
et_barrier(ET_BARRIER_MINION);
uint32_t chunk_id = 0;

for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) {
Expand All @@ -507,7 +516,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
dequant_q4_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0);

FENCE;
flush_to_l2(scp_panel[buf], BLOCK_K, 64);
flush_to_l2_multi(scp_panel[buf], BLOCK_K, 64);
WAIT_CACHEOPS;

chunk_id++;
Expand All @@ -523,6 +532,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
ucache_control(1, REP_RATE, CACHEOP_MAX);
#endif
CLEAR_TENSOR_ERROR;
// Rendezvous with the producer so its counter reset is visible before we read.
et_barrier(ET_BARRIER_MINION);
evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS;
evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS;

Expand Down
15 changes: 13 additions & 2 deletions ggml/src/ggml-et/et-kernels/src/mul_mat_Q5_K_matrix_engine.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,11 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
if (is_hart1) {
scp_signal(ready_ctr, 0);
scp_signal(consumed_ctr, 0);
// L2-SCP persists across kernel dispatches (only zeroed at boot) and the
// two harts have no implicit ordering at entry. Barrier so the consumer
// sees this reset before its first scp_wait, otherwise on a cold first
// dispatch it reads a stale counter and races ahead of the producer.
et_barrier(ET_BARRIER_MINION);
uint32_t wid = 0;

for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) {
Expand Down Expand Up @@ -308,7 +313,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
src0_batch, mb, kb0 + i, nb1_0);
}
FENCE;
flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64);
flush_to_l2_multi(cache_buf[buf], kbn * BLOCK_K, 64);
WAIT_CACHEOPS;

wid++;
Expand All @@ -325,6 +330,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
ucache_control(1, REP_RATE, CACHEOP_MAX);
#endif
CLEAR_TENSOR_ERROR;
// Rendezvous with the producer so its counter reset is visible before we read.
et_barrier(ET_BARRIER_MINION);
evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS;
evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS;

Expand Down Expand Up @@ -425,6 +432,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
if (is_hart1) {
scp_signal(ready_ctr, 0);
scp_signal(consumed_ctr, 0);
// See REUSE path: barrier so the consumer observes the reset before it reads.
et_barrier(ET_BARRIER_MINION);
uint32_t chunk_id = 0;

for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) {
Expand All @@ -448,7 +457,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
dequant_q5_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0);

FENCE;
flush_to_l2(scp_panel[buf], BLOCK_K, 64);
flush_to_l2_multi(scp_panel[buf], BLOCK_K, 64);
WAIT_CACHEOPS;

chunk_id++;
Expand All @@ -464,6 +473,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
ucache_control(1, REP_RATE, CACHEOP_MAX);
#endif
CLEAR_TENSOR_ERROR;
// Rendezvous with the producer so its counter reset is visible before we read.
et_barrier(ET_BARRIER_MINION);
evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS;
evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS;

Expand Down
15 changes: 13 additions & 2 deletions ggml/src/ggml-et/et-kernels/src/mul_mat_Q6_K_matrix_engine.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,11 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
if (is_hart1) {
scp_signal(ready_ctr, 0);
scp_signal(consumed_ctr, 0);
// L2-SCP persists across kernel dispatches (only zeroed at boot) and the
// two harts have no implicit ordering at entry. Barrier so the consumer
// sees this reset before its first scp_wait, otherwise on a cold first
// dispatch it reads a stale counter and races ahead of the producer.
et_barrier(ET_BARRIER_MINION);
uint32_t wid = 0;

for (int64_t unit = my_start; unit < base_units; unit += tiles_stride) {
Expand Down Expand Up @@ -311,7 +316,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
src0_batch, mb, kb0 + i, nb1_0);
}
FENCE;
flush_to_l2(cache_buf[buf], kbn * BLOCK_K, 64);
flush_to_l2_multi(cache_buf[buf], kbn * BLOCK_K, 64);
WAIT_CACHEOPS;

wid++;
Expand All @@ -328,6 +333,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
ucache_control(1, REP_RATE, CACHEOP_MAX);
#endif
CLEAR_TENSOR_ERROR;
// Rendezvous with the producer so its counter reset is visible before we read.
et_barrier(ET_BARRIER_MINION);
evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS;
evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS;

Expand Down Expand Up @@ -428,6 +435,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
if (is_hart1) {
scp_signal(ready_ctr, 0);
scp_signal(consumed_ctr, 0);
// See REUSE path: barrier so the consumer observes the reset before it reads.
et_barrier(ET_BARRIER_MINION);
uint32_t chunk_id = 0;

for (int64_t tile = my_start; tile < base_tiles; tile += tiles_stride) {
Expand All @@ -451,7 +460,7 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
dequant_q6_K_panel(scp_panel[buf], src0_batch, mb, kb, nb1_0);

FENCE;
flush_to_l2(scp_panel[buf], BLOCK_K, 64);
flush_to_l2_multi(scp_panel[buf], BLOCK_K, 64);
WAIT_CACHEOPS;

chunk_id++;
Expand All @@ -467,6 +476,8 @@ int entry_point(struct ggml_et_binary_params *params, void *env) {
ucache_control(1, REP_RATE, CACHEOP_MAX);
#endif
CLEAR_TENSOR_ERROR;
// Rendezvous with the producer so its counter reset is visible before we read.
et_barrier(ET_BARRIER_MINION);
evict_to_l2((const void *) ready_ctr, 1, 64); WAIT_CACHEOPS;
evict_to_l2((const void *) consumed_ctr, 1, 64); WAIT_CACHEOPS;

Expand Down
14 changes: 14 additions & 0 deletions ggml/src/ggml-et/et-kernels/src/platform.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -498,6 +498,20 @@ static inline void __attribute__((always_inline)) flush_to_l2(const void * addr,
: "x31", "memory");
}

// Flush an arbitrary number of lines to L2, working around the 16-line cap of a
// single FlushVA by issuing multiple flushes. Use this whenever nlines may exceed 16.
static inline void __attribute__((always_inline)) flush_to_l2_multi(const void * addr, uint64_t nlines, uint64_t stride) {
const char * p = (const char *) addr;
while (nlines > 16) {
flush_to_l2(p, 16, stride);
p += 16 * stride;
nlines -= 16;
}
if (nlines) {
flush_to_l2(p, nlines, stride);
}
}

// Evict nlines cache lines at stride apart starting at addr from L1 to L2.
// Uses EvictVA (CSR 0x89F). Unlike flush_to_l2, this guarantees the line is
// NOT present in L1 after the operation - subsequent loads will miss and go
Expand Down
Loading