From f6a037a7deddc1a7fc4aa997f8a7100632267586 Mon Sep 17 00:00:00 2001 From: LinxISA Automation Date: Wed, 9 Sep 2026 11:22:29 +0800 Subject: [PATCH 1/2] TileOP: column-reduction B.DIM describes the source tile geometry Issue #63 column remainder. ASL SelectedBundleComparisonShapeMatches requires the B.DIM LB1 to equal the source valid rows, but TCOLSUM/MAX/MIN/PROD/ARGMAX/ARGMIN still bound destination dims in all SS/SD/DS/DD branches: destination ValidRow=1 was encoded into LB1, so a multi-row source silently reduced only row 0. Mirror the PR #69 row-reduction fix on the source side across the six column operations: static branches use tile_shape_in dims, dynamic branches use src.GetValidCol()/GetValidRow(), and the branch guards key on the source dims. Destination static_asserts (1 x N, source column geometry) stay. Docs: state the source-geometry B.DIM rule on the six column operation pages (PR #69 did not update row docs; column pages now lead). --- .../reduce-and-expand/column/TCOLARGMAX.md | 3 +- .../reduce-and-expand/column/TCOLARGMIN.md | 3 +- .../reduce-and-expand/column/TCOLMAX.md | 3 +- .../reduce-and-expand/column/TCOLMIN.md | 3 +- .../reduce-and-expand/column/TCOLPROD.md | 3 +- .../reduce-and-expand/column/TCOLSUM.md | 3 +- include/jcore/template_asm.hpp | 264 +++++++++--------- 7 files changed, 150 insertions(+), 132 deletions(-) diff --git a/docs/tileop-usage/reduce-and-expand/column/TCOLARGMAX.md b/docs/tileop-usage/reduce-and-expand/column/TCOLARGMAX.md index 2ded00e..0b6ed88 100644 --- a/docs/tileop-usage/reduce-and-expand/column/TCOLARGMAX.md +++ b/docs/tileop-usage/reduce-and-expand/column/TCOLARGMAX.md @@ -60,7 +60,8 @@ shape 为 `1 x C`,输出元素是 U32 语义的行索引;输出 Tile dtype ### 编码字段和省略值 - 省略 `B.DATR` 时,padding 值使用 `Null`。 -- `LB0` 给出 `ValidCol`,必须存在且非零;省略 `LB1` 时 `ValidRow=1`,省略 `LB2` 时物理列数等于 `ValidCol`。显式给出的维度不能为零。 +- `LB0`/`LB1`/`LB2` 描述**源 Tile** 几何(源的 `ValidCol`/`ValidRow`/物理列数),且必须与源 descriptor 完全一致;省略 `LB1` 时 `ValidRow=1`,省略 `LB2` 时物理列数等于 `ValidCol`。显式给出的维度不能为零。 +- 列归约的 `LB1` 必须等于**源**的有效行数(即被归约的全部行数),不是目的的单行;目的是规则推导的 `1 x N`,不参与 B.DIM 编码。 - 显式 padding 编码 `00`、`01`、`10`、`11` 分别选择 `Zero`、`Max`、`Min`、`Null`。 `fixp::Options` 内部字段的默认值和合法组合见 [Options 指南](../../options.md)。 diff --git a/docs/tileop-usage/reduce-and-expand/column/TCOLARGMIN.md b/docs/tileop-usage/reduce-and-expand/column/TCOLARGMIN.md index e84cf26..55b59a3 100644 --- a/docs/tileop-usage/reduce-and-expand/column/TCOLARGMIN.md +++ b/docs/tileop-usage/reduce-and-expand/column/TCOLARGMIN.md @@ -60,7 +60,8 @@ shape 为 `1 x C`,输出元素是 U32 语义的行索引;输出 Tile dtype ### 编码字段和省略值 - 省略 `B.DATR` 时,padding 值使用 `Null`。 -- `LB0` 给出 `ValidCol`,必须存在且非零;省略 `LB1` 时 `ValidRow=1`,省略 `LB2` 时物理列数等于 `ValidCol`。显式给出的维度不能为零。 +- `LB0`/`LB1`/`LB2` 描述**源 Tile** 几何(源的 `ValidCol`/`ValidRow`/物理列数),且必须与源 descriptor 完全一致;省略 `LB1` 时 `ValidRow=1`,省略 `LB2` 时物理列数等于 `ValidCol`。显式给出的维度不能为零。 +- 列归约的 `LB1` 必须等于**源**的有效行数(即被归约的全部行数),不是目的的单行;目的是规则推导的 `1 x N`,不参与 B.DIM 编码。 - 显式 padding 编码 `00`、`01`、`10`、`11` 分别选择 `Zero`、`Max`、`Min`、`Null`。 `fixp::Options` 内部字段的默认值和合法组合见 [Options 指南](../../options.md)。 diff --git a/docs/tileop-usage/reduce-and-expand/column/TCOLMAX.md b/docs/tileop-usage/reduce-and-expand/column/TCOLMAX.md index d2465e4..6268d49 100644 --- a/docs/tileop-usage/reduce-and-expand/column/TCOLMAX.md +++ b/docs/tileop-usage/reduce-and-expand/column/TCOLMAX.md @@ -59,7 +59,8 @@ shape 必须为 `1 x C`,输出 dtype 与输入 dtype 相同。 ### 编码字段和省略值 - 省略 `B.DATR` 时,padding 值使用 `Null`。 -- `LB0` 给出 `ValidCol`,必须存在且非零;省略 `LB1` 时 `ValidRow=1`,省略 `LB2` 时物理列数等于 `ValidCol`。显式给出的维度不能为零。 +- `LB0`/`LB1`/`LB2` 描述**源 Tile** 几何(源的 `ValidCol`/`ValidRow`/物理列数),且必须与源 descriptor 完全一致;省略 `LB1` 时 `ValidRow=1`,省略 `LB2` 时物理列数等于 `ValidCol`。显式给出的维度不能为零。 +- 列归约的 `LB1` 必须等于**源**的有效行数(即被归约的全部行数),不是目的的单行;目的是规则推导的 `1 x N`,不参与 B.DIM 编码。 - 显式 padding 编码 `00`、`01`、`10`、`11` 分别选择 `Zero`、`Max`、`Min`、`Null`。 `fixp::Options` 内部字段的默认值和合法组合见 [Options 指南](../../options.md)。 diff --git a/docs/tileop-usage/reduce-and-expand/column/TCOLMIN.md b/docs/tileop-usage/reduce-and-expand/column/TCOLMIN.md index 860d954..ae09018 100644 --- a/docs/tileop-usage/reduce-and-expand/column/TCOLMIN.md +++ b/docs/tileop-usage/reduce-and-expand/column/TCOLMIN.md @@ -59,7 +59,8 @@ shape 必须为 `1 x C`,输出 dtype 与输入 dtype 相同。 ### 编码字段和省略值 - 省略 `B.DATR` 时,padding 值使用 `Null`。 -- `LB0` 给出 `ValidCol`,必须存在且非零;省略 `LB1` 时 `ValidRow=1`,省略 `LB2` 时物理列数等于 `ValidCol`。显式给出的维度不能为零。 +- `LB0`/`LB1`/`LB2` 描述**源 Tile** 几何(源的 `ValidCol`/`ValidRow`/物理列数),且必须与源 descriptor 完全一致;省略 `LB1` 时 `ValidRow=1`,省略 `LB2` 时物理列数等于 `ValidCol`。显式给出的维度不能为零。 +- 列归约的 `LB1` 必须等于**源**的有效行数(即被归约的全部行数),不是目的的单行;目的是规则推导的 `1 x N`,不参与 B.DIM 编码。 - 显式 padding 编码 `00`、`01`、`10`、`11` 分别选择 `Zero`、`Max`、`Min`、`Null`。 `fixp::Options` 内部字段的默认值和合法组合见 [Options 指南](../../options.md)。 diff --git a/docs/tileop-usage/reduce-and-expand/column/TCOLPROD.md b/docs/tileop-usage/reduce-and-expand/column/TCOLPROD.md index 77c8eb9..8c12e93 100644 --- a/docs/tileop-usage/reduce-and-expand/column/TCOLPROD.md +++ b/docs/tileop-usage/reduce-and-expand/column/TCOLPROD.md @@ -59,7 +59,8 @@ shape 必须为 `1 x C`,输出 dtype 与输入 dtype 相同。 ### 编码字段和省略值 - 省略 `B.DATR` 时,padding 值使用 `Null`。 -- `LB0` 给出 `ValidCol`,必须存在且非零;省略 `LB1` 时 `ValidRow=1`,省略 `LB2` 时物理列数等于 `ValidCol`。显式给出的维度不能为零。 +- `LB0`/`LB1`/`LB2` 描述**源 Tile** 几何(源的 `ValidCol`/`ValidRow`/物理列数),且必须与源 descriptor 完全一致;省略 `LB1` 时 `ValidRow=1`,省略 `LB2` 时物理列数等于 `ValidCol`。显式给出的维度不能为零。 +- 列归约的 `LB1` 必须等于**源**的有效行数(即被归约的全部行数),不是目的的单行;目的是规则推导的 `1 x N`,不参与 B.DIM 编码。 - 显式 padding 编码 `00`、`01`、`10`、`11` 分别选择 `Zero`、`Max`、`Min`、`Null`。 `fixp::Options` 内部字段的默认值和合法组合见 [Options 指南](../../options.md)。 diff --git a/docs/tileop-usage/reduce-and-expand/column/TCOLSUM.md b/docs/tileop-usage/reduce-and-expand/column/TCOLSUM.md index ee79512..6480070 100644 --- a/docs/tileop-usage/reduce-and-expand/column/TCOLSUM.md +++ b/docs/tileop-usage/reduce-and-expand/column/TCOLSUM.md @@ -59,7 +59,8 @@ valid shape 必须为 `1 x C`;输出 dtype 与输入 dtype 相同。 ### 编码字段和省略值 - 省略 `B.DATR` 时,padding 值使用 `Null`。 -- `LB0` 给出 `ValidCol`,必须存在且非零;省略 `LB1` 时 `ValidRow=1`,省略 `LB2` 时物理列数等于 `ValidCol`。显式给出的维度不能为零。 +- `LB0`/`LB1`/`LB2` 描述**源 Tile** 几何(源的 `ValidCol`/`ValidRow`/物理列数),且必须与源 descriptor 完全一致;省略 `LB1` 时 `ValidRow=1`,省略 `LB2` 时物理列数等于 `ValidCol`。显式给出的维度不能为零。 +- 列归约的 `LB1` 必须等于**源**的有效行数(即被归约的全部行数),不是目的的单行;目的是规则推导的 `1 x N`,不参与 B.DIM 编码。 - 显式 padding 编码 `00`、`01`、`10`、`11` 分别选择 `Zero`、`Max`、`Min`、`Null`。 `fixp::Options` 内部字段的默认值和合法组合见 [Options 指南](../../options.md)。 diff --git a/include/jcore/template_asm.hpp b/include/jcore/template_asm.hpp index 8bed12d..43163f7 100644 --- a/include/jcore/template_asm.hpp +++ b/include/jcore/template_asm.hpp @@ -14657,15 +14657,17 @@ void TROWARGMIN(tile_shape_out &dst, tile_shape_in &src) { // TCOLSUM: col sum reduction template void TCOLSUM(tile_shape_out &dst, tile_shape_in &src) { - // ASL (reduction-and-expansion): column-axis reduction destination has - // exactly one valid row and inherits the source column geometry. + // ASL (SelectedBundleComparisonShapeMatches): B.DIM describes the SOURCE + // geometry (ValidCol/ValidRow/Col); LB1 must equal the source valid rows. + // The destination is rule-derived: exactly one valid row (1 x N) with the + // source column geometry. static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_out::ValidRow == 1, "TCOLSUM destination must have exactly one valid row (1 x N)"); static_assert(tile_shape_out::ValidCol == DYNAMIC || (tile_shape_out::ValidCol == tile_shape_in::ValidCol && tile_shape_out::Cols == tile_shape_in::Cols), "TCOLSUM destination columns must match the source columns"); - if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { asm volatile( "BSTART.TEPL 80, %D1\n" "B.DIM zero, %c2, ->lb0\n" @@ -14675,54 +14677,54 @@ void TCOLSUM(tile_shape_out &dst, tile_shape_in &src) { "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), - "i"(tile_shape_out::ValidRow), - "i"(tile_shape_out::Cols), + "i"(tile_shape_in::ValidCol), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { + ); } else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { asm volatile( "BSTART.TEPL 80, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), - [dst____dimrow] "r"(dst.GetValidRow()), - "i"(tile_shape_out::Cols), + "i"(tile_shape_in::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { + ); } else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { asm volatile( "BSTART.TEPL 80, %D1\n" - "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - [dst____dimcol] "r"(dst.GetValidCol()), - "i"(tile_shape_out::ValidRow), - "i"(tile_shape_out::Cols), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) ); } else { asm volatile( "BSTART.TEPL 80, %D1\n" - "B.DIM %[dst____dimcol], 0, ->lb0\n" - "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - [dst____dimcol] "r"(dst.GetValidCol()), - [dst____dimrow] "r"(dst.GetValidRow()), - "i"(tile_shape_out::Cols), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) ); } @@ -14731,15 +14733,17 @@ void TCOLSUM(tile_shape_out &dst, tile_shape_in &src) { // TCOLMAX: col max reduction template void TCOLMAX(tile_shape_out &dst, tile_shape_in &src) { - // ASL (reduction-and-expansion): column-axis reduction destination has - // exactly one valid row and inherits the source column geometry. + // ASL (SelectedBundleComparisonShapeMatches): B.DIM describes the SOURCE + // geometry (ValidCol/ValidRow/Col); LB1 must equal the source valid rows. + // The destination is rule-derived: exactly one valid row (1 x N) with the + // source column geometry. static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_out::ValidRow == 1, "TCOLMAX destination must have exactly one valid row (1 x N)"); static_assert(tile_shape_out::ValidCol == DYNAMIC || (tile_shape_out::ValidCol == tile_shape_in::ValidCol && tile_shape_out::Cols == tile_shape_in::Cols), "TCOLMAX destination columns must match the source columns"); - if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { asm volatile( "BSTART.TEPL 81, %D1\n" "B.DIM zero, %c2, ->lb0\n" @@ -14749,54 +14753,54 @@ void TCOLMAX(tile_shape_out &dst, tile_shape_in &src) { "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), - "i"(tile_shape_out::ValidRow), - "i"(tile_shape_out::Cols), + "i"(tile_shape_in::ValidCol), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { + ); } else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { asm volatile( "BSTART.TEPL 81, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), - [dst____dimrow] "r"(dst.GetValidRow()), - "i"(tile_shape_out::Cols), + "i"(tile_shape_in::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { + ); } else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { asm volatile( "BSTART.TEPL 81, %D1\n" - "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - [dst____dimcol] "r"(dst.GetValidCol()), - "i"(tile_shape_out::ValidRow), - "i"(tile_shape_out::Cols), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) ); } else { asm volatile( "BSTART.TEPL 81, %D1\n" - "B.DIM %[dst____dimcol], 0, ->lb0\n" - "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - [dst____dimcol] "r"(dst.GetValidCol()), - [dst____dimrow] "r"(dst.GetValidRow()), - "i"(tile_shape_out::Cols), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) ); } @@ -14805,15 +14809,17 @@ void TCOLMAX(tile_shape_out &dst, tile_shape_in &src) { // TCOLMIN: col min reduction template void TCOLMIN(tile_shape_out &dst, tile_shape_in &src) { - // ASL (reduction-and-expansion): column-axis reduction destination has - // exactly one valid row and inherits the source column geometry. + // ASL (SelectedBundleComparisonShapeMatches): B.DIM describes the SOURCE + // geometry (ValidCol/ValidRow/Col); LB1 must equal the source valid rows. + // The destination is rule-derived: exactly one valid row (1 x N) with the + // source column geometry. static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_out::ValidRow == 1, "TCOLMIN destination must have exactly one valid row (1 x N)"); static_assert(tile_shape_out::ValidCol == DYNAMIC || (tile_shape_out::ValidCol == tile_shape_in::ValidCol && tile_shape_out::Cols == tile_shape_in::Cols), "TCOLMIN destination columns must match the source columns"); - if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { asm volatile( "BSTART.TEPL 82, %D1\n" "B.DIM zero, %c2, ->lb0\n" @@ -14823,54 +14829,54 @@ void TCOLMIN(tile_shape_out &dst, tile_shape_in &src) { "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), - "i"(tile_shape_out::ValidRow), - "i"(tile_shape_out::Cols), + "i"(tile_shape_in::ValidCol), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { + ); } else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { asm volatile( "BSTART.TEPL 82, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), - [dst____dimrow] "r"(dst.GetValidRow()), - "i"(tile_shape_out::Cols), + "i"(tile_shape_in::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { + ); } else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { asm volatile( "BSTART.TEPL 82, %D1\n" - "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - [dst____dimcol] "r"(dst.GetValidCol()), - "i"(tile_shape_out::ValidRow), - "i"(tile_shape_out::Cols), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) ); } else { asm volatile( "BSTART.TEPL 82, %D1\n" - "B.DIM %[dst____dimcol], 0, ->lb0\n" - "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - [dst____dimcol] "r"(dst.GetValidCol()), - [dst____dimrow] "r"(dst.GetValidRow()), - "i"(tile_shape_out::Cols), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) ); } @@ -14879,15 +14885,17 @@ void TCOLMIN(tile_shape_out &dst, tile_shape_in &src) { // TCOLPROD: col product reduction template void TCOLPROD(tile_shape_out &dst, tile_shape_in &src) { - // ASL (reduction-and-expansion): column-axis reduction destination has - // exactly one valid row and inherits the source column geometry. + // ASL (SelectedBundleComparisonShapeMatches): B.DIM describes the SOURCE + // geometry (ValidCol/ValidRow/Col); LB1 must equal the source valid rows. + // The destination is rule-derived: exactly one valid row (1 x N) with the + // source column geometry. static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_out::ValidRow == 1, "TCOLPROD destination must have exactly one valid row (1 x N)"); static_assert(tile_shape_out::ValidCol == DYNAMIC || (tile_shape_out::ValidCol == tile_shape_in::ValidCol && tile_shape_out::Cols == tile_shape_in::Cols), "TCOLPROD destination columns must match the source columns"); - if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { asm volatile( "BSTART.TEPL 83, %D1\n" "B.DIM zero, %c2, ->lb0\n" @@ -14897,54 +14905,54 @@ void TCOLPROD(tile_shape_out &dst, tile_shape_in &src) { "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), - "i"(tile_shape_out::ValidRow), - "i"(tile_shape_out::Cols), + "i"(tile_shape_in::ValidCol), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { + ); } else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { asm volatile( "BSTART.TEPL 83, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), - [dst____dimrow] "r"(dst.GetValidRow()), - "i"(tile_shape_out::Cols), + "i"(tile_shape_in::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { + ); } else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { asm volatile( "BSTART.TEPL 83, %D1\n" - "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - [dst____dimcol] "r"(dst.GetValidCol()), - "i"(tile_shape_out::ValidRow), - "i"(tile_shape_out::Cols), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) ); } else { asm volatile( "BSTART.TEPL 83, %D1\n" - "B.DIM %[dst____dimcol], 0, ->lb0\n" - "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - [dst____dimcol] "r"(dst.GetValidCol()), - [dst____dimrow] "r"(dst.GetValidRow()), - "i"(tile_shape_out::Cols), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) ); } @@ -15023,15 +15031,17 @@ void TCOLEXPAND(tile_shape_out &dst, tile_shape_in &src) { // TCOLARGMAX: col argmax (DavinciOO ext) template void TCOLARGMAX(tile_shape_out &dst, tile_shape_in &src) { - // ASL (reduction-and-expansion): column-axis reduction destination has - // exactly one valid row and inherits the source column geometry. + // ASL (SelectedBundleComparisonShapeMatches): B.DIM describes the SOURCE + // geometry (ValidCol/ValidRow/Col); LB1 must equal the source valid rows. + // The destination is rule-derived: exactly one valid row (1 x N) with the + // source column geometry. static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_out::ValidRow == 1, "TCOLARGMAX destination must have exactly one valid row (1 x N)"); static_assert(tile_shape_out::ValidCol == DYNAMIC || (tile_shape_out::ValidCol == tile_shape_in::ValidCol && tile_shape_out::Cols == tile_shape_in::Cols), "TCOLARGMAX destination columns must match the source columns"); - if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { asm volatile( "BSTART.TEPL 92, %D1\n" "B.DIM zero, %c2, ->lb0\n" @@ -15041,54 +15051,54 @@ void TCOLARGMAX(tile_shape_out &dst, tile_shape_in &src) { "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), - "i"(tile_shape_out::ValidRow), - "i"(tile_shape_out::Cols), + "i"(tile_shape_in::ValidCol), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { + ); } else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { asm volatile( "BSTART.TEPL 92, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), - [dst____dimrow] "r"(dst.GetValidRow()), - "i"(tile_shape_out::Cols), + "i"(tile_shape_in::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { + ); } else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { asm volatile( "BSTART.TEPL 92, %D1\n" - "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - [dst____dimcol] "r"(dst.GetValidCol()), - "i"(tile_shape_out::ValidRow), - "i"(tile_shape_out::Cols), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) ); } else { asm volatile( "BSTART.TEPL 92, %D1\n" - "B.DIM %[dst____dimcol], 0, ->lb0\n" - "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - [dst____dimcol] "r"(dst.GetValidCol()), - [dst____dimrow] "r"(dst.GetValidRow()), - "i"(tile_shape_out::Cols), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) ); } @@ -15097,15 +15107,17 @@ void TCOLARGMAX(tile_shape_out &dst, tile_shape_in &src) { // TCOLARGMIN: col argmin (DavinciOO ext) template void TCOLARGMIN(tile_shape_out &dst, tile_shape_in &src) { - // ASL (reduction-and-expansion): column-axis reduction destination has - // exactly one valid row and inherits the source column geometry. + // ASL (SelectedBundleComparisonShapeMatches): B.DIM describes the SOURCE + // geometry (ValidCol/ValidRow/Col); LB1 must equal the source valid rows. + // The destination is rule-derived: exactly one valid row (1 x N) with the + // source column geometry. static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_out::ValidRow == 1, "TCOLARGMIN destination must have exactly one valid row (1 x N)"); static_assert(tile_shape_out::ValidCol == DYNAMIC || (tile_shape_out::ValidCol == tile_shape_in::ValidCol && tile_shape_out::Cols == tile_shape_in::Cols), "TCOLARGMIN destination columns must match the source columns"); - if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { asm volatile( "BSTART.TEPL 93, %D1\n" "B.DIM zero, %c2, ->lb0\n" @@ -15115,54 +15127,54 @@ void TCOLARGMIN(tile_shape_out &dst, tile_shape_in &src) { "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), - "i"(tile_shape_out::ValidRow), - "i"(tile_shape_out::Cols), + "i"(tile_shape_in::ValidCol), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { + ); } else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { asm volatile( "BSTART.TEPL 93, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), - [dst____dimrow] "r"(dst.GetValidRow()), - "i"(tile_shape_out::Cols), + "i"(tile_shape_in::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { + ); } else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { asm volatile( "BSTART.TEPL 93, %D1\n" - "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - [dst____dimcol] "r"(dst.GetValidCol()), - "i"(tile_shape_out::ValidRow), - "i"(tile_shape_out::Cols), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) ); } else { asm volatile( "BSTART.TEPL 93, %D1\n" - "B.DIM %[dst____dimcol], 0, ->lb0\n" - "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - [dst____dimcol] "r"(dst.GetValidCol()), - [dst____dimrow] "r"(dst.GetValidRow()), - "i"(tile_shape_out::Cols), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) ); } From deca1f166a57e45286d171efd97cfe5bb531accc Mon Sep 17 00:00:00 2001 From: LinxISA Automation Date: Wed, 9 Sep 2026 12:19:39 +0800 Subject: [PATCH 2/2] TileOP: dynamic TLOAD/TSTORE dims use register-form B.DIM Issue #100. When a tile's ValidRow/ValidCol is DYNAMIC, GetValidRow()/ GetValidCol() are runtime values, but the plain TLOAD/TSTORE paths and both Shared TLOAD variants bound them to the "i" immediate constraint, so every dynamic-shape kernel failed with 'invalid operand for inline asm constraint i' (regression window 804eb03..b8669ce). Per ASL B.DIM (ADR-BLOCK-0012 Decision 013/014) a runtime dimension must come from a GPR (RegSrc 0..23, B.DIM , 0, ->lbN); only static dims use the immediate/compressed form. Add per-dimension SS/SD/DS/DD dispatch (mirroring the existing TLOAD_CUBE pattern) to: - TLOAD plain path (Local destination) - TSTORE plain path (Local source) - Shared TLOAD (both the returning and void variants) The B.ASSEMBLE/B.SUBVIEW range-modifier emitters inside TLOAD/TSTORE still bind runtime dims to "i"; that region is part of the in-progress TPARTVIEW/TASSEMBLY work (LLVM #74) and is left for a follow-up. Docs: TLOAD/TSTORE pages now state the dynamic-dim register-form rule. --- .../tlsu/load-store-move/TLOAD.md | 1 + .../tlsu/load-store-move/TSTORE.md | 1 + include/jcore/template_asm.hpp | 217 +++++++++++++++++- 3 files changed, 215 insertions(+), 4 deletions(-) diff --git a/docs/tileop-usage/tlsu/load-store-move/TLOAD.md b/docs/tileop-usage/tlsu/load-store-move/TLOAD.md index 86f8ee1..4bd354b 100644 --- a/docs/tileop-usage/tlsu/load-store-move/TLOAD.md +++ b/docs/tileop-usage/tlsu/load-store-move/TLOAD.md @@ -126,6 +126,7 @@ void load_vector_cubes(float *data16, float *data32, - 省略 `B.DATR` 时使用该操作规定的默认编码;若显式提供该描述符,未使用的字段必须保持为零。 - `LB0` 给出 `ValidCol`,必须存在且非零;省略 `LB1` 时 `ValidRow=1`,省略 `LB2` 时物理列数等于 `ValidCol`。显式给出的维度不能为零。 +- `ValidRow`/`ValidCol` 为运行期(`DYNAMIC`)时,对应 `B.DIM` 使用寄存器源形式(`B.DIM , 0, ->lbN`,RegSrc 为绝对 GPR 0..23),不使用立即数/压缩形式;仅静态维度使用 `C.B.DIMI`/立即数形式(ADR-BLOCK-0012 Decision 013/014)。每个 `LB0/LB1/LB2` 只写一次。 - 省略 `B.IOR` 时使用本操作规定的寄存器或控制默认值;显式编码为零表示实际的零值,不等同于省略该描述符。 `fixp::Options` 内部字段的默认值和合法组合见 [Options 指南](../../options.md)。 diff --git a/docs/tileop-usage/tlsu/load-store-move/TSTORE.md b/docs/tileop-usage/tlsu/load-store-move/TSTORE.md index 64afe4a..12a36c5 100644 --- a/docs/tileop-usage/tlsu/load-store-move/TSTORE.md +++ b/docs/tileop-usage/tlsu/load-store-move/TSTORE.md @@ -131,6 +131,7 @@ void store_vector_cubes(float *data16, float *data32, - 省略 `B.DATR` 时使用该操作规定的默认编码;若显式提供该描述符,未使用的字段必须保持为零。 - `LB0` 给出 `ValidCol`,必须存在且非零;省略 `LB1` 时 `ValidRow=1`,省略 `LB2` 时物理列数等于 `ValidCol`。显式给出的维度不能为零。 +- `ValidRow`/`ValidCol` 为运行期(`DYNAMIC`)时,对应 `B.DIM` 使用寄存器源形式(`B.DIM , 0, ->lbN`,RegSrc 为绝对 GPR 0..23),不使用立即数/压缩形式;仅静态维度使用 `C.B.DIMI`/立即数形式(ADR-BLOCK-0012 Decision 013/014)。每个 `LB0/LB1/LB2` 只写一次。 - 省略 `B.IOR` 时使用本操作规定的寄存器或控制默认值;显式编码为零表示实际的零值,不等同于省略该描述符。 `fixp::Options` 内部字段的默认值和合法组合见 [Options 指南](../../options.md)。 diff --git a/include/jcore/template_asm.hpp b/include/jcore/template_asm.hpp index 43163f7..61cb1e2 100644 --- a/include/jcore/template_asm.hpp +++ b/include/jcore/template_asm.hpp @@ -2398,6 +2398,10 @@ void TLOAD(tile_shape &dst, gm_shape &src) { } } } else { + // ASL B.DIM (ADR-BLOCK-0012 Decision 014): a runtime dimension must come + // from a GPR (RegSrc != zero); only static dims may use the immediate + // form. Per-dim SS/SD/DS/DD dispatch mirrors TLOAD_CUBE. + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { asm volatile( "BSTART.TLSU TLOAD, %D[SrcType]\n" "B.DIM zero, %c[VCOL], ->lb0\n" @@ -2409,11 +2413,60 @@ void TLOAD(tile_shape &dst, gm_shape &src) { : [s0]"r"(src.data()), [SrcType]"i"(type_traits::TypeCode), [TileSize]"i"(tile_type_traits::TilesizeCode), - [VCOL]"i"(valid_col), [VROW]"i"(valid_row), + [VCOL]"i"(tile_shape::ValidCol), [VROW]"i"(tile_shape::ValidRow), + [COL]"i"(tile_shape::Cols), + [GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); + } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TLSU TLOAD, %D[SrcType]\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + "B.IOT mask=1111, last, ->%[d0]<%Z[TileSize]>\n" + "B.IOR [%[s0],%[GmStride]], []\n" + : [d0]"=Tr"(dst.data()) + : [s0]"r"(src.data()), + [SrcType]"i"(type_traits::TypeCode), + [TileSize]"i"(tile_type_traits::TilesizeCode), + [VCOL]"i"(tile_shape::ValidCol), [VROW]"r"(valid_row), + [COL]"i"(tile_shape::Cols), + [GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); + } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TLSU TLOAD, %D[SrcType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + "B.IOT mask=1111, last, ->%[d0]<%Z[TileSize]>\n" + "B.IOR [%[s0],%[GmStride]], []\n" + : [d0]"=Tr"(dst.data()) + : [s0]"r"(src.data()), + [SrcType]"i"(type_traits::TypeCode), + [TileSize]"i"(tile_type_traits::TilesizeCode), + [VCOL]"r"(valid_col), [VROW]"i"(tile_shape::ValidRow), + [COL]"i"(tile_shape::Cols), + [GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); + } else { + asm volatile( + "BSTART.TLSU TLOAD, %D[SrcType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + "B.IOT mask=1111, last, ->%[d0]<%Z[TileSize]>\n" + "B.IOR [%[s0],%[GmStride]], []\n" + : [d0]"=Tr"(dst.data()) + : [s0]"r"(src.data()), + [SrcType]"i"(type_traits::TypeCode), + [TileSize]"i"(tile_type_traits::TilesizeCode), + [VCOL]"r"(valid_col), [VROW]"r"(valid_row), [COL]"i"(tile_shape::Cols), [GmStride]"r"(src.GetStrideBytes(3)) : "memory"); } + } } // TLOAD: GM -> Shared Tile (PTO v0.58 reissue). The destination is one @@ -2430,6 +2483,8 @@ PTO_SHARED_INLINE SharedTile TLOAD(const gm_shape &src) { SharedTile result; const size_t valid_col = result.GetValidCol(); const size_t valid_row = result.GetValidRow(); + // ASL B.DIM: dynamic dims load from a GPR; per-dim SS/SD/DS/DD dispatch. + if constexpr (shp::ValidCol > 0 && shp::ValidRow > 0) { asm volatile( "BSTART.TLSU TLOAD, %D[SrcType]\n" "B.DIM zero, %c[VCOL], ->lb0\n" @@ -2442,10 +2497,62 @@ PTO_SHARED_INLINE SharedTile TLOAD(const gm_shape &src) { [PEMask]"i"(PEMask), [SrcType]"i"(type_traits::TypeCode), [TileSize]"i"(tile_type_traits::TilesizeCode), - [VCOL]"i"(valid_col), [VROW]"i"(valid_row), + [VCOL]"i"(shp::ValidCol), [VROW]"i"(shp::ValidRow), + [COL]"i"(shp::Cols), + [GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); + } else if constexpr (shp::ValidCol > 0 && shp::ValidRow < 0) { + asm volatile( + "BSTART.TLSU TLOAD, %D[SrcType]\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + PTO_PE_MASK_ASM("B.IOS mask=", ", ->%S[Shared]<%Z[TileSize]>\n") + "B.IOR [%[s0],%[GmStride]], []\n" + : [Shared] "=Sr"(result.handle_ref()) + : [s0]"r"(src.data()), + [PEMask]"i"(PEMask), + [SrcType]"i"(type_traits::TypeCode), + [TileSize]"i"(tile_type_traits::TilesizeCode), + [VCOL]"i"(shp::ValidCol), [VROW]"r"(valid_row), + [COL]"i"(shp::Cols), + [GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); + } else if constexpr (shp::ValidCol < 0 && shp::ValidRow > 0) { + asm volatile( + "BSTART.TLSU TLOAD, %D[SrcType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + PTO_PE_MASK_ASM("B.IOS mask=", ", ->%S[Shared]<%Z[TileSize]>\n") + "B.IOR [%[s0],%[GmStride]], []\n" + : [Shared] "=Sr"(result.handle_ref()) + : [s0]"r"(src.data()), + [PEMask]"i"(PEMask), + [SrcType]"i"(type_traits::TypeCode), + [TileSize]"i"(tile_type_traits::TilesizeCode), + [VCOL]"r"(valid_col), [VROW]"i"(shp::ValidRow), + [COL]"i"(shp::Cols), + [GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); + } else { + asm volatile( + "BSTART.TLSU TLOAD, %D[SrcType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + PTO_PE_MASK_ASM("B.IOS mask=", ", ->%S[Shared]<%Z[TileSize]>\n") + "B.IOR [%[s0],%[GmStride]], []\n" + : [Shared] "=Sr"(result.handle_ref()) + : [s0]"r"(src.data()), + [PEMask]"i"(PEMask), + [SrcType]"i"(type_traits::TypeCode), + [TileSize]"i"(tile_type_traits::TilesizeCode), + [VCOL]"r"(valid_col), [VROW]"r"(valid_row), [COL]"i"(shp::Cols), [GmStride]"r"(src.GetStrideBytes(3)) : "memory"); + } return result; } @@ -2459,6 +2566,8 @@ PTO_SHARED_INLINE void TLOAD(SharedTile &dst, const gm_shape &src) { "TLOAD Shared dst logical Tile size must be 128 B..256 KB (SizeCode=1..12)"); const size_t valid_col = dst.GetValidCol(); const size_t valid_row = dst.GetValidRow(); + // ASL B.DIM: dynamic dims load from a GPR; per-dim SS/SD/DS/DD dispatch. + if constexpr (shp::ValidCol > 0 && shp::ValidRow > 0) { asm volatile( "BSTART.TLSU TLOAD, %D[SrcType]\n" "B.DIM zero, %c[VCOL], ->lb0\n" @@ -2471,10 +2580,62 @@ PTO_SHARED_INLINE void TLOAD(SharedTile &dst, const gm_shape &src) { [PEMask]"i"(PEMask), [SrcType]"i"(type_traits::TypeCode), [TileSize]"i"(tile_type_traits::TilesizeCode), - [VCOL]"i"(valid_col), [VROW]"i"(valid_row), + [VCOL]"i"(shp::ValidCol), [VROW]"i"(shp::ValidRow), + [COL]"i"(shp::Cols), + [GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); + } else if constexpr (shp::ValidCol > 0 && shp::ValidRow < 0) { + asm volatile( + "BSTART.TLSU TLOAD, %D[SrcType]\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + PTO_PE_MASK_ASM("B.IOS mask=", ", ->%S[Shared]<%Z[TileSize]>\n") + "B.IOR [%[s0],%[GmStride]], []\n" + : [Shared] "=Sr"(dst.handle_ref()) + : [s0]"r"(src.data()), + [PEMask]"i"(PEMask), + [SrcType]"i"(type_traits::TypeCode), + [TileSize]"i"(tile_type_traits::TilesizeCode), + [VCOL]"i"(shp::ValidCol), [VROW]"r"(valid_row), + [COL]"i"(shp::Cols), + [GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); + } else if constexpr (shp::ValidCol < 0 && shp::ValidRow > 0) { + asm volatile( + "BSTART.TLSU TLOAD, %D[SrcType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + PTO_PE_MASK_ASM("B.IOS mask=", ", ->%S[Shared]<%Z[TileSize]>\n") + "B.IOR [%[s0],%[GmStride]], []\n" + : [Shared] "=Sr"(dst.handle_ref()) + : [s0]"r"(src.data()), + [PEMask]"i"(PEMask), + [SrcType]"i"(type_traits::TypeCode), + [TileSize]"i"(tile_type_traits::TilesizeCode), + [VCOL]"r"(valid_col), [VROW]"i"(shp::ValidRow), + [COL]"i"(shp::Cols), + [GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); + } else { + asm volatile( + "BSTART.TLSU TLOAD, %D[SrcType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + PTO_PE_MASK_ASM("B.IOS mask=", ", ->%S[Shared]<%Z[TileSize]>\n") + "B.IOR [%[s0],%[GmStride]], []\n" + : [Shared] "=Sr"(dst.handle_ref()) + : [s0]"r"(src.data()), + [PEMask]"i"(PEMask), + [SrcType]"i"(type_traits::TypeCode), + [TileSize]"i"(tile_type_traits::TilesizeCode), + [VCOL]"r"(valid_col), [VROW]"r"(valid_row), [COL]"i"(shp::Cols), [GmStride]"r"(src.GetStrideBytes(3)) : "memory"); + } } // TSTORE: Tile -> GM (BSTART.TLSU TSTORE). dst[r0+i, c0+j] = src[i,j]. @@ -2618,6 +2779,8 @@ void TSTORE(gm_shape &dst, tile_shape &src) { } } } else { + // ASL B.DIM: dynamic dims load from a GPR; per-dim SS/SD/DS/DD dispatch. + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { asm volatile( "BSTART.TLSU TSTORE, %D[SrcType]\n" "B.DIM zero, %c[VCOL], ->lb0\n" @@ -2628,10 +2791,56 @@ void TSTORE(gm_shape &dst, tile_shape &src) { : : [d0]"r"(dst.data()), [s0]"Tr"(src.data()), [SrcType]"i"(type_traits::TypeCode), - [VCOL]"i"(valid_col), [VROW]"i"(valid_row), + [VCOL]"i"(tile_shape::ValidCol), [VROW]"i"(tile_shape::ValidRow), [COL]"i"(tile_shape::Cols), [GmStride]"r"(dst.GetStrideBytes(3)) : "memory"); + } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TLSU TSTORE, %D[SrcType]\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + "B.IOT %[s0], mask=1111, last\n" + "B.IOR [%[d0],%[GmStride]], []\n" + : + : [d0]"r"(dst.data()), [s0]"Tr"(src.data()), + [SrcType]"i"(type_traits::TypeCode), + [VCOL]"i"(tile_shape::ValidCol), [VROW]"r"(valid_row), + [COL]"i"(tile_shape::Cols), + [GmStride]"r"(dst.GetStrideBytes(3)) + : "memory"); + } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TLSU TSTORE, %D[SrcType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + "B.IOT %[s0], mask=1111, last\n" + "B.IOR [%[d0],%[GmStride]], []\n" + : + : [d0]"r"(dst.data()), [s0]"Tr"(src.data()), + [SrcType]"i"(type_traits::TypeCode), + [VCOL]"r"(valid_col), [VROW]"i"(tile_shape::ValidRow), + [COL]"i"(tile_shape::Cols), + [GmStride]"r"(dst.GetStrideBytes(3)) + : "memory"); + } else { + asm volatile( + "BSTART.TLSU TSTORE, %D[SrcType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + "B.IOT %[s0], mask=1111, last\n" + "B.IOR [%[d0],%[GmStride]], []\n" + : + : [d0]"r"(dst.data()), [s0]"Tr"(src.data()), + [SrcType]"i"(type_traits::TypeCode), + [VCOL]"r"(valid_col), [VROW]"r"(valid_row), + [COL]"i"(tile_shape::Cols), + [GmStride]"r"(dst.GetStrideBytes(3)) + : "memory"); + } } }