diff --git a/docs/tileop-usage/tlsu/load-store-move/TLOAD.md b/docs/tileop-usage/tlsu/load-store-move/TLOAD.md index 86f8ee1..4bd354b 100644 --- a/docs/tileop-usage/tlsu/load-store-move/TLOAD.md +++ b/docs/tileop-usage/tlsu/load-store-move/TLOAD.md @@ -126,6 +126,7 @@ void load_vector_cubes(float *data16, float *data32, - 省略 `B.DATR` 时使用该操作规定的默认编码;若显式提供该描述符,未使用的字段必须保持为零。 - `LB0` 给出 `ValidCol`,必须存在且非零;省略 `LB1` 时 `ValidRow=1`,省略 `LB2` 时物理列数等于 `ValidCol`。显式给出的维度不能为零。 +- `ValidRow`/`ValidCol` 为运行期(`DYNAMIC`)时,对应 `B.DIM` 使用寄存器源形式(`B.DIM , 0, ->lbN`,RegSrc 为绝对 GPR 0..23),不使用立即数/压缩形式;仅静态维度使用 `C.B.DIMI`/立即数形式(ADR-BLOCK-0012 Decision 013/014)。每个 `LB0/LB1/LB2` 只写一次。 - 省略 `B.IOR` 时使用本操作规定的寄存器或控制默认值;显式编码为零表示实际的零值,不等同于省略该描述符。 `fixp::Options` 内部字段的默认值和合法组合见 [Options 指南](../../options.md)。 diff --git a/docs/tileop-usage/tlsu/load-store-move/TSTORE.md b/docs/tileop-usage/tlsu/load-store-move/TSTORE.md index 64afe4a..12a36c5 100644 --- a/docs/tileop-usage/tlsu/load-store-move/TSTORE.md +++ b/docs/tileop-usage/tlsu/load-store-move/TSTORE.md @@ -131,6 +131,7 @@ void store_vector_cubes(float *data16, float *data32, - 省略 `B.DATR` 时使用该操作规定的默认编码;若显式提供该描述符,未使用的字段必须保持为零。 - `LB0` 给出 `ValidCol`,必须存在且非零;省略 `LB1` 时 `ValidRow=1`,省略 `LB2` 时物理列数等于 `ValidCol`。显式给出的维度不能为零。 +- `ValidRow`/`ValidCol` 为运行期(`DYNAMIC`)时,对应 `B.DIM` 使用寄存器源形式(`B.DIM , 0, ->lbN`,RegSrc 为绝对 GPR 0..23),不使用立即数/压缩形式;仅静态维度使用 `C.B.DIMI`/立即数形式(ADR-BLOCK-0012 Decision 013/014)。每个 `LB0/LB1/LB2` 只写一次。 - 省略 `B.IOR` 时使用本操作规定的寄存器或控制默认值;显式编码为零表示实际的零值,不等同于省略该描述符。 `fixp::Options` 内部字段的默认值和合法组合见 [Options 指南](../../options.md)。 diff --git a/include/jcore/template_asm.hpp b/include/jcore/template_asm.hpp index 43163f7..61cb1e2 100644 --- a/include/jcore/template_asm.hpp +++ b/include/jcore/template_asm.hpp @@ -2398,6 +2398,10 @@ void TLOAD(tile_shape &dst, gm_shape &src) { } } } else { + // ASL B.DIM (ADR-BLOCK-0012 Decision 014): a runtime dimension must come + // from a GPR (RegSrc != zero); only static dims may use the immediate + // form. Per-dim SS/SD/DS/DD dispatch mirrors TLOAD_CUBE. + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { asm volatile( "BSTART.TLSU TLOAD, %D[SrcType]\n" "B.DIM zero, %c[VCOL], ->lb0\n" @@ -2409,11 +2413,60 @@ void TLOAD(tile_shape &dst, gm_shape &src) { : [s0]"r"(src.data()), [SrcType]"i"(type_traits::TypeCode), [TileSize]"i"(tile_type_traits::TilesizeCode), - [VCOL]"i"(valid_col), [VROW]"i"(valid_row), + [VCOL]"i"(tile_shape::ValidCol), [VROW]"i"(tile_shape::ValidRow), + [COL]"i"(tile_shape::Cols), + [GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); + } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TLSU TLOAD, %D[SrcType]\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + "B.IOT mask=1111, last, ->%[d0]<%Z[TileSize]>\n" + "B.IOR [%[s0],%[GmStride]], []\n" + : [d0]"=Tr"(dst.data()) + : [s0]"r"(src.data()), + [SrcType]"i"(type_traits::TypeCode), + [TileSize]"i"(tile_type_traits::TilesizeCode), + [VCOL]"i"(tile_shape::ValidCol), [VROW]"r"(valid_row), + [COL]"i"(tile_shape::Cols), + [GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); + } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TLSU TLOAD, %D[SrcType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + "B.IOT mask=1111, last, ->%[d0]<%Z[TileSize]>\n" + "B.IOR [%[s0],%[GmStride]], []\n" + : [d0]"=Tr"(dst.data()) + : [s0]"r"(src.data()), + [SrcType]"i"(type_traits::TypeCode), + [TileSize]"i"(tile_type_traits::TilesizeCode), + [VCOL]"r"(valid_col), [VROW]"i"(tile_shape::ValidRow), + [COL]"i"(tile_shape::Cols), + [GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); + } else { + asm volatile( + "BSTART.TLSU TLOAD, %D[SrcType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + "B.IOT mask=1111, last, ->%[d0]<%Z[TileSize]>\n" + "B.IOR [%[s0],%[GmStride]], []\n" + : [d0]"=Tr"(dst.data()) + : [s0]"r"(src.data()), + [SrcType]"i"(type_traits::TypeCode), + [TileSize]"i"(tile_type_traits::TilesizeCode), + [VCOL]"r"(valid_col), [VROW]"r"(valid_row), [COL]"i"(tile_shape::Cols), [GmStride]"r"(src.GetStrideBytes(3)) : "memory"); } + } } // TLOAD: GM -> Shared Tile (PTO v0.58 reissue). The destination is one @@ -2430,6 +2483,8 @@ PTO_SHARED_INLINE SharedTile TLOAD(const gm_shape &src) { SharedTile result; const size_t valid_col = result.GetValidCol(); const size_t valid_row = result.GetValidRow(); + // ASL B.DIM: dynamic dims load from a GPR; per-dim SS/SD/DS/DD dispatch. + if constexpr (shp::ValidCol > 0 && shp::ValidRow > 0) { asm volatile( "BSTART.TLSU TLOAD, %D[SrcType]\n" "B.DIM zero, %c[VCOL], ->lb0\n" @@ -2442,10 +2497,62 @@ PTO_SHARED_INLINE SharedTile TLOAD(const gm_shape &src) { [PEMask]"i"(PEMask), [SrcType]"i"(type_traits::TypeCode), [TileSize]"i"(tile_type_traits::TilesizeCode), - [VCOL]"i"(valid_col), [VROW]"i"(valid_row), + [VCOL]"i"(shp::ValidCol), [VROW]"i"(shp::ValidRow), + [COL]"i"(shp::Cols), + [GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); + } else if constexpr (shp::ValidCol > 0 && shp::ValidRow < 0) { + asm volatile( + "BSTART.TLSU TLOAD, %D[SrcType]\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + PTO_PE_MASK_ASM("B.IOS mask=", ", ->%S[Shared]<%Z[TileSize]>\n") + "B.IOR [%[s0],%[GmStride]], []\n" + : [Shared] "=Sr"(result.handle_ref()) + : [s0]"r"(src.data()), + [PEMask]"i"(PEMask), + [SrcType]"i"(type_traits::TypeCode), + [TileSize]"i"(tile_type_traits::TilesizeCode), + [VCOL]"i"(shp::ValidCol), [VROW]"r"(valid_row), + [COL]"i"(shp::Cols), + [GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); + } else if constexpr (shp::ValidCol < 0 && shp::ValidRow > 0) { + asm volatile( + "BSTART.TLSU TLOAD, %D[SrcType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + PTO_PE_MASK_ASM("B.IOS mask=", ", ->%S[Shared]<%Z[TileSize]>\n") + "B.IOR [%[s0],%[GmStride]], []\n" + : [Shared] "=Sr"(result.handle_ref()) + : [s0]"r"(src.data()), + [PEMask]"i"(PEMask), + [SrcType]"i"(type_traits::TypeCode), + [TileSize]"i"(tile_type_traits::TilesizeCode), + [VCOL]"r"(valid_col), [VROW]"i"(shp::ValidRow), + [COL]"i"(shp::Cols), + [GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); + } else { + asm volatile( + "BSTART.TLSU TLOAD, %D[SrcType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + PTO_PE_MASK_ASM("B.IOS mask=", ", ->%S[Shared]<%Z[TileSize]>\n") + "B.IOR [%[s0],%[GmStride]], []\n" + : [Shared] "=Sr"(result.handle_ref()) + : [s0]"r"(src.data()), + [PEMask]"i"(PEMask), + [SrcType]"i"(type_traits::TypeCode), + [TileSize]"i"(tile_type_traits::TilesizeCode), + [VCOL]"r"(valid_col), [VROW]"r"(valid_row), [COL]"i"(shp::Cols), [GmStride]"r"(src.GetStrideBytes(3)) : "memory"); + } return result; } @@ -2459,6 +2566,8 @@ PTO_SHARED_INLINE void TLOAD(SharedTile &dst, const gm_shape &src) { "TLOAD Shared dst logical Tile size must be 128 B..256 KB (SizeCode=1..12)"); const size_t valid_col = dst.GetValidCol(); const size_t valid_row = dst.GetValidRow(); + // ASL B.DIM: dynamic dims load from a GPR; per-dim SS/SD/DS/DD dispatch. + if constexpr (shp::ValidCol > 0 && shp::ValidRow > 0) { asm volatile( "BSTART.TLSU TLOAD, %D[SrcType]\n" "B.DIM zero, %c[VCOL], ->lb0\n" @@ -2471,10 +2580,62 @@ PTO_SHARED_INLINE void TLOAD(SharedTile &dst, const gm_shape &src) { [PEMask]"i"(PEMask), [SrcType]"i"(type_traits::TypeCode), [TileSize]"i"(tile_type_traits::TilesizeCode), - [VCOL]"i"(valid_col), [VROW]"i"(valid_row), + [VCOL]"i"(shp::ValidCol), [VROW]"i"(shp::ValidRow), + [COL]"i"(shp::Cols), + [GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); + } else if constexpr (shp::ValidCol > 0 && shp::ValidRow < 0) { + asm volatile( + "BSTART.TLSU TLOAD, %D[SrcType]\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + PTO_PE_MASK_ASM("B.IOS mask=", ", ->%S[Shared]<%Z[TileSize]>\n") + "B.IOR [%[s0],%[GmStride]], []\n" + : [Shared] "=Sr"(dst.handle_ref()) + : [s0]"r"(src.data()), + [PEMask]"i"(PEMask), + [SrcType]"i"(type_traits::TypeCode), + [TileSize]"i"(tile_type_traits::TilesizeCode), + [VCOL]"i"(shp::ValidCol), [VROW]"r"(valid_row), + [COL]"i"(shp::Cols), + [GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); + } else if constexpr (shp::ValidCol < 0 && shp::ValidRow > 0) { + asm volatile( + "BSTART.TLSU TLOAD, %D[SrcType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + PTO_PE_MASK_ASM("B.IOS mask=", ", ->%S[Shared]<%Z[TileSize]>\n") + "B.IOR [%[s0],%[GmStride]], []\n" + : [Shared] "=Sr"(dst.handle_ref()) + : [s0]"r"(src.data()), + [PEMask]"i"(PEMask), + [SrcType]"i"(type_traits::TypeCode), + [TileSize]"i"(tile_type_traits::TilesizeCode), + [VCOL]"r"(valid_col), [VROW]"i"(shp::ValidRow), + [COL]"i"(shp::Cols), + [GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); + } else { + asm volatile( + "BSTART.TLSU TLOAD, %D[SrcType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + PTO_PE_MASK_ASM("B.IOS mask=", ", ->%S[Shared]<%Z[TileSize]>\n") + "B.IOR [%[s0],%[GmStride]], []\n" + : [Shared] "=Sr"(dst.handle_ref()) + : [s0]"r"(src.data()), + [PEMask]"i"(PEMask), + [SrcType]"i"(type_traits::TypeCode), + [TileSize]"i"(tile_type_traits::TilesizeCode), + [VCOL]"r"(valid_col), [VROW]"r"(valid_row), [COL]"i"(shp::Cols), [GmStride]"r"(src.GetStrideBytes(3)) : "memory"); + } } // TSTORE: Tile -> GM (BSTART.TLSU TSTORE). dst[r0+i, c0+j] = src[i,j]. @@ -2618,6 +2779,8 @@ void TSTORE(gm_shape &dst, tile_shape &src) { } } } else { + // ASL B.DIM: dynamic dims load from a GPR; per-dim SS/SD/DS/DD dispatch. + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { asm volatile( "BSTART.TLSU TSTORE, %D[SrcType]\n" "B.DIM zero, %c[VCOL], ->lb0\n" @@ -2628,10 +2791,56 @@ void TSTORE(gm_shape &dst, tile_shape &src) { : : [d0]"r"(dst.data()), [s0]"Tr"(src.data()), [SrcType]"i"(type_traits::TypeCode), - [VCOL]"i"(valid_col), [VROW]"i"(valid_row), + [VCOL]"i"(tile_shape::ValidCol), [VROW]"i"(tile_shape::ValidRow), [COL]"i"(tile_shape::Cols), [GmStride]"r"(dst.GetStrideBytes(3)) : "memory"); + } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TLSU TSTORE, %D[SrcType]\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + "B.IOT %[s0], mask=1111, last\n" + "B.IOR [%[d0],%[GmStride]], []\n" + : + : [d0]"r"(dst.data()), [s0]"Tr"(src.data()), + [SrcType]"i"(type_traits::TypeCode), + [VCOL]"i"(tile_shape::ValidCol), [VROW]"r"(valid_row), + [COL]"i"(tile_shape::Cols), + [GmStride]"r"(dst.GetStrideBytes(3)) + : "memory"); + } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TLSU TSTORE, %D[SrcType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + "B.IOT %[s0], mask=1111, last\n" + "B.IOR [%[d0],%[GmStride]], []\n" + : + : [d0]"r"(dst.data()), [s0]"Tr"(src.data()), + [SrcType]"i"(type_traits::TypeCode), + [VCOL]"r"(valid_col), [VROW]"i"(tile_shape::ValidRow), + [COL]"i"(tile_shape::Cols), + [GmStride]"r"(dst.GetStrideBytes(3)) + : "memory"); + } else { + asm volatile( + "BSTART.TLSU TSTORE, %D[SrcType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + "B.IOT %[s0], mask=1111, last\n" + "B.IOR [%[d0],%[GmStride]], []\n" + : + : [d0]"r"(dst.data()), [s0]"Tr"(src.data()), + [SrcType]"i"(type_traits::TypeCode), + [VCOL]"r"(valid_col), [VROW]"r"(valid_row), + [COL]"i"(tile_shape::Cols), + [GmStride]"r"(dst.GetStrideBytes(3)) + : "memory"); + } } }