From 15cd0ea3c6fee0e870e3cd8f14b96bbf1f2e52ff Mon Sep 17 00:00:00 2001 From: LinxISA Automation Date: Mon, 7 Sep 2026 20:00:21 +0800 Subject: [PATCH 1/3] TileOP: fix static valid-shape dim lowering --- include/jcore/template_asm.hpp | 650 ++++++++++---------- test/test_static_valid_shape_lowering.py | 37 ++ test/test_v058_engine_contract.py | 8 +- test/tileop_api/src/ValidShapeImmediate.cpp | 139 +---- 4 files changed, 376 insertions(+), 458 deletions(-) create mode 100644 test/test_static_valid_shape_lowering.py diff --git a/include/jcore/template_asm.hpp b/include/jcore/template_asm.hpp index 2d7f23f..0f7fef9 100644 --- a/include/jcore/template_asm.hpp +++ b/include/jcore/template_asm.hpp @@ -67,15 +67,15 @@ void TMAX_T(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { asm volatile( "BSTART.VPAR 0b0000100011, %c3\n" "B.IOT %1, %2, mask=1111, last, ->%0<%Z4>\n" - "B.DIM %5, 0, ->lb0\n" - "B.DIM %6, 0, ->lb1\n" + "B.DIM zero, %c5, ->lb0\n" + "B.DIM zero, %c6, ->lb1\n" "" : "=Tr"(dst.data()) : "Tr"(src0.data()), "Tr"(src1.data()), \ "i"(type_traits::TypeCode), "i"(tile_type_traits::TilesizeCode), - "ri"(src0.GetValidCol()), "ri"(src0.GetValidRow()) + "i"(src0.GetValidCol()), "i"(src0.GetValidRow()) ); } @@ -84,15 +84,15 @@ void TSUB_EXP_EXPAND_T(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 asm volatile( "BSTART.VPAR 0b0001000011, %c3\n" "B.IOT %1, %2, mask=1111, last, ->%0<%Z4>\n" - "B.DIM %5, 0, ->lb0\n" - "B.DIM %6, 0, ->lb1\n" + "B.DIM zero, %c5, ->lb0\n" + "B.DIM zero, %c6, ->lb1\n" "" : "=Tr"(dst.data()) : "Tr"(src0.data()), "Tr"(src1.data()), \ "i"(type_traits::TypeCode), "i"(tile_type_traits::TilesizeCode), - "ri"(src0.GetValidCol()), "ri"(src0.GetValidRow()) + "i"(src0.GetValidCol()), "i"(src0.GetValidRow()) ); } @@ -103,14 +103,14 @@ void TMUL_ADD_ROWSUM_T(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 "B.IOT %1, %2, mask=1111, 0, ->%0<%Z5>\n" "B.IOT %3, mask=1111, last\n" "B.DIM %6, 0, ->lb0\n" - "B.DIM %7, 0, ->lb1\n" + "B.DIM zero, %c7, ->lb1\n" "" : "=Tr"(dst.data()) : "Tr"(src0.data()), "Tr"(src1.data()), "Tr"(src2.data()), "i"(type_traits::TypeCode), "i"(tile_type_traits::TilesizeCode), - "ri"(src0.GetValidCol()), "ri"(src0.GetValidRow()) + "i"(src0.GetValidCol()), "i"(src0.GetValidRow()) ); } @@ -121,14 +121,14 @@ void TADD_MUL_EXPAND_T(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 "B.IOT %1, %2, mask=1111, 0, ->%0<%Z5>\n" "B.IOT %3, mask=1111, last\n" "B.DIM %6, 0, ->lb0\n" - "B.DIM %7, 0, ->lb1\n" + "B.DIM zero, %c7, ->lb1\n" "" : "=Tr"(dst.data()) : "Tr"(src0.data()), "Tr"(src1.data()), "Tr"(src2.data()), "i"(type_traits::TypeCode), "i"(tile_type_traits::TilesizeCode), - "ri"(src0.GetValidCol()), "ri"(src0.GetValidRow()) + "i"(src0.GetValidCol()), "i"(src0.GetValidRow()) ); } @@ -166,8 +166,8 @@ void TCVT_T(tile_shape_out &dst, tile_shape_in &src) { asm volatile( "BSTART.TEPL 27, %D1\n" "B.DATR %D2, RNONE\n" - "B.DIM %5, 0, ->lb0\n" - "B.DIM %6, 0, ->lb1\n" + "B.DIM zero, %c5, ->lb0\n" + "B.DIM zero, %c6, ->lb1\n" "B.IOT %3, mask=1111, last, ->%0<%Z4>\n" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), @@ -191,8 +191,8 @@ void TCVT_T(tile_shape_out &dst, tile_shape_in &src) { asm volatile( "BSTART.TEPL 27, %D1\n" "B.DATR %D2, RNONE\n" - "B.DIM %5, 0, ->lb0\n" - "B.DIM %6, 0, ->lb1\n" + "B.DIM zero, %c5, ->lb0\n" + "B.DIM zero, %c6, ->lb1\n" "B.DIM zero, %c7, ->lb2\n" "B.IOT %3, mask=1111, last, ->%0<%Z4>\n" : "=Tr"(dst.data()) @@ -214,14 +214,14 @@ void TMOV_##LAYOUT_NAME(tile_shape_out &dst, tile_shape_in &src) { "BSTART.TLSU TMOV, %D2\n" \ "B.DATR " #LAYOUT_NAME ", DTYPE_NONE, Zero\n" \ "B.IOT %1, mask=1111, last, ->%0<%Z3>\n" \ - "B.DIM %4, 0, ->lb0\n" \ - "B.DIM %5, 0, ->lb1\n" \ + "B.DIM zero, %c4, ->lb0\n" \ + "B.DIM zero, %c5, ->lb1\n" \ : "=Tr"(dst.data()) \ : "Tr"(src.data()), \ "i"(type_traits::TypeCode), \ "i"(tile_type_traits::TilesizeCode), \ - "ri"(src.GetValidCol()), \ - "ri"(src.GetValidRow()) \ + "i"(src.GetValidCol()), \ + "i"(src.GetValidRow()) \ ); \ } @@ -250,8 +250,8 @@ inline void TMOV(Tile &dst, const Tile &src) { const size_t valid_row = src.GetValidRow(); asm volatile( "BSTART.TLSU TMOV, %D[DataType]\n" - "B.DIM %[ValidCol], 0, ->lb0\n" - "B.DIM %[ValidRow], 0, ->lb1\n" + "B.DIM zero, %c[ValidCol], ->lb0\n" + "B.DIM zero, %c[ValidRow], ->lb1\n" "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[TileSize]>\n" : [Dst] "=&Tr"(dst.data()) : [Src] "Tr"(src.data()), @@ -267,15 +267,15 @@ void TMOV_DN2NZ_DYN(tile_shape_out &dst, tile_shape_in &src) { "BSTART.TLSU TMOV, %D2\n" "B.DATR DN2NZ, DTYPE_NONE, Zero\n" "B.IOT %1, mask=1111, last, ->%0<%Z3>\n" - "B.DIM %4, 0, ->lb0\n" - "B.DIM %5, 0, ->lb1\n" + "B.DIM zero, %c4, ->lb0\n" + "B.DIM zero, %c5, ->lb1\n" : "=Tr"(dst.data()) : "Tr"(src.data()), "i"(type_traits::TypeCode), "i"(tile_type_traits::TilesizeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()) + "i"(src.GetValidCol()), + "i"(src.GetValidRow()) ); } @@ -1830,8 +1830,8 @@ void TLOAD(tile_shape &dst, gm_shape &src) { static_cast(dst.GetRangeBase()); asm volatile( "BSTART.TLSU TLOAD, %D[SrcType]\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[COL], ->lb2\n" "B.IOS mask=1111, ->%S[d0]<%Z[TileSize]>\n" "B.ASSEMBLE %c[Init], %c[Last], %[RegSrc], %c[Off], %c[ParentSize]\n" @@ -1840,7 +1840,7 @@ void TLOAD(tile_shape &dst, gm_shape &src) { : [s0]"r"(src.data()), [SrcType]"i"(type_traits::TypeCode), [TileSize]"i"(tile_type_traits::TilesizeCode), - [VCOL]"ri"(valid_col), [VROW]"ri"(valid_row), + [VCOL]"i"(valid_col), [VROW]"i"(valid_row), [COL]"i"(ParentTile::Cols), [GmStride]"r"(src.GetStrideBytes(3)), [Init]"i"(static_cast(tile_shape::INIT)), @@ -1856,8 +1856,8 @@ void TLOAD(tile_shape &dst, gm_shape &src) { static_cast(dst.GetRangeBase()); \ asm volatile( \ "BSTART.TLSU TLOAD, %D[SrcType]\n" \ - "B.DIM %[VCOL], 0, ->lb0\n" \ - "B.DIM %[VROW], 0, ->lb1\n" \ + "B.DIM zero, %c[VCOL], ->lb0\n" \ + "B.DIM zero, %c[VROW], ->lb1\n" \ "B.DIM zero, %c[COL], ->lb2\n" \ "B.IOS mask=1111, ->%S[d0]<%Z[TileSize]>\n" \ "B.ASSEMBLE %c[Init], %c[Last], r" #N ", %c[Off], %c[ParentSize]\n" \ @@ -1866,7 +1866,7 @@ void TLOAD(tile_shape &dst, gm_shape &src) { : [s0]"r"(src.data()), \ [SrcType]"i"(type_traits::TypeCode), \ [TileSize]"i"(tile_type_traits::TilesizeCode), \ - [VCOL]"ri"(valid_col), [VROW]"ri"(valid_row), \ + [VCOL]"i"(valid_col), [VROW]"i"(valid_row), \ [COL]"i"(ParentTile::Cols), \ [GmStride]"r"(src.GetStrideBytes(3)), \ [Init]"i"(static_cast(tile_shape::INIT)), \ @@ -1899,8 +1899,8 @@ void TLOAD(tile_shape &dst, gm_shape &src) { static_cast(dst.GetRangeBase()); asm volatile( "BSTART.TLSU TLOAD, %D[SrcType]\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[COL], ->lb2\n" "B.IOT mask=1111, last, ->%[d0]<%Z[TileSize]>\n" "B.ASSEMBLE %c[Init], %c[Last], %[RegSrc], %c[Off], %c[ParentSize]\n" @@ -1909,7 +1909,7 @@ void TLOAD(tile_shape &dst, gm_shape &src) { : [s0]"r"(src.data()), [SrcType]"i"(type_traits::TypeCode), [TileSize]"i"(tile_type_traits::TilesizeCode), - [VCOL]"ri"(valid_col), [VROW]"ri"(valid_row), + [VCOL]"i"(valid_col), [VROW]"i"(valid_row), [COL]"i"(tile_shape::Cols), [GmStride]"r"(src.GetStrideBytes(3)), [Init]"i"(static_cast(tile_shape::INIT)), @@ -1925,8 +1925,8 @@ void TLOAD(tile_shape &dst, gm_shape &src) { static_cast(dst.GetRangeBase()); \ asm volatile( \ "BSTART.TLSU TLOAD, %D[SrcType]\n" \ - "B.DIM %[VCOL], 0, ->lb0\n" \ - "B.DIM %[VROW], 0, ->lb1\n" \ + "B.DIM zero, %c[VCOL], ->lb0\n" \ + "B.DIM zero, %c[VROW], ->lb1\n" \ "B.DIM zero, %c[COL], ->lb2\n" \ "B.IOT mask=1111, last, ->%[d0]<%Z[TileSize]>\n" \ "B.ASSEMBLE %c[Init], %c[Last], r" #N ", %c[Off], %c[ParentSize]\n" \ @@ -1935,7 +1935,7 @@ void TLOAD(tile_shape &dst, gm_shape &src) { : [s0]"r"(src.data()), \ [SrcType]"i"(type_traits::TypeCode), \ [TileSize]"i"(tile_type_traits::TilesizeCode), \ - [VCOL]"ri"(valid_col), [VROW]"ri"(valid_row), \ + [VCOL]"i"(valid_col), [VROW]"i"(valid_row), \ [COL]"i"(tile_shape::Cols), \ [GmStride]"r"(src.GetStrideBytes(3)), \ [Init]"i"(static_cast(tile_shape::INIT)), \ @@ -1963,8 +1963,8 @@ void TLOAD(tile_shape &dst, gm_shape &src) { } else { asm volatile( "BSTART.TLSU TLOAD, %D[SrcType]\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[COL], ->lb2\n" "B.IOT mask=1111, last, ->%[d0]<%Z[TileSize]>\n" "B.IOR [%[s0],%[GmStride]], []\n" @@ -1972,7 +1972,7 @@ void TLOAD(tile_shape &dst, gm_shape &src) { : [s0]"r"(src.data()), [SrcType]"i"(type_traits::TypeCode), [TileSize]"i"(tile_type_traits::TilesizeCode), - [VCOL]"ri"(valid_col), [VROW]"ri"(valid_row), + [VCOL]"i"(valid_col), [VROW]"i"(valid_row), [COL]"i"(tile_shape::Cols), [GmStride]"r"(src.GetStrideBytes(3)) : "memory"); @@ -1995,8 +1995,8 @@ PTO_SHARED_INLINE SharedTile TLOAD(const gm_shape &src) { const size_t valid_row = result.GetValidRow(); asm volatile( "BSTART.TLSU TLOAD, %D[SrcType]\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[COL], ->lb2\n" PTO_PE_MASK_ASM("B.IOS mask=", ", ->%S[Shared]<%Z[TileSize]>\n") "B.IOR [%[s0],%[GmStride]], []\n" @@ -2005,7 +2005,7 @@ PTO_SHARED_INLINE SharedTile TLOAD(const gm_shape &src) { [PEMask]"i"(PEMask), [SrcType]"i"(type_traits::TypeCode), [TileSize]"i"(tile_type_traits::TilesizeCode), - [VCOL]"ri"(valid_col), [VROW]"ri"(valid_row), + [VCOL]"i"(valid_col), [VROW]"i"(valid_row), [COL]"i"(shp::Cols), [GmStride]"r"(src.GetStrideBytes(3)) : "memory"); @@ -2024,8 +2024,8 @@ PTO_SHARED_INLINE void TLOAD(SharedTile &dst, const gm_shape &src) { const size_t valid_row = dst.GetValidRow(); asm volatile( "BSTART.TLSU TLOAD, %D[SrcType]\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[COL], ->lb2\n" PTO_PE_MASK_ASM("B.IOS mask=", ", ->%S[Shared]<%Z[TileSize]>\n") "B.IOR [%[s0],%[GmStride]], []\n" @@ -2034,7 +2034,7 @@ PTO_SHARED_INLINE void TLOAD(SharedTile &dst, const gm_shape &src) { [PEMask]"i"(PEMask), [SrcType]"i"(type_traits::TypeCode), [TileSize]"i"(tile_type_traits::TilesizeCode), - [VCOL]"ri"(valid_col), [VROW]"ri"(valid_row), + [VCOL]"i"(valid_col), [VROW]"i"(valid_row), [COL]"i"(shp::Cols), [GmStride]"r"(src.GetStrideBytes(3)) : "memory"); @@ -2062,8 +2062,8 @@ void TSTORE(gm_shape &dst, tile_shape &src) { static_cast(src.GetRangeBase()); asm volatile( "BSTART.TLSU TSTORE, %D[SrcType]\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[COL], ->lb2\n" "B.IOS %S[s0], mask=1111\n" "B.SUBVIEW %c[SrcSelect], %[RegSrc], %c[Off], %c[SubSize]\n" @@ -2071,7 +2071,7 @@ void TSTORE(gm_shape &dst, tile_shape &src) { : : [d0]"r"(dst.data()), [s0]"Sr"(src.handle()), [SrcType]"i"(type_traits::TypeCode), - [VCOL]"ri"(valid_col), [VROW]"ri"(valid_row), + [VCOL]"i"(valid_col), [VROW]"i"(valid_row), [COL]"i"(ParentTile::Cols), [GmStride]"r"(dst.GetStrideBytes(3)), [SrcSelect]"i"(0), [RegSrc]"r"(range_base), @@ -2085,8 +2085,8 @@ void TSTORE(gm_shape &dst, tile_shape &src) { static_cast(src.GetRangeBase()); \ asm volatile( \ "BSTART.TLSU TSTORE, %D[SrcType]\n" \ - "B.DIM %[VCOL], 0, ->lb0\n" \ - "B.DIM %[VROW], 0, ->lb1\n" \ + "B.DIM zero, %c[VCOL], ->lb0\n" \ + "B.DIM zero, %c[VROW], ->lb1\n" \ "B.DIM zero, %c[COL], ->lb2\n" \ "B.IOS %S[s0], mask=1111\n" \ "B.SUBVIEW %c[SrcSelect], r" #N ", %c[Off], %c[SubSize]\n" \ @@ -2094,7 +2094,7 @@ void TSTORE(gm_shape &dst, tile_shape &src) { : \ : [d0]"r"(dst.data()), [s0]"Sr"(src.handle()), \ [SrcType]"i"(type_traits::TypeCode), \ - [VCOL]"ri"(valid_col), [VROW]"ri"(valid_row), \ + [VCOL]"i"(valid_col), [VROW]"i"(valid_row), \ [COL]"i"(ParentTile::Cols), \ [GmStride]"r"(dst.GetStrideBytes(3)), \ [SrcSelect]"i"(0), [RegSrc]"r"(range_base), \ @@ -2124,8 +2124,8 @@ void TSTORE(gm_shape &dst, tile_shape &src) { static_cast(src.GetRangeBase()); asm volatile( "BSTART.TLSU TSTORE, %D[SrcType]\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[COL], ->lb2\n" "B.IOT %[s0], mask=1111, last\n" "B.SUBVIEW %c[SrcSelect], %[RegSrc], %c[Off], %c[SubSize]\n" @@ -2133,7 +2133,7 @@ void TSTORE(gm_shape &dst, tile_shape &src) { : : [d0]"r"(dst.data()), [s0]"Tr"(src.data()), [SrcType]"i"(type_traits::TypeCode), - [VCOL]"ri"(valid_col), [VROW]"ri"(valid_row), + [VCOL]"i"(valid_col), [VROW]"i"(valid_row), [COL]"i"(tile_shape::Cols), [GmStride]"r"(dst.GetStrideBytes(3)), [SrcSelect]"i"(0), [RegSrc]"r"(range_base), @@ -2147,8 +2147,8 @@ void TSTORE(gm_shape &dst, tile_shape &src) { static_cast(src.GetRangeBase()); \ asm volatile( \ "BSTART.TLSU TSTORE, %D[SrcType]\n" \ - "B.DIM %[VCOL], 0, ->lb0\n" \ - "B.DIM %[VROW], 0, ->lb1\n" \ + "B.DIM zero, %c[VCOL], ->lb0\n" \ + "B.DIM zero, %c[VROW], ->lb1\n" \ "B.DIM zero, %c[COL], ->lb2\n" \ "B.IOT %[s0], mask=1111, last\n" \ "B.SUBVIEW %c[SrcSelect], r" #N ", %c[Off], %c[SubSize]\n" \ @@ -2156,7 +2156,7 @@ void TSTORE(gm_shape &dst, tile_shape &src) { : \ : [d0]"r"(dst.data()), [s0]"Tr"(src.data()), \ [SrcType]"i"(type_traits::TypeCode), \ - [VCOL]"ri"(valid_col), [VROW]"ri"(valid_row), \ + [VCOL]"i"(valid_col), [VROW]"i"(valid_row), \ [COL]"i"(tile_shape::Cols), \ [GmStride]"r"(dst.GetStrideBytes(3)), \ [SrcSelect]"i"(0), \ @@ -2183,15 +2183,15 @@ void TSTORE(gm_shape &dst, tile_shape &src) { } else { asm volatile( "BSTART.TLSU TSTORE, %D[SrcType]\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[COL], ->lb2\n" "B.IOT %[s0], mask=1111, last\n" "B.IOR [%[d0],%[GmStride]], []\n" : : [d0]"r"(dst.data()), [s0]"Tr"(src.data()), [SrcType]"i"(type_traits::TypeCode), - [VCOL]"ri"(valid_col), [VROW]"ri"(valid_row), + [VCOL]"i"(valid_col), [VROW]"i"(valid_row), [COL]"i"(tile_shape::Cols), [GmStride]"r"(dst.GetStrideBytes(3)) : "memory"); @@ -2221,8 +2221,8 @@ void TLOAD_CUBE(cube_shape &dst, gm_shape &src) { asm volatile( "BSTART.TLSU TLOAD, %D[DataType]\n" "B.DATR ND2M32.normal, Zero\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.IOT mask=1111, last, ->%[Dst]<%Z[SizeCode]>\n" "B.IOR [%[Base],%[RowStrideBytes]], []\n" : [Dst] "=Tr"(dst.data()) @@ -2236,8 +2236,8 @@ void TLOAD_CUBE(cube_shape &dst, gm_shape &src) { asm volatile( "BSTART.TLSU TLOAD, %D[DataType]\n" "B.DATR ND2M16.normal, Zero\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.IOT mask=1111, last, ->%[Dst]<%Z[SizeCode]>\n" "B.IOR [%[Base],%[RowStrideBytes]], []\n" : [Dst] "=Tr"(dst.data()) @@ -2251,8 +2251,8 @@ void TLOAD_CUBE(cube_shape &dst, gm_shape &src) { asm volatile( "BSTART.TLSU TLOAD, %D[DataType]\n" "B.DATR ND2N8.normal, Zero\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.IOT mask=1111, last, ->%[Dst]<%Z[SizeCode]>\n" "B.IOR [%[Base],%[RowStrideBytes]], []\n" : [Dst] "=Tr"(dst.data()) @@ -2285,8 +2285,8 @@ void TSTORE_CUBE(gm_shape &dst, const cube_shape &src) { asm volatile( "BSTART.TLSU TSTORE, %D[DataType]\n" "B.DATR M322ND.normal, Null\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.IOT %[Src], mask=1111, last\n" "B.IOR [%[Base],%[RowStrideBytes]], []\n" : @@ -2299,8 +2299,8 @@ void TSTORE_CUBE(gm_shape &dst, const cube_shape &src) { asm volatile( "BSTART.TLSU TSTORE, %D[DataType]\n" "B.DATR M162ND.normal, Null\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.IOT %[Src], mask=1111, last\n" "B.IOR [%[Base],%[RowStrideBytes]], []\n" : @@ -2313,8 +2313,8 @@ void TSTORE_CUBE(gm_shape &dst, const cube_shape &src) { asm volatile( "BSTART.TLSU TSTORE, %D[DataType]\n" "B.DATR N82ND.normal, Null\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.IOT %[Src], mask=1111, last\n" "B.IOR [%[Base],%[RowStrideBytes]], []\n" : @@ -2360,8 +2360,8 @@ PTO_SHARED_INLINE void TSTORE(gm_shape &dst, const SharedTileT &src) { const size_t valid_row = src.GetValidRow(); asm volatile( "BSTART.TLSU TSTORE, %D[SrcType]\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[COL], ->lb2\n" "B.IOS %S[Shared], mask=1111\n" "B.IOR [%[d0],%[GmStride]], []\n" @@ -2394,8 +2394,8 @@ PTO_SHARED_INLINE void TSTORE_PART(gm_shape &dst, const SharedTileT &src) { const size_t valid_row = src.GetValidRow(); asm volatile( "BSTART.TLSU TSTORE.SPART, %D[SrcType]\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[COL], ->lb2\n" PTO_PE_MASK_ASM("B.IOS %S[Shared], mask=", "\n") "B.IOR [%[d0],%[GmStride]], []\n" @@ -2422,8 +2422,8 @@ void TPREFETCH(const gm_shape &src, uint32_t valid_col, uint32_t valid_row) { gm_shape::Cols == DYNAMIC ? rowStride : gm_shape::Cols; asm volatile( "BSTART.TLSU TPREFETCH, %D[DataType]\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM %[Col], 0, ->lb2\n" "B.IOR [%[Base], %[Stride]], []\n" : @@ -2477,8 +2477,8 @@ void MGATHER_CAS(DstTile &observedOld, uint64_t base, "MGATHER_CAS tiles must match the resolved ValidRow x ValidCol"); asm volatile( "BSTART.TLSU MGATHER.CAS, %D[DataType]\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[Col], ->lb2\n" "B.IOT %[Idx], %[Exp], mask=1111\n" "B.IOT %[Rep], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" @@ -7078,8 +7078,8 @@ void TADD(tile_shape &dst, tile_shape &src0, tile_shape &src1) { } else { asm volatile( "BSTART.TEPL 0, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -7118,8 +7118,8 @@ void TSUB(tile_shape &dst, tile_shape &src0, tile_shape &src1) { } else { asm volatile( "BSTART.TEPL 1, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -7160,8 +7160,8 @@ void TMUL(tile_shape &dst, tile_shape &src0, tile_shape &src1) { const size_t valid_row = src0.GetValidRow(); asm volatile( "BSTART.TEPL 2, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -7200,8 +7200,8 @@ void TDIV(tile_shape &dst, tile_shape &src0, tile_shape &src1) { } else { asm volatile( "BSTART.TEPL 3, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -7240,8 +7240,8 @@ void TREM(tile_shape &dst, tile_shape &src0, tile_shape &src1) { } else { asm volatile( "BSTART.TEPL 4, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -7288,8 +7288,8 @@ void TAND(tile_shape &dst, tile_shape &src0, tile_shape &src1) { } else { asm volatile( "BSTART.TEPL 6, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -7328,8 +7328,8 @@ void TOR(tile_shape &dst, tile_shape &src0, tile_shape &src1) { } else { asm volatile( "BSTART.TEPL 7, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -7368,8 +7368,8 @@ void TXOR(tile_shape &dst, tile_shape &src0, tile_shape &src1) { } else { asm volatile( "BSTART.TEPL 8, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -7408,8 +7408,8 @@ void TSHL(tile_shape &dst, tile_shape &src0, tile_shape &src1) { } else { asm volatile( "BSTART.TEPL 9, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -7448,8 +7448,8 @@ void TSHR(tile_shape &dst, tile_shape &src0, tile_shape &src1) { } else { asm volatile( "BSTART.TEPL 10, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -7488,8 +7488,8 @@ void TMAX(tile_shape &dst, tile_shape &src0, tile_shape &src1) { } else { asm volatile( "BSTART.TEPL 11, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -7528,8 +7528,8 @@ void TMIN(tile_shape &dst, tile_shape &src0, tile_shape &src1) { } else { asm volatile( "BSTART.TEPL 12, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -7559,8 +7559,8 @@ void TCMP(tile_shape_out &dst, tile_shape_in &src0, tile_shape_in &src1) { asm volatile( "BSTART.TEPL 13, %D[TCode]\n" "B.DATR Zero, EQ\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[Cols], ->lb2\n" "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" "" @@ -7577,8 +7577,8 @@ void TCMP(tile_shape_out &dst, tile_shape_in &src0, tile_shape_in &src1) { asm volatile( "BSTART.TEPL 13, %D[TCode]\n" "B.DATR Zero, NE\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[Cols], ->lb2\n" "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" "" @@ -7595,8 +7595,8 @@ void TCMP(tile_shape_out &dst, tile_shape_in &src0, tile_shape_in &src1) { asm volatile( "BSTART.TEPL 13, %D[TCode]\n" "B.DATR Zero, LT\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[Cols], ->lb2\n" "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" "" @@ -7613,8 +7613,8 @@ void TCMP(tile_shape_out &dst, tile_shape_in &src0, tile_shape_in &src1) { asm volatile( "BSTART.TEPL 13, %D[TCode]\n" "B.DATR Zero, GT\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[Cols], ->lb2\n" "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" "" @@ -7631,8 +7631,8 @@ void TCMP(tile_shape_out &dst, tile_shape_in &src0, tile_shape_in &src1) { asm volatile( "BSTART.TEPL 13, %D[TCode]\n" "B.DATR Zero, LE\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[Cols], ->lb2\n" "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" "" @@ -7649,8 +7649,8 @@ void TCMP(tile_shape_out &dst, tile_shape_in &src0, tile_shape_in &src1) { asm volatile( "BSTART.TEPL 13, %D[TCode]\n" "B.DATR Zero, GE\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[Cols], ->lb2\n" "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" "" @@ -7756,15 +7756,15 @@ void TABS(tile_shape_out &dst, const tile_shape_in &src) { } else { asm volatile( "BSTART.TEPL 15, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) @@ -7794,15 +7794,15 @@ void TNOT(tile_shape &dst, tile_shape &src) { } else { asm volatile( "BSTART.TEPL 16, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) @@ -7832,15 +7832,15 @@ void TNEG(tile_shape &dst, tile_shape &src) { } else { asm volatile( "BSTART.TEPL 17, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) @@ -7870,15 +7870,15 @@ void TEXP(tile_shape &dst, tile_shape &src) { } else { asm volatile( "BSTART.TEPL 18, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) @@ -7908,15 +7908,15 @@ void TLOG(tile_shape &dst, tile_shape &src) { } else { asm volatile( "BSTART.TEPL 19, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) @@ -7931,8 +7931,8 @@ void TRECIP(tile_shape &dst, tile_shape &src) { const size_t valid_row = src.GetValidRow(); asm volatile( "BSTART.TEPL 20, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" @@ -7968,15 +7968,15 @@ void TSQRT(tile_shape &dst, tile_shape &src) { } else { asm volatile( "BSTART.TEPL 21, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) @@ -8006,15 +8006,15 @@ void TRSQRT(tile_shape &dst, tile_shape &src) { } else { asm volatile( "BSTART.TEPL 22, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) @@ -8044,15 +8044,15 @@ void TRELU(tile_shape &dst, tile_shape &src) { } else { asm volatile( "BSTART.TEPL 23, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) @@ -8095,8 +8095,8 @@ void TADDS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { const size_t valid_row = src.GetValidRow(); asm volatile( "BSTART.TEPL 32, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7],[]\n" @@ -8142,16 +8142,16 @@ void TSUBS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { volatile typename tile_shape::DType sv = s; asm volatile( "BSTART.TEPL 33, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode), @@ -8170,8 +8170,8 @@ void TMULS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { const size_t valid_row = src.GetValidRow(); asm volatile( "BSTART.TEPL 34, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7],[]\n" @@ -8217,16 +8217,16 @@ void TDIVS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { volatile typename tile_shape::DType sv = s; asm volatile( "BSTART.TEPL 35, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode), @@ -8265,16 +8265,16 @@ void TREMS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { volatile typename tile_shape::DType sv = s; asm volatile( "BSTART.TEPL 36, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode), @@ -8321,16 +8321,16 @@ void TANDS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { volatile typename tile_shape::DType sv = s; asm volatile( "BSTART.TEPL 38, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode), @@ -8369,16 +8369,16 @@ void TORS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { volatile typename tile_shape::DType sv = s; asm volatile( "BSTART.TEPL 39, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode), @@ -8417,16 +8417,16 @@ void TXORS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { volatile typename tile_shape::DType sv = s; asm volatile( "BSTART.TEPL 40, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode), @@ -8465,16 +8465,16 @@ void TSHLS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { volatile typename tile_shape::DType sv = s; asm volatile( "BSTART.TEPL 41, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode), @@ -8513,16 +8513,16 @@ void TSHRS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { volatile typename tile_shape::DType sv = s; asm volatile( "BSTART.TEPL 42, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode), @@ -8561,16 +8561,16 @@ void TMAXS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { volatile typename tile_shape::DType sv = s; asm volatile( "BSTART.TEPL 43, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode), @@ -8609,16 +8609,16 @@ void TMINS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { volatile typename tile_shape::DType sv = s; asm volatile( "BSTART.TEPL 44, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode), @@ -8645,8 +8645,8 @@ void TCMPS(tile_shape_out &dst, tile_shape_in &src, asm volatile( "BSTART.TEPL 45, %D[TCode]\n" "B.DATR Zero, EQ\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[Cols], ->lb2\n" "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" "B.IOR [%[Scalar]],[]\n" @@ -8664,8 +8664,8 @@ void TCMPS(tile_shape_out &dst, tile_shape_in &src, asm volatile( "BSTART.TEPL 45, %D[TCode]\n" "B.DATR Zero, NE\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[Cols], ->lb2\n" "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" "B.IOR [%[Scalar]],[]\n" @@ -8683,8 +8683,8 @@ void TCMPS(tile_shape_out &dst, tile_shape_in &src, asm volatile( "BSTART.TEPL 45, %D[TCode]\n" "B.DATR Zero, LT\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[Cols], ->lb2\n" "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" "B.IOR [%[Scalar]],[]\n" @@ -8702,8 +8702,8 @@ void TCMPS(tile_shape_out &dst, tile_shape_in &src, asm volatile( "BSTART.TEPL 45, %D[TCode]\n" "B.DATR Zero, GT\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[Cols], ->lb2\n" "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" "B.IOR [%[Scalar]],[]\n" @@ -8721,8 +8721,8 @@ void TCMPS(tile_shape_out &dst, tile_shape_in &src, asm volatile( "BSTART.TEPL 45, %D[TCode]\n" "B.DATR Zero, LE\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[Cols], ->lb2\n" "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" "B.IOR [%[Scalar]],[]\n" @@ -8740,8 +8740,8 @@ void TCMPS(tile_shape_out &dst, tile_shape_in &src, asm volatile( "BSTART.TEPL 45, %D[TCode]\n" "B.DATR Zero, GE\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[Cols], ->lb2\n" "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" "B.IOR [%[Scalar]],[]\n" @@ -8805,8 +8805,8 @@ void TSELS(tile_shape &dst, tile_shape &src0, typename tile_shape::DType s, tile volatile typename tile_shape::DType sv = s; asm volatile( "BSTART.TEPL 58, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "B.IOR [%8],[]\n" @@ -8852,8 +8852,8 @@ void TEXPANDS(tile_shape &dst, typename tile_shape::DType s) { volatile typename tile_shape::DType sv = s; asm volatile( "BSTART.TEPL 59, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT mask=1111, last, ->%0<%Z5>\n" "B.IOR [%6],[]\n" @@ -8897,8 +8897,8 @@ void TFMA(tile_shape &dst, tile_shape &src0, tile_shape &src1, tile_shape &src2) } else { asm volatile( "BSTART.TEPL 28, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111\n" "B.IOT %7, mask=1111, last, ->%0<%Z8>\n" @@ -8951,16 +8951,16 @@ void TEXTRACT(tile_shape_out &dst, tile_shape_in &src, int32_t indexRow, int32_t volatile int32_t icv = indexCol; asm volatile( "BSTART.TEPL 98, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7,%8],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode), @@ -9003,16 +9003,16 @@ void TINSERT(tile_shape_out &dst, tile_shape_in &src, int32_t indexRow, int32_t volatile int32_t icv = indexCol; asm volatile( "BSTART.TEPL 99, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7,%8],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode), @@ -9048,8 +9048,8 @@ void TIMG2COL(tile_shape_out &dst, gm_shape &src, TIMG2COLParams params) { asm volatile( "BSTART.TIMG2COL %D[DataType]\n" "B.DATR %c[Layout], DTYPE_NONE, Zero\n" - "B.DIM %[ValidCol], 0, ->lb0\n" - "B.DIM %[ValidRow], 0, ->lb1\n" + "B.DIM zero, %c[ValidCol], ->lb0\n" + "B.DIM zero, %c[ValidRow], ->lb1\n" "B.DIM zero, %c[TotalCol], ->lb2\n" "B.IOR [%[GMBase], zero, zero], []\n" "B.IOR [%[Param0], %[Param1], %[Param2]], []\n" @@ -9095,15 +9095,15 @@ void TFILLPAD(tile_shape_out &dst, tile_shape_in &src) { } else { asm volatile( "BSTART.TEPL 101, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) @@ -9138,7 +9138,7 @@ void TCI(tile_shape &dst, T s) { volatile uint32_t directionValue = descending; asm volatile( "BSTART.TEPL 102, %D[DataType]\n" - "B.DIM %[ValidCol], 0, ->lb0\n" + "B.DIM zero, %c[ValidCol], ->lb0\n" "B.DIM zero, %c[PhysicalCol], ->lb2\n" "B.IOR [%[Start],%[Direction]],[]\n" "B.IOT mask=1111, last, ->%[Dst]<%Z[TileSize]>\n" @@ -9173,8 +9173,8 @@ void TTRI(tile_shape &dst) { } else { asm volatile( "BSTART.TEPL 103, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT mask=1111, last, ->%0<%Z5>\n" "" @@ -9242,8 +9242,8 @@ void TQUANT(tile_shape_out &dst, tile_shape_in &src, float multiplier = 1.0f, // LLVM currently names encoded RMode zero RNONE. PTO ISA 0.58.3 defines // that encoding as the operation default, which is RNE for TQUANT. "B.DATR %D[__pto_DstType], RNONE, sat\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[Col], ->lb2\n" "B.IOR [%[Mult], %[ZP]], []\n" "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" @@ -9260,8 +9260,8 @@ void TQUANT(tile_shape_out &dst, tile_shape_in &src, float multiplier = 1.0f, asm volatile( "BSTART.TEPL 106, %D[SType]\n" "B.DATR %D[__pto_DstType], RNONE\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[Col], ->lb2\n" "B.IOR [%[Mult], %[ZP]], []\n" "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" @@ -9278,8 +9278,8 @@ void TQUANT(tile_shape_out &dst, tile_shape_in &src, float multiplier = 1.0f, asm volatile( "BSTART.TEPL 106, %D[SType]\n" PTO_RMODE_DATR_ASM(", sat\n") - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[Col], ->lb2\n" "B.IOR [%[Mult], %[ZP]], []\n" "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" @@ -9297,8 +9297,8 @@ void TQUANT(tile_shape_out &dst, tile_shape_in &src, float multiplier = 1.0f, asm volatile( "BSTART.TEPL 106, %D[SType]\n" PTO_RMODE_DATR_ASM("\n") - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[Col], ->lb2\n" "B.IOR [%[Mult], %[ZP]], []\n" "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" @@ -9355,8 +9355,8 @@ void TDEQUANT(tile_shape_out &dst, tile_shape_in &src, float multiplier = 1.0f, asm volatile( "BSTART.TEPL 107, %D[SType]\n" "B.DATR FP32, RNONE\n" - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[Col], ->lb2\n" "B.IOR [%[Mult], %[ZP]], []\n" "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" @@ -9374,8 +9374,8 @@ void TDEQUANT(tile_shape_out &dst, tile_shape_in &src, float multiplier = 1.0f, asm volatile( "BSTART.TEPL 107, %D[SType]\n" PTO_RMODE_DATR_ASM("\n") - "B.DIM %[VCOL], 0, ->lb0\n" - "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" "B.DIM zero, %c[Col], ->lb2\n" "B.IOR [%[Mult], %[ZP]], []\n" "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" @@ -9567,15 +9567,15 @@ void TTRANS(tile_shape_out &dst, tile_shape_in &src) { } else { asm volatile( "BSTART.TEPL 110, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) @@ -9607,8 +9607,8 @@ void TGATHER(tile_shape_out &dst, tile_shape_in &src, tile_shape_off &off) { } else { asm volatile( "BSTART.TEPL 111, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7],[]\n" @@ -9649,8 +9649,8 @@ void TSCATTER(tile_shape_out &dst, tile_shape_in &src, tile_shape_off &off) { } else { asm volatile( "BSTART.TEPL 112, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7],[]\n" @@ -9690,8 +9690,8 @@ void TPARTADD(tile_shape &dst, tile_shape &src0, tile_shape &src1) { } else { asm volatile( "BSTART.TEPL 113, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -9730,8 +9730,8 @@ void TPARTMUL(tile_shape &dst, tile_shape &src0, tile_shape &src1) { } else { asm volatile( "BSTART.TEPL 114, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -9770,8 +9770,8 @@ void TPARTMAX(tile_shape &dst, tile_shape &src0, tile_shape &src1) { } else { asm volatile( "BSTART.TEPL 115, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -9810,8 +9810,8 @@ void TPARTMIN(tile_shape &dst, tile_shape &src0, tile_shape &src1) { } else { asm volatile( "BSTART.TEPL 116, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -9860,8 +9860,8 @@ void TROWSUM(tile_shape_out &dst, tile_shape_in &src) { const size_t valid_row = src.GetValidRow(); asm volatile( "BSTART.TEPL 64, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" @@ -9905,15 +9905,15 @@ void TROWMAX(tile_shape_out &dst, tile_shape_in &src) { } else { asm volatile( "BSTART.TEPL 65, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) @@ -9950,15 +9950,15 @@ void TROWMIN(tile_shape_out &dst, tile_shape_in &src) { } else { asm volatile( "BSTART.TEPL 66, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) @@ -9995,15 +9995,15 @@ void TROWPROD(tile_shape_out &dst, tile_shape_in &src) { } else { asm volatile( "BSTART.TEPL 67, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) @@ -10037,8 +10037,8 @@ void TROWEXPAND(tile_shape_out &dst, tile_shape_in &src) { } else { asm volatile( "BSTART.TEPL 68, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" @@ -10082,15 +10082,15 @@ void TROWARGMAX(tile_shape_out &dst, tile_shape_in &src) { } else { asm volatile( "BSTART.TEPL 76, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) @@ -10127,15 +10127,15 @@ void TROWARGMIN(tile_shape_out &dst, tile_shape_in &src) { } else { asm volatile( "BSTART.TEPL 77, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src.GetValidCol()), - "ri"(src.GetValidRow()), + "i"(src.GetValidCol()), + "i"(src.GetValidRow()), "i"(tile_shape_in::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) @@ -10173,8 +10173,8 @@ void TCOLSUM(tile_shape_out &dst, tile_shape_in &src) { } else { asm volatile( "BSTART.TEPL 80, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" @@ -10219,8 +10219,8 @@ void TCOLMAX(tile_shape_out &dst, tile_shape_in &src) { } else { asm volatile( "BSTART.TEPL 81, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" @@ -10265,8 +10265,8 @@ void TCOLMIN(tile_shape_out &dst, tile_shape_in &src) { } else { asm volatile( "BSTART.TEPL 82, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" @@ -10311,8 +10311,8 @@ void TCOLPROD(tile_shape_out &dst, tile_shape_in &src) { } else { asm volatile( "BSTART.TEPL 83, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" @@ -10353,8 +10353,8 @@ void TCOLEXPAND(tile_shape_out &dst, tile_shape_in &src) { } else { asm volatile( "BSTART.TEPL 84, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" @@ -10399,8 +10399,8 @@ void TCOLARGMAX(tile_shape_out &dst, tile_shape_in &src) { } else { asm volatile( "BSTART.TEPL 92, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" @@ -10445,8 +10445,8 @@ void TCOLARGMIN(tile_shape_out &dst, tile_shape_in &src) { } else { asm volatile( "BSTART.TEPL 93, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" @@ -10503,8 +10503,8 @@ void TROWEXPANDADD(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr "TROWEXPANDADD: src0/dst dtype must match"); asm volatile( "BSTART.TEPL 69, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -10562,8 +10562,8 @@ void TROWEXPANDSUB(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr "TROWEXPANDSUB: src0/dst dtype must match"); asm volatile( "BSTART.TEPL 70, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -10623,8 +10623,8 @@ void TROWEXPANDMUL(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr const size_t valid_row = src0.GetValidRow(); asm volatile( "BSTART.TEPL 71, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -10682,8 +10682,8 @@ void TROWEXPANDDIV(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr "TROWEXPANDDIV: src0/dst dtype must match"); asm volatile( "BSTART.TEPL 72, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -10741,8 +10741,8 @@ void TROWEXPANDMAX(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr "TROWEXPANDMAX: src0/dst dtype must match"); asm volatile( "BSTART.TEPL 73, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -10800,8 +10800,8 @@ void TROWEXPANDMIN(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr "TROWEXPANDMIN: src0/dst dtype must match"); asm volatile( "BSTART.TEPL 74, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -10859,8 +10859,8 @@ void TROWEXPANDEXPDIF(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 "TROWEXPANDEXPDIF: src0/dst dtype must match"); asm volatile( "BSTART.TEPL 75, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -10918,8 +10918,8 @@ void TCOLEXPANDADD(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr "TCOLEXPANDADD: src0/dst dtype must match"); asm volatile( "BSTART.TEPL 85, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -10977,8 +10977,8 @@ void TCOLEXPANDSUB(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr "TCOLEXPANDSUB: src0/dst dtype must match"); asm volatile( "BSTART.TEPL 86, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -11036,8 +11036,8 @@ void TCOLEXPANDMUL(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr "TCOLEXPANDMUL: src0/dst dtype must match"); asm volatile( "BSTART.TEPL 87, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -11095,8 +11095,8 @@ void TCOLEXPANDDIV(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr "TCOLEXPANDDIV: src0/dst dtype must match"); asm volatile( "BSTART.TEPL 88, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -11154,8 +11154,8 @@ void TCOLEXPANDMAX(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr "TCOLEXPANDMAX: src0/dst dtype must match"); asm volatile( "BSTART.TEPL 89, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -11213,8 +11213,8 @@ void TCOLEXPANDMIN(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr "TCOLEXPANDMIN: src0/dst dtype must match"); asm volatile( "BSTART.TEPL 90, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -11272,8 +11272,8 @@ void TCOLEXPANDEXPDIF(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 "TCOLEXPANDEXPDIF: src0/dst dtype must match"); asm volatile( "BSTART.TEPL 91, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" @@ -11343,8 +11343,8 @@ void TCONCAT(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { "TCONCAT: src0/dst row count must match"); asm volatile( "BSTART.TEPL 96, %D1\n" - "B.DIM %2, 0, ->lb0\n" - "B.DIM %3, 0, ->lb1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" diff --git a/test/test_static_valid_shape_lowering.py b/test/test_static_valid_shape_lowering.py new file mode 100644 index 0000000..9f98627 --- /dev/null +++ b/test/test_static_valid_shape_lowering.py @@ -0,0 +1,37 @@ +import subprocess +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +HEADER = (ROOT / "include/jcore/template_asm.hpp").read_text(encoding="utf-8") +FIXTURE = (ROOT / "test/tileop_api/src/ValidShapeImmediate.cpp").read_text(encoding="utf-8") +LLVM_MC = Path("/home/zhuwei/linx-llvm/build/bin/llvm-mc") + +class TestStaticValidShapeLowering(unittest.TestCase): + def test_shared_tload_uses_zero_in_dim_first_operand(self) -> None: + self.assertIn('"B.DIM zero, %c[VCOL], ->lb0\\n"', HEADER) + self.assertIn('"B.DIM zero, %c[VROW], ->lb1\\n"', HEADER) + self.assertIn('[VCOL]"i"(valid_col)', HEADER) + self.assertIn('[VROW]"i"(valid_row)', HEADER) + self.assertNotIn('[VCOL]"ri"(valid_col)', HEADER) + self.assertNotIn('[VROW]"ri"(valid_row)', HEADER) + + def test_shared_tload_keeps_c_b_dimi_compression_coverage(self) -> None: + self.assertIn('"B.DIM zero, %c[COL], ->lb2\\n"', HEADER) + self.assertTrue(LLVM_MC.exists()) + result = subprocess.run( + [str(LLVM_MC), "-triple=linx64v5", "-show-encoding"], + input=b"B.DIM zero, 16, ->lb0\n", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True, + ) + self.assertIn(b"C.B.DIMI", result.stdout) + + def test_fixture_declares_static_and_dynamic_tiles(self) -> None: + self.assertIn('using S = Tile;', FIXTURE) + self.assertIn('using D = Tile;', FIXTURE) + self.assertIn('TLOAD(dst, src);', FIXTURE) + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_v058_engine_contract.py b/test/test_v058_engine_contract.py index 6029629..84699ad 100644 --- a/test/test_v058_engine_contract.py +++ b/test/test_v058_engine_contract.py @@ -147,8 +147,8 @@ def test_tcvt_emits_dimensions_before_terminating_iot(self) -> None: for instruction in ( '"BSTART.TEPL 27, %D1\\n"', '"B.DATR %D2, RNONE\\n"', - '"B.DIM %5, 0, ->lb0\\n"', - '"B.DIM %6, 0, ->lb1\\n"', + '"B.DIM zero, %c5, ->lb0\\n"', + '"B.DIM zero, %c6, ->lb1\\n"', '"B.DIM zero, %c7, ->lb2\\n"', '"B.IOT %3, mask=1111, last, ->%0<%Z4>\\n"'): self.assertIn(instruction, ordinary_branch) @@ -265,10 +265,10 @@ def test_valid_shape_immediate_fixture_separates_static_and_dynamic_paths(self) fixture = (ROOT / "test" / "tileop_api" / "src" / "ValidShapeImmediate.cpp").read_text( encoding="utf-8" ) - self.assertIn('"i" + C.B.DIMI path', fixture) - self.assertIn('"r" + B.DIM path', fixture) self.assertIn('using S = Tile;', fixture) self.assertIn('using D = Tile;', fixture) + self.assertIn('static void static_path(SR &dst, GM &src)', fixture) + self.assertIn('static void dynamic_path(DR &dst, GM &src)', fixture) def test_static_valid_shape_bindings_remain_immediate_eligible(self) -> None: header = self.header diff --git a/test/tileop_api/src/ValidShapeImmediate.cpp b/test/tileop_api/src/ValidShapeImmediate.cpp index e368693..1f935ff 100644 --- a/test/tileop_api/src/ValidShapeImmediate.cpp +++ b/test/tileop_api/src/ValidShapeImmediate.cpp @@ -1,136 +1,17 @@ -// Regression coverage for static/dynamic valid-shape immediate lowering. -// -// The 2026-09-01 valid-shape batch converted ordinary TileOP inline asm so -// that a statically-typed Tile (ValidRow/ValidCol > 0) binds LB0/LB1 as "i" -// immediates (C.B.DIMI eligible), while dynamic Tiles (ValidRow/ValidCol == -// -1) keep the "r" register form. This fixture instantiates every converted -// family on both path kinds so template-instantiation errors, wrong-template -// name typos (e.g. the original TCONCAT "tile_shape" bug), and operand -// misbindings are caught by the compile gate. -// -// It is a compile-only regression: main() is minimal. The objective gate is -// `make TESTCASE=ValidShapeImmediate object` under the matching Linx -// toolchain; locally it must at least pass -fsyntax-only on both paths. +#include -#include - -using namespace pto; - -// Static half: a fully-static Tile drives the "i" + C.B.DIMI path. using S = Tile; -using SR = Tile; // row-reduce dst / row-expand source -using SC = Tile; // col-reduce dst / col-expand source -using SE = Tile; // expand destination -using SO = Tile; +using D = Tile; +using SR = SharedTile; +using DR = SharedTile; +using GM = Global; -__attribute__((noinline)) void static_path(S &d, S &a, S &b, S &c, - SR &r, SC &rc, SE &ex, - SO &off, float v) { - // elementwise batch - TADD(d, a, b); - TSUB(d, a, b); - TMUL(d, a, b); - TDIV(d, a, b); - TREM(d, a, b); - TAND(d, a, b); - TOR(d, a, b); - TXOR(d, a, b); - TSHL(d, a, b); - TSHR(d, a, b); - TMAX(d, a, b); - TMIN(d, a, b); - // reduce / broadcast batch - TROWSUM(r, a); - TROWMAX(r, a); - TROWMIN(r, a); - TROWARGMAX(r, a); - TROWEXPAND(ex, r); - TCOLSUM(rc, a); - TCOLEXPAND(ex, rc); - TROWEXPANDADD(ex, d, r); - TROWEXPANDSUB(ex, d, r); - TROWEXPANDMUL(ex, d, r); - // unary / scalar batch - TABS(d, a); - TNOT(d, a); - TNEG(d, a); - TEXP(d, a); - TLOG(d, a); - TSQRT(d, a); - TRSQRT(d, a); - TRELU(d, a); - TSUBS(d, a, v); - TDIVS(d, a, v); - TREMS(d, a, v); - TANDS(d, a, v); - TORS(d, a, v); - TXORS(d, a, v); - TSHLS(d, a, v); - TSHRS(d, a, v); - TMAXS(d, a, v); - TMINS(d, a, v); - TEXPANDS(d, v); - TFMA(d, a, b, c); - // part / gather batch - TSEL(d, a, b); - TPARTADD(d, a, b); - TPARTMUL(d, a, b); - TPARTMAX(d, a, b); - TPARTMIN(d, a, b); - TGATHER(d, a, off); - TSCATTER(a, d, off); - // movement / concat batch (TCONCAT regression: dst-derived valid shape) - TEXTRACT(c, a, 1, 2); - TINSERT(c, a, 1, 2); - TTRANS(c, a); - // TCONCAT requires src0/src1/dst to share dtype and row count; the valid - // shape used by its B.DIM is dst-derived (tile_shape_out), which regresses - // the original "tile_shape" undeclared-identifier bug. - TCONCAT(d, a, b); +static void static_path(SR &dst, GM &src) { + TLOAD(dst, src); } -// Dynamic half: same families on dynamic Tiles drive the "r" + B.DIM path. -using D = Tile; -using DR = Tile; // row-reduce dst / row-expand source -using DC = Tile; // col-reduce dst / col-expand source -using DE = Tile; // expand destination -using DO = Tile; - -__attribute__((noinline)) void dynamic_path(D &d, D &a, D &b, D &c, - DR &r, DC &rc, DE &ex, - DO &off, float v) { - TADD(d, a, b); - TSUB(d, a, b); - TMUL(d, a, b); - TROWSUM(r, a); - TROWMAX(r, a); - TROWEXPAND(ex, r); - TROWEXPANDADD(ex, d, r); - TABS(d, a); - TEXP(d, a); - TSUBS(d, a, v); - TMAXS(d, a, v); - TSEL(d, a, b); - TPARTMUL(d, a, b); - TGATHER(d, a, off); - TSCATTER(a, d, off); - TEXTRACT(c, a, 1, 2); - TTRANS(c, a); - TCONCAT(d, a, b); +static void dynamic_path(DR &dst, GM &src) { + TLOAD(dst, src); } -int main() { - S s, sa, sb, sc; - SR sr; - SC src; - SE sex; - SO soff; - D d, da, db, dc; - DR dr; - DC drc; - DE dex; - DO doff; - static_path(s, sa, sb, sc, sr, src, sex, soff, 1.0f); - dynamic_path(d, da, db, dc, dr, drc, dex, doff, 1.0f); - return 0; -} \ No newline at end of file +int main() { return 0; } From 3c495c081760f65bd4becc4aabf4ca023a6a77a6 Mon Sep 17 00:00:00 2001 From: LinxISA Automation Date: Mon, 7 Sep 2026 23:36:40 +0800 Subject: [PATCH 2/3] TileOP: per-dimension static/dynamic B.DIM lowering Split every lb0/lb1 B.DIM site into four if-constexpr branches on ValidCol/ValidRow (SS/SD/DS/DD): - static dim -> B.DIM zero, %c[imm] (compresses to C.B.DIMI) - dynamic dim -> B.DIM %[reg], 0 lb2 always stays the immediate form (only TPREFETCH's dynamic-GM fallback keeps a runtime register). Also: - make TMATMUL N/K immediate-form (M stays a register: group_M is runtime for cooperative forms) - fix THISTOGRAM: missing space/PadValue in the B.DATR paste made the macro un-compilable; template now four-branch parameterized - fix TLOAD2/TLOAD4/TSTORE2 lb2 immediate-slot form - remove fa-56 legacy dead code: TMAX_T, TSUB_EXP_EXPAND_T, TMUL_ADD_ROWSUM_T, TADD_MUL_EXPAND_T, TMOV_DN2NZ_DYN and the DEFINE_TMOV_LAYOUT variants (no callers; VPAR raw-opcode spelling does not assemble) - fix pre-existing BLayout::ND2M16/32 -> LayoutCvtEnum in TIMG2COL --- include/jcore/template_asm.hpp | 10500 ++++++++++++++++++++++--------- 1 file changed, 7560 insertions(+), 2940 deletions(-) diff --git a/include/jcore/template_asm.hpp b/include/jcore/template_asm.hpp index 0f7fef9..e35c6d7 100644 --- a/include/jcore/template_asm.hpp +++ b/include/jcore/template_asm.hpp @@ -62,76 +62,6 @@ void ACCCVT_RMAX_SCALE_NZ2DN(tile_shape_max &, tile_shape_out &, "to the active TMATMUL operation with B.FPATR and its RowMax operands"); } -template -void TMAX_T(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { - asm volatile( - "BSTART.VPAR 0b0000100011, %c3\n" - "B.IOT %1, %2, mask=1111, last, ->%0<%Z4>\n" - "B.DIM zero, %c5, ->lb0\n" - "B.DIM zero, %c6, ->lb1\n" - - "" - : "=Tr"(dst.data()) - : "Tr"(src0.data()), "Tr"(src1.data()), \ - "i"(type_traits::TypeCode), - "i"(tile_type_traits::TilesizeCode), - "i"(src0.GetValidCol()), "i"(src0.GetValidRow()) - ); -} - -template -void TSUB_EXP_EXPAND_T(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { - asm volatile( - "BSTART.VPAR 0b0001000011, %c3\n" - "B.IOT %1, %2, mask=1111, last, ->%0<%Z4>\n" - "B.DIM zero, %c5, ->lb0\n" - "B.DIM zero, %c6, ->lb1\n" - - "" - : "=Tr"(dst.data()) - : "Tr"(src0.data()), "Tr"(src1.data()), \ - "i"(type_traits::TypeCode), - "i"(tile_type_traits::TilesizeCode), - "i"(src0.GetValidCol()), "i"(src0.GetValidRow()) - ); -} - -template -void TMUL_ADD_ROWSUM_T(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1, tile_shape_in2 &src2) { - asm volatile( - "BSTART.VPAR 0b0001100011, %c4\n" - "B.IOT %1, %2, mask=1111, 0, ->%0<%Z5>\n" - "B.IOT %3, mask=1111, last\n" - "B.DIM %6, 0, ->lb0\n" - "B.DIM zero, %c7, ->lb1\n" - - "" - : "=Tr"(dst.data()) - : "Tr"(src0.data()), "Tr"(src1.data()), "Tr"(src2.data()), - "i"(type_traits::TypeCode), - "i"(tile_type_traits::TilesizeCode), - "i"(src0.GetValidCol()), "i"(src0.GetValidRow()) - ); -} - -template -void TADD_MUL_EXPAND_T(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1, tile_shape_in2 &src2) { - asm volatile( - "BSTART.VPAR 0b0010000011, %c4\n" - "B.IOT %1, %2, mask=1111, 0, ->%0<%Z5>\n" - "B.IOT %3, mask=1111, last\n" - "B.DIM %6, 0, ->lb0\n" - "B.DIM zero, %c7, ->lb1\n" - - "" - : "=Tr"(dst.data()) - : "Tr"(src0.data()), "Tr"(src1.data()), "Tr"(src2.data()), - "i"(type_traits::TypeCode), - "i"(tile_type_traits::TilesizeCode), - "i"(src0.GetValidCol()), "i"(src0.GetValidRow()) - ); -} - template void TCVT_T(tile_shape_out &dst, tile_shape_in &src) { static_assert((tile_shape_out::ValidRow == DYNAMIC || @@ -163,10 +93,41 @@ void TCVT_T(tile_shape_out &dst, tile_shape_in &src) { static_assert(tile_shape_out::TilesizeCode >= __tilesize_128B && tile_shape_out::TilesizeCode <= __tilesize_64KB, "TCVT CUBE_M16/M32 destination TSize must be 128 B..64 KiB"); - asm volatile( + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 27, %D1\n" + "B.DATR %D2, RNONE\n" + "B.DIM zero, %c5, ->lb0\n" + "B.DIM zero, %c6, ->lb1\n" + "B.IOT %3, mask=1111, last, ->%0<%Z4>\n" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(type_traits::TypeCode), + "Tr"(src.data()), + "i"(tile_shape_out::TilesizeCode), + "i"(tile_shape_in::ValidCol), + "i"(tile_shape_in::ValidRow) + ); } + else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { +asm volatile( "BSTART.TEPL 27, %D1\n" "B.DATR %D2, RNONE\n" "B.DIM zero, %c5, ->lb0\n" + "B.DIM %[tcvt_row], 0, ->lb1\n" + "B.IOT %3, mask=1111, last, ->%0<%Z4>\n" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(type_traits::TypeCode), + "Tr"(src.data()), + "i"(tile_shape_out::TilesizeCode), + "i"(tile_shape_in::ValidCol), + [tcvt_row] "r"(src.GetValidRow()) + ); } + else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 27, %D1\n" + "B.DATR %D2, RNONE\n" + "B.DIM %[tcvt_col], 0, ->lb0\n" "B.DIM zero, %c6, ->lb1\n" "B.IOT %3, mask=1111, last, ->%0<%Z4>\n" : "=Tr"(dst.data()) @@ -174,9 +135,24 @@ void TCVT_T(tile_shape_out &dst, tile_shape_in &src) { "i"(type_traits::TypeCode), "Tr"(src.data()), "i"(tile_shape_out::TilesizeCode), - "ri"(valid_col), - "ri"(valid_row) - ); + [tcvt_col] "r"(src.GetValidCol()), + "i"(tile_shape_in::ValidRow) + ); } + else { +asm volatile( + "BSTART.TEPL 27, %D1\n" + "B.DATR %D2, RNONE\n" + "B.DIM %[tcvt_col], 0, ->lb0\n" + "B.DIM %[tcvt_row], 0, ->lb1\n" + "B.IOT %3, mask=1111, last, ->%0<%Z4>\n" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(type_traits::TypeCode), + "Tr"(src.data()), + "i"(tile_shape_out::TilesizeCode), + [tcvt_col] "r"(src.GetValidCol()), + [tcvt_row] "r"(src.GetValidRow()) + ); } } else { static_assert(!tile_shape_out::IsCubeLayout, "TCVT to a CUBE layout requires a CUBE_M16/M32 source"); @@ -188,10 +164,45 @@ void TCVT_T(tile_shape_out &dst, tile_shape_in &src) { tile_shape_out::ValidCol == tile_shape_in::ValidCol, "ordinary TCVT source and destination must have identical " "ValidRow/ValidCol"); - asm volatile( + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 27, %D1\n" + "B.DATR %D2, RNONE\n" + "B.DIM zero, %c5, ->lb0\n" + "B.DIM zero, %c6, ->lb1\n" + "B.DIM zero, %c7, ->lb2\n" + "B.IOT %3, mask=1111, last, ->%0<%Z4>\n" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(type_traits::TypeCode), + "Tr"(src.data()), + "i"(tile_shape_out::TilesizeCode), + "i"(tile_shape_in::ValidCol), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_out::Cols) + ); } + else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { +asm volatile( "BSTART.TEPL 27, %D1\n" "B.DATR %D2, RNONE\n" "B.DIM zero, %c5, ->lb0\n" + "B.DIM %[tcvt_row], 0, ->lb1\n" + "B.DIM zero, %c7, ->lb2\n" + "B.IOT %3, mask=1111, last, ->%0<%Z4>\n" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(type_traits::TypeCode), + "Tr"(src.data()), + "i"(tile_shape_out::TilesizeCode), + "i"(tile_shape_in::ValidCol), + [tcvt_row] "r"(src.GetValidRow()), + "i"(tile_shape_out::Cols) + ); } + else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 27, %D1\n" + "B.DATR %D2, RNONE\n" + "B.DIM %[tcvt_col], 0, ->lb0\n" "B.DIM zero, %c6, ->lb1\n" "B.DIM zero, %c7, ->lb2\n" "B.IOT %3, mask=1111, last, ->%0<%Z4>\n" @@ -200,40 +211,30 @@ void TCVT_T(tile_shape_out &dst, tile_shape_in &src) { "i"(type_traits::TypeCode), "Tr"(src.data()), "i"(tile_shape_out::TilesizeCode), - "ri"(valid_col), - "ri"(valid_row), + [tcvt_col] "r"(src.GetValidCol()), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_out::Cols) + ); } + else { +asm volatile( + "BSTART.TEPL 27, %D1\n" + "B.DATR %D2, RNONE\n" + "B.DIM %[tcvt_col], 0, ->lb0\n" + "B.DIM %[tcvt_row], 0, ->lb1\n" + "B.DIM zero, %c7, ->lb2\n" + "B.IOT %3, mask=1111, last, ->%0<%Z4>\n" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(type_traits::TypeCode), + "Tr"(src.data()), + "i"(tile_shape_out::TilesizeCode), + [tcvt_col] "r"(src.GetValidCol()), + [tcvt_row] "r"(src.GetValidRow()), "i"(tile_shape_out::Cols) - ); + ); } } } -#define DEFINE_TMOV_LAYOUT(LAYOUT_NAME) \ -template \ -void TMOV_##LAYOUT_NAME(tile_shape_out &dst, tile_shape_in &src) { \ - asm volatile( \ - "BSTART.TLSU TMOV, %D2\n" \ - "B.DATR " #LAYOUT_NAME ", DTYPE_NONE, Zero\n" \ - "B.IOT %1, mask=1111, last, ->%0<%Z3>\n" \ - "B.DIM zero, %c4, ->lb0\n" \ - "B.DIM zero, %c5, ->lb1\n" \ - : "=Tr"(dst.data()) \ - : "Tr"(src.data()), \ - "i"(type_traits::TypeCode), \ - "i"(tile_type_traits::TilesizeCode), \ - "i"(src.GetValidCol()), \ - "i"(src.GetValidRow()) \ - ); \ -} - -DEFINE_TMOV_LAYOUT(ND2NZ) -DEFINE_TMOV_LAYOUT(NZ2ND) -DEFINE_TMOV_LAYOUT(ND2ZN) -DEFINE_TMOV_LAYOUT(DN2ZN) -DEFINE_TMOV_LAYOUT(DN2NZ) -DEFINE_TMOV_LAYOUT(NZ2DN) -DEFINE_TMOV_LAYOUT(NZ2ZN) -DEFINE_TMOV_LAYOUT(ZN2NZ) -DEFINE_TMOV_LAYOUT(NORM) // PTO ISA 0.58 generic Local-to-Local TMOV(dst, src). Engine TLSU function 2. // Copies the payload and definedness from src to dst; this is not a dtype @@ -248,7 +249,8 @@ inline void TMOV(Tile &dst, const Tile &src) { "TMOV logical Tile size must be 128 B..256 KiB (SizeCode=1..12)"); const size_t valid_col = src.GetValidCol(); const size_t valid_row = src.GetValidRow(); - asm volatile( + if constexpr (Tile::ValidCol > 0 && Tile::ValidRow > 0) { +asm volatile( "BSTART.TLSU TMOV, %D[DataType]\n" "B.DIM zero, %c[ValidCol], ->lb0\n" "B.DIM zero, %c[ValidRow], ->lb1\n" @@ -258,65 +260,86 @@ inline void TMOV(Tile &dst, const Tile &src) { [DataType] "i"(type_traits::TypeCode), [TileSize] "i"( tile_type_traits::TilesizeCode), - [ValidCol] "ri"(valid_col), [ValidRow] "ri"(valid_row)); -} - -template -void TMOV_DN2NZ_DYN(tile_shape_out &dst, tile_shape_in &src) { - asm volatile( - "BSTART.TLSU TMOV, %D2\n" - "B.DATR DN2NZ, DTYPE_NONE, Zero\n" - "B.IOT %1, mask=1111, last, ->%0<%Z3>\n" - "B.DIM zero, %c4, ->lb0\n" - "B.DIM zero, %c5, ->lb1\n" - - : "=Tr"(dst.data()) - : "Tr"(src.data()), - "i"(type_traits::TypeCode), - "i"(tile_type_traits::TilesizeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()) - ); + [ValidCol] "i"(Tile::ValidCol), [ValidRow] "i"(Tile::ValidRow)); } + else if constexpr (Tile::ValidCol > 0 && Tile::ValidRow < 0) { +asm volatile( + "BSTART.TLSU TMOV, %D[DataType]\n" + "B.DIM zero, %c[ValidCol], ->lb0\n" + "B.DIM %[ValidRow], 0, ->lb1\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[TileSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [DataType] "i"(type_traits::TypeCode), + [TileSize] "i"( + tile_type_traits::TilesizeCode), + [ValidCol] "i"(Tile::ValidCol), [ValidRow] "r"(src.GetValidRow())); } + else if constexpr (Tile::ValidCol < 0 && Tile::ValidRow > 0) { +asm volatile( + "BSTART.TLSU TMOV, %D[DataType]\n" + "B.DIM %[ValidCol], 0, ->lb0\n" + "B.DIM zero, %c[ValidRow], ->lb1\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[TileSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [DataType] "i"(type_traits::TypeCode), + [TileSize] "i"( + tile_type_traits::TilesizeCode), + [ValidCol] "r"(src.GetValidCol()), [ValidRow] "i"(Tile::ValidRow)); } + else { +asm volatile( + "BSTART.TLSU TMOV, %D[DataType]\n" + "B.DIM %[ValidCol], 0, ->lb0\n" + "B.DIM %[ValidRow], 0, ->lb1\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[TileSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [DataType] "i"(type_traits::TypeCode), + [TileSize] "i"( + tile_type_traits::TilesizeCode), + [ValidCol] "r"(src.GetValidCol()), [ValidRow] "r"(src.GetValidRow())); } } template void THISTOGRAM(tile_shape_out &dst, tile_shape_in &src, tile_shape_in &Idx, int ByteId) { -#define THISTOGRAM_ASM(BYTE_NAME) \ +#define THISTOGRAM_ASM(BYTE_NAME, VCOL_TPL, VROW_TPL, VCOL_CON, VROW_CON) \ asm volatile( \ "BSTART.TEPL 104, %D1\n" \ - "B.DATR %D2," BYTE_NAME "\n" \ - "B.DIM %3, 0, ->LB0\n" \ - "B.DIM %4, 0, ->LB1\n" \ + "B.DATR %D2, " BYTE_NAME ", Null\n" \ + "B.DIM " VCOL_TPL ", ->LB0\n" \ + "B.DIM " VROW_TPL ", ->LB1\n" \ "B.DIM zero, %c5, ->LB2\n" \ "B.IOT %6, %7, mask=1111, last, ->%0<%Z8>\n" \ "" \ : "=Tr"(dst.data()) \ : "i"(type_traits::TypeCode), \ "i"(type_traits::TypeCode), \ - "ri"(src.GetValidCol()), \ - "ri"(src.GetValidRow()), \ + VCOL_CON, \ + VROW_CON, \ "i"(tile_shape_in::Cols), \ "Tr"(src.data()), \ "Tr"(Idx.data()), \ "i"(tile_type_traits::TilesizeCode)) - switch (ByteId) { - case 0: - THISTOGRAM_ASM("Byte0"); - break; - case 1: - THISTOGRAM_ASM("Byte1"); - break; - case 2: - THISTOGRAM_ASM("Byte2"); - break; - case 3: - THISTOGRAM_ASM("Byte3"); - break; - default: - return; // ByteId > 3 或 < 0，无效 +#define THISTOGRAM_DISPATCH(VCOL_TPL, VROW_TPL, VCOL_CON, VROW_CON) \ + switch (ByteId) { \ + case 0: THISTOGRAM_ASM("Byte0", VCOL_TPL, VROW_TPL, VCOL_CON, VROW_CON); break; \ + case 1: THISTOGRAM_ASM("Byte1", VCOL_TPL, VROW_TPL, VCOL_CON, VROW_CON); break; \ + case 2: THISTOGRAM_ASM("Byte2", VCOL_TPL, VROW_TPL, VCOL_CON, VROW_CON); break; \ + case 3: THISTOGRAM_ASM("Byte3", VCOL_TPL, VROW_TPL, VCOL_CON, VROW_CON); break; \ + default: return; /* ByteId > 3 或 < 0，无效 */ \ + } + + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { + THISTOGRAM_DISPATCH("zero, %c3", "zero, %c4", "i"(tile_shape_in::ValidCol), "i"(tile_shape_in::ValidRow)) + } else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { + THISTOGRAM_DISPATCH("zero, %c3", "%4, 0", "i"(tile_shape_in::ValidCol), "r"(src.GetValidRow())) + } else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { + THISTOGRAM_DISPATCH("%3, 0", "zero, %c4", "r"(src.GetValidCol()), "i"(tile_shape_in::ValidRow)) + } else { + THISTOGRAM_DISPATCH("%3, 0", "%4, 0", "r"(src.GetValidCol()), "r"(src.GetValidRow())) } +#undef THISTOGRAM_DISPATCH #undef THISTOGRAM_ASM } @@ -325,12 +348,67 @@ template void TLOAD2_ND2NZ(tile_shape &dst1, tile_shape &dst0, gm_shape &src) { static_assert(gm_shape::isRowMajor && is_Nz_layout::value, "GM_SHAPE should ND and TILE_SHAPE should be Nz "); - asm volatile( + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { +asm volatile( + "BSTART.TLSU TLOAD, %D[__pto_SrcType]\n" + "B.DATR ND2NZ, %D[__pto_DstType], Null\n" + "B.DIM zero, %c[__pto_VCOL], ->lb0\n" + "B.DIM zero, %c[__pto_VROW], ->lb1\n" + "B.DIM zero, %c[__pto_COL], ->lb2\n" + "B.IOT mask=1111, 0, ->%[__pto_d0]<%Z[__pto_TileSize]>\n" + "B.IOT mask=1111, last, ->%[__pto_d1]<%Z[__pto_TileSize]>\n" + "B.IOR [%[__pto_s0],%[__pto_GmStride]], []\n" + : [__pto_d0]"=Tr"(dst0.data()),[__pto_d1]"=Tr"(dst1.data()) + : [__pto_s0]"r"(src.data()), + [__pto_DstType]"i"(type_traits::TypeCode), + [__pto_SrcType]"i"(type_traits::TypeCode), + [__pto_TileSize]"i"(tile_type_traits::TilesizeCode), + [__pto_VCOL]"i"(tile_shape::ValidCol*2), [__pto_VROW]"i"(tile_shape::ValidRow), [__pto_COL]"i"(tile_shape::Cols*2), + [__pto_GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); } + else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { +asm volatile( + "BSTART.TLSU TLOAD, %D[__pto_SrcType]\n" + "B.DATR ND2NZ, %D[__pto_DstType], Null\n" + "B.DIM zero, %c[__pto_VCOL], ->lb0\n" + "B.DIM %[__pto_VROW], 0, ->lb1\n" + "B.DIM zero, %c[__pto_COL], ->lb2\n" + "B.IOT mask=1111, 0, ->%[__pto_d0]<%Z[__pto_TileSize]>\n" + "B.IOT mask=1111, last, ->%[__pto_d1]<%Z[__pto_TileSize]>\n" + "B.IOR [%[__pto_s0],%[__pto_GmStride]], []\n" + : [__pto_d0]"=Tr"(dst0.data()),[__pto_d1]"=Tr"(dst1.data()) + : [__pto_s0]"r"(src.data()), + [__pto_DstType]"i"(type_traits::TypeCode), + [__pto_SrcType]"i"(type_traits::TypeCode), + [__pto_TileSize]"i"(tile_type_traits::TilesizeCode), + [__pto_VCOL]"i"(tile_shape::ValidCol*2), [__pto_VROW]"r"(dst1.GetValidRow()), [__pto_COL]"i"(tile_shape::Cols*2), + [__pto_GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); } + else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { +asm volatile( + "BSTART.TLSU TLOAD, %D[__pto_SrcType]\n" + "B.DATR ND2NZ, %D[__pto_DstType], Null\n" + "B.DIM %[__pto_VCOL], 0, ->lb0\n" + "B.DIM zero, %c[__pto_VROW], ->lb1\n" + "B.DIM zero, %c[__pto_COL], ->lb2\n" + "B.IOT mask=1111, 0, ->%[__pto_d0]<%Z[__pto_TileSize]>\n" + "B.IOT mask=1111, last, ->%[__pto_d1]<%Z[__pto_TileSize]>\n" + "B.IOR [%[__pto_s0],%[__pto_GmStride]], []\n" + : [__pto_d0]"=Tr"(dst0.data()),[__pto_d1]"=Tr"(dst1.data()) + : [__pto_s0]"r"(src.data()), + [__pto_DstType]"i"(type_traits::TypeCode), + [__pto_SrcType]"i"(type_traits::TypeCode), + [__pto_TileSize]"i"(tile_type_traits::TilesizeCode), + [__pto_VCOL]"r"(dst1.GetValidCol()*2), [__pto_VROW]"i"(tile_shape::ValidRow), [__pto_COL]"i"(tile_shape::Cols*2), + [__pto_GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); } + else { +asm volatile( "BSTART.TLSU TLOAD, %D[__pto_SrcType]\n" "B.DATR ND2NZ, %D[__pto_DstType], Null\n" "B.DIM %[__pto_VCOL], 0, ->lb0\n" "B.DIM %[__pto_VROW], 0, ->lb1\n" - "B.DIM zero, %[__pto_COL], ->lb2\n" + "B.DIM zero, %c[__pto_COL], ->lb2\n" "B.IOT mask=1111, 0, ->%[__pto_d0]<%Z[__pto_TileSize]>\n" "B.IOT mask=1111, last, ->%[__pto_d1]<%Z[__pto_TileSize]>\n" "B.IOR [%[__pto_s0],%[__pto_GmStride]], []\n" @@ -339,21 +417,76 @@ void TLOAD2_ND2NZ(tile_shape &dst1, tile_shape &dst0, gm_shape &src) { [__pto_DstType]"i"(type_traits::TypeCode), [__pto_SrcType]"i"(type_traits::TypeCode), [__pto_TileSize]"i"(tile_type_traits::TilesizeCode), - [__pto_VCOL]"ri"(dst1.GetValidCol()*2), [__pto_VROW]"ri"(dst1.GetValidRow()), [__pto_COL]"i"(tile_shape::Cols*2), + [__pto_VCOL]"r"(dst1.GetValidCol()*2), [__pto_VROW]"r"(dst1.GetValidRow()), [__pto_COL]"i"(tile_shape::Cols*2), [__pto_GmStride]"r"(src.GetStrideBytes(3)) - : "memory"); + : "memory"); } } template void TLOAD2_ND2ZN(tile_shape &dst1, tile_shape &dst0, gm_shape &src) { static_assert(gm_shape::isRowMajor && is_Zn_layout::value, "GM_SHAPE should ND and TILE_SHAPE should be Zn "); - asm volatile( + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { +asm volatile( + "BSTART.TLSU TLOAD, %D[__pto_SrcType]\n" + "B.DATR ND2ZN, %D[__pto_DstType], Null\n" + "B.DIM zero, %c[__pto_VCOL], ->lb0\n" + "B.DIM zero, %c[__pto_VROW], ->lb1\n" + "B.DIM zero, %c[__pto_COL], ->lb2\n" + "B.IOT mask=1111, 0, ->%[__pto_d0]<%Z[__pto_TileSize]>\n" + "B.IOT mask=1111, last, ->%[__pto_d1]<%Z[__pto_TileSize]>\n" + "B.IOR [%[__pto_s0],%[__pto_GmStride]], []\n" + : [__pto_d0]"=Tr"(dst0.data()),[__pto_d1]"=Tr"(dst1.data()) + : [__pto_s0]"r"(src.data()), + [__pto_DstType]"i"(type_traits::TypeCode), + [__pto_SrcType]"i"(type_traits::TypeCode), + [__pto_TileSize]"i"(tile_type_traits::TilesizeCode), + [__pto_VCOL]"i"(tile_shape::ValidCol*2), [__pto_VROW]"i"(tile_shape::ValidRow), [__pto_COL]"i"(tile_shape::Cols*2), + [__pto_GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); } + else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { +asm volatile( + "BSTART.TLSU TLOAD, %D[__pto_SrcType]\n" + "B.DATR ND2ZN, %D[__pto_DstType], Null\n" + "B.DIM zero, %c[__pto_VCOL], ->lb0\n" + "B.DIM %[__pto_VROW], 0, ->lb1\n" + "B.DIM zero, %c[__pto_COL], ->lb2\n" + "B.IOT mask=1111, 0, ->%[__pto_d0]<%Z[__pto_TileSize]>\n" + "B.IOT mask=1111, last, ->%[__pto_d1]<%Z[__pto_TileSize]>\n" + "B.IOR [%[__pto_s0],%[__pto_GmStride]], []\n" + : [__pto_d0]"=Tr"(dst0.data()),[__pto_d1]"=Tr"(dst1.data()) + : [__pto_s0]"r"(src.data()), + [__pto_DstType]"i"(type_traits::TypeCode), + [__pto_SrcType]"i"(type_traits::TypeCode), + [__pto_TileSize]"i"(tile_type_traits::TilesizeCode), + [__pto_VCOL]"i"(tile_shape::ValidCol*2), [__pto_VROW]"r"(dst1.GetValidRow()), [__pto_COL]"i"(tile_shape::Cols*2), + [__pto_GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); } + else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { +asm volatile( + "BSTART.TLSU TLOAD, %D[__pto_SrcType]\n" + "B.DATR ND2ZN, %D[__pto_DstType], Null\n" + "B.DIM %[__pto_VCOL], 0, ->lb0\n" + "B.DIM zero, %c[__pto_VROW], ->lb1\n" + "B.DIM zero, %c[__pto_COL], ->lb2\n" + "B.IOT mask=1111, 0, ->%[__pto_d0]<%Z[__pto_TileSize]>\n" + "B.IOT mask=1111, last, ->%[__pto_d1]<%Z[__pto_TileSize]>\n" + "B.IOR [%[__pto_s0],%[__pto_GmStride]], []\n" + : [__pto_d0]"=Tr"(dst0.data()),[__pto_d1]"=Tr"(dst1.data()) + : [__pto_s0]"r"(src.data()), + [__pto_DstType]"i"(type_traits::TypeCode), + [__pto_SrcType]"i"(type_traits::TypeCode), + [__pto_TileSize]"i"(tile_type_traits::TilesizeCode), + [__pto_VCOL]"r"(dst1.GetValidCol()*2), [__pto_VROW]"i"(tile_shape::ValidRow), [__pto_COL]"i"(tile_shape::Cols*2), + [__pto_GmStride]"r"(src.GetStrideBytes(3)) + : "memory"); } + else { +asm volatile( "BSTART.TLSU TLOAD, %D[__pto_SrcType]\n" "B.DATR ND2ZN, %D[__pto_DstType], Null\n" "B.DIM %[__pto_VCOL], 0, ->lb0\n" "B.DIM %[__pto_VROW], 0, ->lb1\n" - "B.DIM zero, %[__pto_COL], ->lb2\n" + "B.DIM zero, %c[__pto_COL], ->lb2\n" "B.IOT mask=1111, 0, ->%[__pto_d0]<%Z[__pto_TileSize]>\n" "B.IOT mask=1111, last, ->%[__pto_d1]<%Z[__pto_TileSize]>\n" "B.IOR [%[__pto_s0],%[__pto_GmStride]], []\n" @@ -362,9 +495,9 @@ void TLOAD2_ND2ZN(tile_shape &dst1, tile_shape &dst0, gm_shape &src) { [__pto_DstType]"i"(type_traits::TypeCode), [__pto_SrcType]"i"(type_traits::TypeCode), [__pto_TileSize]"i"(tile_type_traits::TilesizeCode), - [__pto_VCOL]"ri"(dst1.GetValidCol()*2), [__pto_VROW]"ri"(dst1.GetValidRow()), [__pto_COL]"i"(tile_shape::Cols*2), + [__pto_VCOL]"r"(dst1.GetValidCol()*2), [__pto_VROW]"r"(dst1.GetValidRow()), [__pto_COL]"i"(tile_shape::Cols*2), [__pto_GmStride]"r"(src.GetStrideBytes(3)) - : "memory"); + : "memory"); } } template @@ -376,7 +509,7 @@ void TLOAD2_DN2ZN(tile_shape &dst1, tile_shape &dst0, gm_shape &src) { "B.DATR DN2ZN, %D[__pto_DstType], Null\n" "B.DIM %[__pto_VCOL], 0, ->lb0\n" "B.DIM %[__pto_VROW], 0, ->lb1\n" - "B.DIM zero, %[__pto_COL], ->lb2\n" + "B.DIM zero, %c[__pto_COL], ->lb2\n" "B.IOT mask=1111, 0, ->%[__pto_d0]<%Z[__pto_TileSize]>\n" "B.IOT mask=1111, last, ->%[__pto_d1]<%Z[__pto_TileSize]>\n" "B.IOR [%[__pto_s0],%[__pto_GmStride]], []\n" @@ -394,21 +527,70 @@ template void TSTORE2_DN2DN(gm_shape &dst, tile_shape &src1, tile_shape &src0) { static_assert(!gm_shape::isRowMajor && !tile_shape::isRowMajor, "GM_SHAPE should DN and TILE_SHAPE should be DN"); - asm volatile( + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { +asm volatile( + "BSTART.TLSU TSTORE, %D[__pto_SrcType]\n" + "B.DATR NORM, %D[__pto_DstType], Null\n" + "B.DIM zero, %c[__pto_VCOL], ->lb0\n" + "B.DIM zero, %c[__pto_VROW], ->lb1\n" + "B.DIM zero, %c[__pto_COL], ->lb2\n" + "B.IOT %[__pto_s0], %[s1], mask=1111, last\n" + "B.IOR [%[__pto_d0],%[__pto_GmStride]], []\n" + : + : [__pto_d0]"r"(dst.data()), [__pto_s0]"Tr"(src0.data()), [s1]"Tr"(src1.data()), + [__pto_DstType]"i"(type_traits::TypeCode), + [__pto_SrcType]"i"(type_traits::TypeCode), + [__pto_VCOL]"i"(tile_shape::ValidRow*2), [__pto_VROW]"i"(tile_shape::ValidCol), [__pto_COL]"i"(tile_shape::Rows*2), + [__pto_GmStride]"r"(dst.GetStrideBytes(4)) + : "memory"); } + else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { +asm volatile( + "BSTART.TLSU TSTORE, %D[__pto_SrcType]\n" + "B.DATR NORM, %D[__pto_DstType], Null\n" + "B.DIM zero, %c[__pto_VCOL], ->lb0\n" + "B.DIM %[__pto_VROW], 0, ->lb1\n" + "B.DIM zero, %c[__pto_COL], ->lb2\n" + "B.IOT %[__pto_s0], %[s1], mask=1111, last\n" + "B.IOR [%[__pto_d0],%[__pto_GmStride]], []\n" + : + : [__pto_d0]"r"(dst.data()), [__pto_s0]"Tr"(src0.data()), [s1]"Tr"(src1.data()), + [__pto_DstType]"i"(type_traits::TypeCode), + [__pto_SrcType]"i"(type_traits::TypeCode), + [__pto_VCOL]"i"(tile_shape::ValidRow*2), [__pto_VROW]"r"(src0.GetValidCol()), [__pto_COL]"i"(tile_shape::Rows*2), + [__pto_GmStride]"r"(dst.GetStrideBytes(4)) + : "memory"); } + else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { +asm volatile( + "BSTART.TLSU TSTORE, %D[__pto_SrcType]\n" + "B.DATR NORM, %D[__pto_DstType], Null\n" + "B.DIM %[__pto_VCOL], 0, ->lb0\n" + "B.DIM zero, %c[__pto_VROW], ->lb1\n" + "B.DIM zero, %c[__pto_COL], ->lb2\n" + "B.IOT %[__pto_s0], %[s1], mask=1111, last\n" + "B.IOR [%[__pto_d0],%[__pto_GmStride]], []\n" + : + : [__pto_d0]"r"(dst.data()), [__pto_s0]"Tr"(src0.data()), [s1]"Tr"(src1.data()), + [__pto_DstType]"i"(type_traits::TypeCode), + [__pto_SrcType]"i"(type_traits::TypeCode), + [__pto_VCOL]"r"(src0.GetValidRow()*2), [__pto_VROW]"i"(tile_shape::ValidCol), [__pto_COL]"i"(tile_shape::Rows*2), + [__pto_GmStride]"r"(dst.GetStrideBytes(4)) + : "memory"); } + else { +asm volatile( "BSTART.TLSU TSTORE, %D[__pto_SrcType]\n" "B.DATR NORM, %D[__pto_DstType], Null\n" "B.DIM %[__pto_VCOL], 0, ->lb0\n" "B.DIM %[__pto_VROW], 0, ->lb1\n" - "B.DIM zero, %[__pto_COL], ->lb2\n" + "B.DIM zero, %c[__pto_COL], ->lb2\n" "B.IOT %[__pto_s0], %[s1], mask=1111, last\n" "B.IOR [%[__pto_d0],%[__pto_GmStride]], []\n" : : [__pto_d0]"r"(dst.data()), [__pto_s0]"Tr"(src0.data()), [s1]"Tr"(src1.data()), [__pto_DstType]"i"(type_traits::TypeCode), [__pto_SrcType]"i"(type_traits::TypeCode), - [__pto_VCOL]"ri"(src0.GetValidRow()*2), [__pto_VROW]"ri"(src0.GetValidCol()), [__pto_COL]"i"(tile_shape::Rows*2), + [__pto_VCOL]"r"(src0.GetValidRow()*2), [__pto_VROW]"r"(src0.GetValidCol()), [__pto_COL]"i"(tile_shape::Rows*2), [__pto_GmStride]"r"(dst.GetStrideBytes(4)) - : "memory"); + : "memory"); } } template @@ -420,7 +602,7 @@ void TLOAD4_ND2NZ(tile_shape &dst3, tile_shape &dst2, tile_shape &dst1, tile_sha "B.DATR ND2NZ, %D[__pto_DstType], Null\n" "B.DIM %[__pto_VCOL], 0, ->lb0\n" "B.DIM %[__pto_VROW], 0, ->lb1\n" - "B.DIM zero, %[__pto_COL], ->lb2\n" + "B.DIM zero, %c[__pto_COL], ->lb2\n" "B.IOT mask=1111, 0, ->%[__pto_d0]<%Z[__pto_TileSize]>\n" "B.IOT mask=1111, 0, ->%[__pto_d1]<%Z[__pto_TileSize]>\n" "B.IOT mask=1111, 0, ->%[d2]<%Z[__pto_TileSize]>\n" @@ -445,7 +627,7 @@ void TLOAD4_ND2ZN(tile_shape &dst3, tile_shape &dst2, tile_shape &dst1, tile_sha "B.DATR ND2ZN, %D[__pto_DstType], Null\n" "B.DIM %[__pto_VCOL], 0, ->lb0\n" "B.DIM %[__pto_VROW], 0, ->lb1\n" - "B.DIM zero, %[__pto_COL], ->lb2\n" + "B.DIM zero, %c[__pto_COL], ->lb2\n" "B.IOT mask=1111, 0, ->%[__pto_d0]<%Z[__pto_TileSize]>\n" "B.IOT mask=1111, 0, ->%[__pto_d1]<%Z[__pto_TileSize]>\n" "B.IOT mask=1111, 0, ->%[d2]<%Z[__pto_TileSize]>\n" @@ -470,7 +652,7 @@ void TLOAD4_DN2ZN(tile_shape &dst3, tile_shape &dst2, tile_shape &dst1, tile_sha "B.DATR DN2ZN, %D[__pto_DstType], Null\n" "B.DIM %[__pto_VCOL], 0, ->lb0\n" "B.DIM %[__pto_VROW], 0, ->lb1\n" - "B.DIM zero, %[__pto_COL], ->lb2\n" + "B.DIM zero, %c[__pto_COL], ->lb2\n" "B.IOT mask=1111, 0, ->%[__pto_d0]<%Z[__pto_TileSize]>\n" "B.IOT mask=1111, 0, ->%[__pto_d1]<%Z[__pto_TileSize]>\n" "B.IOT mask=1111, 0, ->%[d2]<%Z[__pto_TileSize]>\n" @@ -501,11 +683,12 @@ inline void MGATHER(tile_shape_out &dst, const gm_shape &src, static_assert(tile_type_traits::IsValidActiveSize, "MGATHER dst logical Tile size must be 128 B..256 KiB (SizeCode=1..12) " "per DavinciOO v5 B.IOT encoding"); - asm volatile( + if constexpr (tile_shape_offset::ValidCol > 0 && tile_shape_offset::ValidRow > 0) { +asm volatile( "BSTART.TLSU MGATHER, %D[DataType]\n" "B.DATR Null\n" - "B.DIM %[ValidCol], 0, ->LB0\n" - "B.DIM %[ValidRow], 0, ->LB1\n" + "B.DIM zero, %c[ValidCol], ->LB0\n" + "B.DIM zero, %c[ValidRow], ->LB1\n" "B.DIM zero, %c[Col], ->LB2\n" "B.IOT %[off], mask=1111, last, ->%[dst]<%Z[TileSize]>\n" "B.IOR [%[base], %[GmStride]], []\n" @@ -515,36 +698,148 @@ inline void MGATHER(tile_shape_out &dst, const gm_shape &src, [PadValue] "i"(static_cast(Pad)), [TileSize] "i"( tile_type_traits::TilesizeCode), - [ValidCol] "ri"(offset.GetValidCol()), - [ValidRow] "ri"(offset.GetValidRow()), + [ValidCol] "i"(tile_shape_offset::ValidCol), + [ValidRow] "i"(tile_shape_offset::ValidRow), [Col] "i"(tile_shape_offset::Cols), [GmStride] "r"(src.GetStride(3)) - : "memory"); -} - -template -inline void MSCATTER(gm_shape &dst, const tile_shape_in &src, - const tile_shape_offset &offset) { - static_assert(tile_shape_offset::ValidCol <= tile_shape_offset::Cols, ""); - static_assert(tile_type_traits::IsValidActiveSize, - "MSCATTER src logical Tile size must be 128 B..256 KiB (SizeCode=1..12) " - "per DavinciOO v5 B.IOT encoding"); - asm volatile( - "BSTART.TLSU MSCATTER, %D[DataType]\n" - "B.DIM %[ValidCol], 0, ->LB0\n" + : "memory"); } + else if constexpr (tile_shape_offset::ValidCol > 0 && tile_shape_offset::ValidRow < 0) { +asm volatile( + "BSTART.TLSU MGATHER, %D[DataType]\n" + "B.DATR Null\n" + "B.DIM zero, %c[ValidCol], ->LB0\n" "B.DIM %[ValidRow], 0, ->LB1\n" "B.DIM zero, %c[Col], ->LB2\n" + "B.IOT %[off], mask=1111, last, ->%[dst]<%Z[TileSize]>\n" + "B.IOR [%[base], %[GmStride]], []\n" + : [dst] "=Tr"(dst.data()) + : [base] "r"(src.data()), [off] "Tr"(offset.data()), + [DataType] "i"(type_traits::TypeCode), + [PadValue] "i"(static_cast(Pad)), + [TileSize] "i"( + tile_type_traits::TilesizeCode), + [ValidCol] "i"(tile_shape_offset::ValidCol), + [ValidRow] "r"(offset.GetValidRow()), + [Col] "i"(tile_shape_offset::Cols), + [GmStride] "r"(src.GetStride(3)) + : "memory"); } + else if constexpr (tile_shape_offset::ValidCol < 0 && tile_shape_offset::ValidRow > 0) { +asm volatile( + "BSTART.TLSU MGATHER, %D[DataType]\n" + "B.DATR Null\n" + "B.DIM %[ValidCol], 0, ->LB0\n" + "B.DIM zero, %c[ValidRow], ->LB1\n" + "B.DIM zero, %c[Col], ->LB2\n" + "B.IOT %[off], mask=1111, last, ->%[dst]<%Z[TileSize]>\n" + "B.IOR [%[base], %[GmStride]], []\n" + : [dst] "=Tr"(dst.data()) + : [base] "r"(src.data()), [off] "Tr"(offset.data()), + [DataType] "i"(type_traits::TypeCode), + [PadValue] "i"(static_cast(Pad)), + [TileSize] "i"( + tile_type_traits::TilesizeCode), + [ValidCol] "r"(offset.GetValidCol()), + [ValidRow] "i"(tile_shape_offset::ValidRow), + [Col] "i"(tile_shape_offset::Cols), + [GmStride] "r"(src.GetStride(3)) + : "memory"); } + else { +asm volatile( + "BSTART.TLSU MGATHER, %D[DataType]\n" + "B.DATR Null\n" + "B.DIM %[ValidCol], 0, ->LB0\n" + "B.DIM %[ValidRow], 0, ->LB1\n" + "B.DIM zero, %c[Col], ->LB2\n" + "B.IOT %[off], mask=1111, last, ->%[dst]<%Z[TileSize]>\n" + "B.IOR [%[base], %[GmStride]], []\n" + : [dst] "=Tr"(dst.data()) + : [base] "r"(src.data()), [off] "Tr"(offset.data()), + [DataType] "i"(type_traits::TypeCode), + [PadValue] "i"(static_cast(Pad)), + [TileSize] "i"( + tile_type_traits::TilesizeCode), + [ValidCol] "r"(offset.GetValidCol()), + [ValidRow] "r"(offset.GetValidRow()), + [Col] "i"(tile_shape_offset::Cols), + [GmStride] "r"(src.GetStride(3)) + : "memory"); } +} + +template +inline void MSCATTER(gm_shape &dst, const tile_shape_in &src, + const tile_shape_offset &offset) { + static_assert(tile_shape_offset::ValidCol <= tile_shape_offset::Cols, ""); + static_assert(tile_type_traits::IsValidActiveSize, + "MSCATTER src logical Tile size must be 128 B..256 KiB (SizeCode=1..12) " + "per DavinciOO v5 B.IOT encoding"); + if constexpr (tile_shape_offset::ValidCol > 0 && tile_shape_offset::ValidRow > 0) { +asm volatile( + "BSTART.TLSU MSCATTER, %D[DataType]\n" + "B.DIM zero, %c[ValidCol], ->LB0\n" + "B.DIM zero, %c[ValidRow], ->LB1\n" + "B.DIM zero, %c[Col], ->LB2\n" "B.IOT %[src], %[off], mask=1111, last\n" "B.IOR [%[base], %[GmStride]], []\n" : : [base] "r"(dst.data()), [src] "Tr"(src.data()), [off] "Tr"(offset.data()), [DataType] "i"(type_traits::TypeCode), - [ValidCol] "ri"(offset.GetValidCol()), - [ValidRow] "ri"(offset.GetValidRow()), + [ValidCol] "i"(tile_shape_offset::ValidCol), + [ValidRow] "i"(tile_shape_offset::ValidRow), [Col] "i"(tile_shape_offset::Cols), [GmStride] "r"(dst.GetStride(3)) - : "memory"); + : "memory"); } + else if constexpr (tile_shape_offset::ValidCol > 0 && tile_shape_offset::ValidRow < 0) { +asm volatile( + "BSTART.TLSU MSCATTER, %D[DataType]\n" + "B.DIM zero, %c[ValidCol], ->LB0\n" + "B.DIM %[ValidRow], 0, ->LB1\n" + "B.DIM zero, %c[Col], ->LB2\n" + "B.IOT %[src], %[off], mask=1111, last\n" + "B.IOR [%[base], %[GmStride]], []\n" + : + : [base] "r"(dst.data()), [src] "Tr"(src.data()), + [off] "Tr"(offset.data()), + [DataType] "i"(type_traits::TypeCode), + [ValidCol] "i"(tile_shape_offset::ValidCol), + [ValidRow] "r"(offset.GetValidRow()), + [Col] "i"(tile_shape_offset::Cols), + [GmStride] "r"(dst.GetStride(3)) + : "memory"); } + else if constexpr (tile_shape_offset::ValidCol < 0 && tile_shape_offset::ValidRow > 0) { +asm volatile( + "BSTART.TLSU MSCATTER, %D[DataType]\n" + "B.DIM %[ValidCol], 0, ->LB0\n" + "B.DIM zero, %c[ValidRow], ->LB1\n" + "B.DIM zero, %c[Col], ->LB2\n" + "B.IOT %[src], %[off], mask=1111, last\n" + "B.IOR [%[base], %[GmStride]], []\n" + : + : [base] "r"(dst.data()), [src] "Tr"(src.data()), + [off] "Tr"(offset.data()), + [DataType] "i"(type_traits::TypeCode), + [ValidCol] "r"(offset.GetValidCol()), + [ValidRow] "i"(tile_shape_offset::ValidRow), + [Col] "i"(tile_shape_offset::Cols), + [GmStride] "r"(dst.GetStride(3)) + : "memory"); } + else { +asm volatile( + "BSTART.TLSU MSCATTER, %D[DataType]\n" + "B.DIM %[ValidCol], 0, ->LB0\n" + "B.DIM %[ValidRow], 0, ->LB1\n" + "B.DIM zero, %c[Col], ->LB2\n" + "B.IOT %[src], %[off], mask=1111, last\n" + "B.IOR [%[base], %[GmStride]], []\n" + : + : [base] "r"(dst.data()), [src] "Tr"(src.data()), + [off] "Tr"(offset.data()), + [DataType] "i"(type_traits::TypeCode), + [ValidCol] "r"(offset.GetValidCol()), + [ValidRow] "r"(offset.GetValidRow()), + [Col] "i"(tile_shape_offset::Cols), + [GmStride] "r"(dst.GetStride(3)) + : "memory"); } } template ::IsValidActiveSize, "MGATHER_MASK dst logical Tile size must be 128 B..256 KiB (SizeCode=1..12) " "per DavinciOO v5 B.IOT encoding"); - asm volatile( + if constexpr (tile_shape_offset::ValidCol > 0 && tile_shape_offset::ValidRow > 0) { +asm volatile( + "BSTART.TLSU MGATHER.MASK, %D[DataType]\n" + "B.DATR Null\n" + "B.DIM zero, %c[ValidCol], ->LB0\n" + "B.DIM zero, %c[ValidRow], ->LB1\n" + "B.DIM zero, %c[Col], ->LB2\n" + "B.IOT %[off], %[mask], mask=1111, last, ->%[dst]<%Z[TileSize]>\n" + "B.IOR [%[base], %[GmStride]], []\n" + : [dst] "=Tr"(dst.data()) + : [base] "r"(src.data()), [off] "Tr"(offset.data()), + [mask] "Tr"(mask.data()), + [DataType] "i"(type_traits::TypeCode), + [PadValue] "i"(static_cast(Pad)), + [TileSize] "i"( + tile_type_traits::TilesizeCode), + [ValidCol] "i"(tile_shape_offset::ValidCol), + [ValidRow] "i"(tile_shape_offset::ValidRow), + [Col] "i"(tile_shape_offset::Cols), + [GmStride] "r"(src.GetStride(3)) + : "memory"); } + else if constexpr (tile_shape_offset::ValidCol > 0 && tile_shape_offset::ValidRow < 0) { +asm volatile( + "BSTART.TLSU MGATHER.MASK, %D[DataType]\n" + "B.DATR Null\n" + "B.DIM zero, %c[ValidCol], ->LB0\n" + "B.DIM %[ValidRow], 0, ->LB1\n" + "B.DIM zero, %c[Col], ->LB2\n" + "B.IOT %[off], %[mask], mask=1111, last, ->%[dst]<%Z[TileSize]>\n" + "B.IOR [%[base], %[GmStride]], []\n" + : [dst] "=Tr"(dst.data()) + : [base] "r"(src.data()), [off] "Tr"(offset.data()), + [mask] "Tr"(mask.data()), + [DataType] "i"(type_traits::TypeCode), + [PadValue] "i"(static_cast(Pad)), + [TileSize] "i"( + tile_type_traits::TilesizeCode), + [ValidCol] "i"(tile_shape_offset::ValidCol), + [ValidRow] "r"(offset.GetValidRow()), + [Col] "i"(tile_shape_offset::Cols), + [GmStride] "r"(src.GetStride(3)) + : "memory"); } + else if constexpr (tile_shape_offset::ValidCol < 0 && tile_shape_offset::ValidRow > 0) { +asm volatile( + "BSTART.TLSU MGATHER.MASK, %D[DataType]\n" + "B.DATR Null\n" + "B.DIM %[ValidCol], 0, ->LB0\n" + "B.DIM zero, %c[ValidRow], ->LB1\n" + "B.DIM zero, %c[Col], ->LB2\n" + "B.IOT %[off], %[mask], mask=1111, last, ->%[dst]<%Z[TileSize]>\n" + "B.IOR [%[base], %[GmStride]], []\n" + : [dst] "=Tr"(dst.data()) + : [base] "r"(src.data()), [off] "Tr"(offset.data()), + [mask] "Tr"(mask.data()), + [DataType] "i"(type_traits::TypeCode), + [PadValue] "i"(static_cast(Pad)), + [TileSize] "i"( + tile_type_traits::TilesizeCode), + [ValidCol] "r"(offset.GetValidCol()), + [ValidRow] "i"(tile_shape_offset::ValidRow), + [Col] "i"(tile_shape_offset::Cols), + [GmStride] "r"(src.GetStride(3)) + : "memory"); } + else { +asm volatile( "BSTART.TLSU MGATHER.MASK, %D[DataType]\n" "B.DATR Null\n" "B.DIM %[ValidCol], 0, ->LB0\n" @@ -572,11 +931,11 @@ inline void MGATHER_MASK(tile_shape_out &dst, const gm_shape &src, [PadValue] "i"(static_cast(Pad)), [TileSize] "i"( tile_type_traits::TilesizeCode), - [ValidCol] "ri"(offset.GetValidCol()), - [ValidRow] "ri"(offset.GetValidRow()), + [ValidCol] "r"(offset.GetValidCol()), + [ValidRow] "r"(offset.GetValidRow()), [Col] "i"(tile_shape_offset::Cols), [GmStride] "r"(src.GetStride(3)) - : "memory"); + : "memory"); } } template ::IsValidActiveSize, "MSCATTER_MASK src logical Tile size must be 128 B..256 KiB (SizeCode=1..12) " "per DavinciOO v5 B.IOT encoding"); - asm volatile( + if constexpr (tile_shape_offset::ValidCol > 0 && tile_shape_offset::ValidRow > 0) { +asm volatile( + "BSTART.TLSU MSCATTER.MASK, %D[DataType]\n" + "B.DIM zero, %c[ValidCol], ->LB0\n" + "B.DIM zero, %c[ValidRow], ->LB1\n" + "B.DIM zero, %c[Col], ->LB2\n" + "B.IOT %[src], %[off], mask=1111\n" + "B.IOT %[mask], mask=1111, last\n" + "B.IOR [%[base], %[GmStride]], []\n" + : + : [base] "r"(dst.data()), [src] "Tr"(src.data()), + [off] "Tr"(offset.data()), [mask] "Tr"(mask.data()), + [DataType] "i"(type_traits::TypeCode), + [ValidCol] "i"(tile_shape_offset::ValidCol), + [ValidRow] "i"(tile_shape_offset::ValidRow), + [Col] "i"(tile_shape_offset::Cols), + [GmStride] "r"(dst.GetStride(3)) + : "memory"); } + else if constexpr (tile_shape_offset::ValidCol > 0 && tile_shape_offset::ValidRow < 0) { +asm volatile( + "BSTART.TLSU MSCATTER.MASK, %D[DataType]\n" + "B.DIM zero, %c[ValidCol], ->LB0\n" + "B.DIM %[ValidRow], 0, ->LB1\n" + "B.DIM zero, %c[Col], ->LB2\n" + "B.IOT %[src], %[off], mask=1111\n" + "B.IOT %[mask], mask=1111, last\n" + "B.IOR [%[base], %[GmStride]], []\n" + : + : [base] "r"(dst.data()), [src] "Tr"(src.data()), + [off] "Tr"(offset.data()), [mask] "Tr"(mask.data()), + [DataType] "i"(type_traits::TypeCode), + [ValidCol] "i"(tile_shape_offset::ValidCol), + [ValidRow] "r"(offset.GetValidRow()), + [Col] "i"(tile_shape_offset::Cols), + [GmStride] "r"(dst.GetStride(3)) + : "memory"); } + else if constexpr (tile_shape_offset::ValidCol < 0 && tile_shape_offset::ValidRow > 0) { +asm volatile( + "BSTART.TLSU MSCATTER.MASK, %D[DataType]\n" + "B.DIM %[ValidCol], 0, ->LB0\n" + "B.DIM zero, %c[ValidRow], ->LB1\n" + "B.DIM zero, %c[Col], ->LB2\n" + "B.IOT %[src], %[off], mask=1111\n" + "B.IOT %[mask], mask=1111, last\n" + "B.IOR [%[base], %[GmStride]], []\n" + : + : [base] "r"(dst.data()), [src] "Tr"(src.data()), + [off] "Tr"(offset.data()), [mask] "Tr"(mask.data()), + [DataType] "i"(type_traits::TypeCode), + [ValidCol] "r"(offset.GetValidCol()), + [ValidRow] "i"(tile_shape_offset::ValidRow), + [Col] "i"(tile_shape_offset::Cols), + [GmStride] "r"(dst.GetStride(3)) + : "memory"); } + else { +asm volatile( "BSTART.TLSU MSCATTER.MASK, %D[DataType]\n" "B.DIM %[ValidCol], 0, ->LB0\n" "B.DIM %[ValidRow], 0, ->LB1\n" @@ -600,11 +1014,11 @@ inline void MSCATTER_MASK(gm_shape &dst, const tile_shape_in &src, : [base] "r"(dst.data()), [src] "Tr"(src.data()), [off] "Tr"(offset.data()), [mask] "Tr"(mask.data()), [DataType] "i"(type_traits::TypeCode), - [ValidCol] "ri"(offset.GetValidCol()), - [ValidRow] "ri"(offset.GetValidRow()), + [ValidCol] "r"(offset.GetValidCol()), + [ValidRow] "r"(offset.GetValidRow()), [Col] "i"(tile_shape_offset::Cols), [GmStride] "r"(dst.GetStride(3)) - : "memory"); + : "memory"); } } #ifndef LINX_CVT_INLINE @@ -2218,7 +2632,8 @@ void TLOAD_CUBE(cube_shape &dst, gm_shape &src) { // with load padding Zero. Selector text is constant, not an integer // immediate, so it assembles as the ISA mnemonic. if constexpr (cube_shape::BFractal == BLayout::CubeM32) { - asm volatile( + if constexpr (cube_shape::ValidCol > 0 && cube_shape::ValidRow > 0) { +asm volatile( "BSTART.TLSU TLOAD, %D[DataType]\n" "B.DATR ND2M32.normal, Zero\n" "B.DIM zero, %c[VCOL], ->lb0\n" @@ -2230,14 +2645,14 @@ void TLOAD_CUBE(cube_shape &dst, gm_shape &src) { [RowStrideBytes] "r"(src.GetStrideBytes(3)), [DataType] "i"(type_traits::TypeCode), [SizeCode] "i"(cube_shape::TilesizeCode), - [VCOL] "ri"(valid_col), [VROW] "ri"(valid_row) - : "memory"); - } else if constexpr (cube_shape::BFractal == BLayout::CubeM16) { - asm volatile( + [VCOL] "i"(cube_shape::ValidCol), [VROW] "i"(cube_shape::ValidRow) + : "memory"); } + else if constexpr (cube_shape::ValidCol > 0 && cube_shape::ValidRow < 0) { +asm volatile( "BSTART.TLSU TLOAD, %D[DataType]\n" - "B.DATR ND2M16.normal, Zero\n" + "B.DATR ND2M32.normal, Zero\n" "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM %[VROW], 0, ->lb1\n" "B.IOT mask=1111, last, ->%[Dst]<%Z[SizeCode]>\n" "B.IOR [%[Base],%[RowStrideBytes]], []\n" : [Dst] "=Tr"(dst.data()) @@ -2245,13 +2660,13 @@ void TLOAD_CUBE(cube_shape &dst, gm_shape &src) { [RowStrideBytes] "r"(src.GetStrideBytes(3)), [DataType] "i"(type_traits::TypeCode), [SizeCode] "i"(cube_shape::TilesizeCode), - [VCOL] "ri"(valid_col), [VROW] "ri"(valid_row) - : "memory"); - } else { // CubeN8 - asm volatile( + [VCOL] "i"(cube_shape::ValidCol), [VROW] "r"(valid_row) + : "memory"); } + else if constexpr (cube_shape::ValidCol < 0 && cube_shape::ValidRow > 0) { +asm volatile( "BSTART.TLSU TLOAD, %D[DataType]\n" - "B.DATR ND2N8.normal, Zero\n" - "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DATR ND2M32.normal, Zero\n" + "B.DIM %[VCOL], 0, ->lb0\n" "B.DIM zero, %c[VROW], ->lb1\n" "B.IOT mask=1111, last, ->%[Dst]<%Z[SizeCode]>\n" "B.IOR [%[Base],%[RowStrideBytes]], []\n" @@ -2260,75 +2675,341 @@ void TLOAD_CUBE(cube_shape &dst, gm_shape &src) { [RowStrideBytes] "r"(src.GetStrideBytes(3)), [DataType] "i"(type_traits::TypeCode), [SizeCode] "i"(cube_shape::TilesizeCode), - [VCOL] "ri"(valid_col), [VROW] "ri"(valid_row) - : "memory"); - } -} - -// PTO ISA 0.58.3 persistent Local CUBE CELL -> GM transport. The source -// descriptor survives the operation and only its logical valid rectangle is -// exported. -template - requires(cube_shape::IsCubeLayout) -void TSTORE_CUBE(gm_shape &dst, const cube_shape &src) { - static_assert(std::is_same_v, - "TSTORE_CUBE requires matching GM and CUBE dtypes"); - static_assert(cube_shape::IsValidActiveSize, - "TSTORE_CUBE Local CUBE capacity must be 128 B..256 KiB"); - const size_t valid_col = src.GetValidCol(); - const size_t valid_row = src.GetValidRow(); - // Persistent CUBE cell layout selects the canonical Local->GM transport - // selector (PTO-ISA ADR-0070); store padding is Null. Selector text is - // constant per layout class, not an integer immediate. - if constexpr (cube_shape::BFractal == BLayout::CubeM32) { - asm volatile( - "BSTART.TLSU TSTORE, %D[DataType]\n" - "B.DATR M322ND.normal, Null\n" - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.IOT %[Src], mask=1111, last\n" + [VCOL] "r"(valid_col), [VROW] "i"(cube_shape::ValidRow) + : "memory"); } + else { +asm volatile( + "BSTART.TLSU TLOAD, %D[DataType]\n" + "B.DATR ND2M32.normal, Zero\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.IOT mask=1111, last, ->%[Dst]<%Z[SizeCode]>\n" "B.IOR [%[Base],%[RowStrideBytes]], []\n" - : - : [Base] "r"(dst.data()), [Src] "Tr"(src.data()), - [RowStrideBytes] "r"(dst.GetStrideBytes(3)), + : [Dst] "=Tr"(dst.data()) + : [Base] "r"(src.data()), + [RowStrideBytes] "r"(src.GetStrideBytes(3)), [DataType] "i"(type_traits::TypeCode), - [VCOL] "ri"(valid_col), [VROW] "ri"(valid_row) - : "memory"); + [SizeCode] "i"(cube_shape::TilesizeCode), + [VCOL] "r"(valid_col), [VROW] "r"(valid_row) + : "memory"); } } else if constexpr (cube_shape::BFractal == BLayout::CubeM16) { - asm volatile( - "BSTART.TLSU TSTORE, %D[DataType]\n" - "B.DATR M162ND.normal, Null\n" + if constexpr (cube_shape::ValidCol > 0 && cube_shape::ValidRow > 0) { +asm volatile( + "BSTART.TLSU TLOAD, %D[DataType]\n" + "B.DATR ND2M16.normal, Zero\n" "B.DIM zero, %c[VCOL], ->lb0\n" "B.DIM zero, %c[VROW], ->lb1\n" - "B.IOT %[Src], mask=1111, last\n" + "B.IOT mask=1111, last, ->%[Dst]<%Z[SizeCode]>\n" "B.IOR [%[Base],%[RowStrideBytes]], []\n" - : - : [Base] "r"(dst.data()), [Src] "Tr"(src.data()), - [RowStrideBytes] "r"(dst.GetStrideBytes(3)), + : [Dst] "=Tr"(dst.data()) + : [Base] "r"(src.data()), + [RowStrideBytes] "r"(src.GetStrideBytes(3)), [DataType] "i"(type_traits::TypeCode), - [VCOL] "ri"(valid_col), [VROW] "ri"(valid_row) - : "memory"); - } else { // CubeN8 - asm volatile( - "BSTART.TLSU TSTORE, %D[DataType]\n" - "B.DATR N82ND.normal, Null\n" + [SizeCode] "i"(cube_shape::TilesizeCode), + [VCOL] "i"(cube_shape::ValidCol), [VROW] "i"(cube_shape::ValidRow) + : "memory"); } + else if constexpr (cube_shape::ValidCol > 0 && cube_shape::ValidRow < 0) { +asm volatile( + "BSTART.TLSU TLOAD, %D[DataType]\n" + "B.DATR ND2M16.normal, Zero\n" "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.IOT %[Src], mask=1111, last\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.IOT mask=1111, last, ->%[Dst]<%Z[SizeCode]>\n" "B.IOR [%[Base],%[RowStrideBytes]], []\n" - : - : [Base] "r"(dst.data()), [Src] "Tr"(src.data()), - [RowStrideBytes] "r"(dst.GetStrideBytes(3)), + : [Dst] "=Tr"(dst.data()) + : [Base] "r"(src.data()), + [RowStrideBytes] "r"(src.GetStrideBytes(3)), [DataType] "i"(type_traits::TypeCode), - [VCOL] "ri"(valid_col), [VROW] "ri"(valid_row) - : "memory"); - } -} - -// Unified transport entry points. CUBE Tiles require the explicit ND<->CELL -// layout conversion implemented by TLOAD_CUBE/TSTORE_CUBE; dispatching from -// the Tile layout keeps the common kernel spelling uniform without removing + [SizeCode] "i"(cube_shape::TilesizeCode), + [VCOL] "i"(cube_shape::ValidCol), [VROW] "r"(valid_row) + : "memory"); } + else if constexpr (cube_shape::ValidCol < 0 && cube_shape::ValidRow > 0) { +asm volatile( + "BSTART.TLSU TLOAD, %D[DataType]\n" + "B.DATR ND2M16.normal, Zero\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.IOT mask=1111, last, ->%[Dst]<%Z[SizeCode]>\n" + "B.IOR [%[Base],%[RowStrideBytes]], []\n" + : [Dst] "=Tr"(dst.data()) + : [Base] "r"(src.data()), + [RowStrideBytes] "r"(src.GetStrideBytes(3)), + [DataType] "i"(type_traits::TypeCode), + [SizeCode] "i"(cube_shape::TilesizeCode), + [VCOL] "r"(valid_col), [VROW] "i"(cube_shape::ValidRow) + : "memory"); } + else { +asm volatile( + "BSTART.TLSU TLOAD, %D[DataType]\n" + "B.DATR ND2M16.normal, Zero\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.IOT mask=1111, last, ->%[Dst]<%Z[SizeCode]>\n" + "B.IOR [%[Base],%[RowStrideBytes]], []\n" + : [Dst] "=Tr"(dst.data()) + : [Base] "r"(src.data()), + [RowStrideBytes] "r"(src.GetStrideBytes(3)), + [DataType] "i"(type_traits::TypeCode), + [SizeCode] "i"(cube_shape::TilesizeCode), + [VCOL] "r"(valid_col), [VROW] "r"(valid_row) + : "memory"); } + } else { // CubeN8 + if constexpr (cube_shape::ValidCol > 0 && cube_shape::ValidRow > 0) { +asm volatile( + "BSTART.TLSU TLOAD, %D[DataType]\n" + "B.DATR ND2N8.normal, Zero\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.IOT mask=1111, last, ->%[Dst]<%Z[SizeCode]>\n" + "B.IOR [%[Base],%[RowStrideBytes]], []\n" + : [Dst] "=Tr"(dst.data()) + : [Base] "r"(src.data()), + [RowStrideBytes] "r"(src.GetStrideBytes(3)), + [DataType] "i"(type_traits::TypeCode), + [SizeCode] "i"(cube_shape::TilesizeCode), + [VCOL] "i"(cube_shape::ValidCol), [VROW] "i"(cube_shape::ValidRow) + : "memory"); } + else if constexpr (cube_shape::ValidCol > 0 && cube_shape::ValidRow < 0) { +asm volatile( + "BSTART.TLSU TLOAD, %D[DataType]\n" + "B.DATR ND2N8.normal, Zero\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.IOT mask=1111, last, ->%[Dst]<%Z[SizeCode]>\n" + "B.IOR [%[Base],%[RowStrideBytes]], []\n" + : [Dst] "=Tr"(dst.data()) + : [Base] "r"(src.data()), + [RowStrideBytes] "r"(src.GetStrideBytes(3)), + [DataType] "i"(type_traits::TypeCode), + [SizeCode] "i"(cube_shape::TilesizeCode), + [VCOL] "i"(cube_shape::ValidCol), [VROW] "r"(valid_row) + : "memory"); } + else if constexpr (cube_shape::ValidCol < 0 && cube_shape::ValidRow > 0) { +asm volatile( + "BSTART.TLSU TLOAD, %D[DataType]\n" + "B.DATR ND2N8.normal, Zero\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.IOT mask=1111, last, ->%[Dst]<%Z[SizeCode]>\n" + "B.IOR [%[Base],%[RowStrideBytes]], []\n" + : [Dst] "=Tr"(dst.data()) + : [Base] "r"(src.data()), + [RowStrideBytes] "r"(src.GetStrideBytes(3)), + [DataType] "i"(type_traits::TypeCode), + [SizeCode] "i"(cube_shape::TilesizeCode), + [VCOL] "r"(valid_col), [VROW] "i"(cube_shape::ValidRow) + : "memory"); } + else { +asm volatile( + "BSTART.TLSU TLOAD, %D[DataType]\n" + "B.DATR ND2N8.normal, Zero\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.IOT mask=1111, last, ->%[Dst]<%Z[SizeCode]>\n" + "B.IOR [%[Base],%[RowStrideBytes]], []\n" + : [Dst] "=Tr"(dst.data()) + : [Base] "r"(src.data()), + [RowStrideBytes] "r"(src.GetStrideBytes(3)), + [DataType] "i"(type_traits::TypeCode), + [SizeCode] "i"(cube_shape::TilesizeCode), + [VCOL] "r"(valid_col), [VROW] "r"(valid_row) + : "memory"); } + } +} + +// PTO ISA 0.58.3 persistent Local CUBE CELL -> GM transport. The source +// descriptor survives the operation and only its logical valid rectangle is +// exported. +template + requires(cube_shape::IsCubeLayout) +void TSTORE_CUBE(gm_shape &dst, const cube_shape &src) { + static_assert(std::is_same_v, + "TSTORE_CUBE requires matching GM and CUBE dtypes"); + static_assert(cube_shape::IsValidActiveSize, + "TSTORE_CUBE Local CUBE capacity must be 128 B..256 KiB"); + const size_t valid_col = src.GetValidCol(); + const size_t valid_row = src.GetValidRow(); + // Persistent CUBE cell layout selects the canonical Local->GM transport + // selector (PTO-ISA ADR-0070); store padding is Null. Selector text is + // constant per layout class, not an integer immediate. + if constexpr (cube_shape::BFractal == BLayout::CubeM32) { + if constexpr (cube_shape::ValidCol > 0 && cube_shape::ValidRow > 0) { +asm volatile( + "BSTART.TLSU TSTORE, %D[DataType]\n" + "B.DATR M322ND.normal, Null\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.IOT %[Src], mask=1111, last\n" + "B.IOR [%[Base],%[RowStrideBytes]], []\n" + : + : [Base] "r"(dst.data()), [Src] "Tr"(src.data()), + [RowStrideBytes] "r"(dst.GetStrideBytes(3)), + [DataType] "i"(type_traits::TypeCode), + [VCOL] "i"(cube_shape::ValidCol), [VROW] "i"(cube_shape::ValidRow) + : "memory"); } + else if constexpr (cube_shape::ValidCol > 0 && cube_shape::ValidRow < 0) { +asm volatile( + "BSTART.TLSU TSTORE, %D[DataType]\n" + "B.DATR M322ND.normal, Null\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.IOT %[Src], mask=1111, last\n" + "B.IOR [%[Base],%[RowStrideBytes]], []\n" + : + : [Base] "r"(dst.data()), [Src] "Tr"(src.data()), + [RowStrideBytes] "r"(dst.GetStrideBytes(3)), + [DataType] "i"(type_traits::TypeCode), + [VCOL] "i"(cube_shape::ValidCol), [VROW] "r"(valid_row) + : "memory"); } + else if constexpr (cube_shape::ValidCol < 0 && cube_shape::ValidRow > 0) { +asm volatile( + "BSTART.TLSU TSTORE, %D[DataType]\n" + "B.DATR M322ND.normal, Null\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.IOT %[Src], mask=1111, last\n" + "B.IOR [%[Base],%[RowStrideBytes]], []\n" + : + : [Base] "r"(dst.data()), [Src] "Tr"(src.data()), + [RowStrideBytes] "r"(dst.GetStrideBytes(3)), + [DataType] "i"(type_traits::TypeCode), + [VCOL] "r"(valid_col), [VROW] "i"(cube_shape::ValidRow) + : "memory"); } + else { +asm volatile( + "BSTART.TLSU TSTORE, %D[DataType]\n" + "B.DATR M322ND.normal, Null\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.IOT %[Src], mask=1111, last\n" + "B.IOR [%[Base],%[RowStrideBytes]], []\n" + : + : [Base] "r"(dst.data()), [Src] "Tr"(src.data()), + [RowStrideBytes] "r"(dst.GetStrideBytes(3)), + [DataType] "i"(type_traits::TypeCode), + [VCOL] "r"(valid_col), [VROW] "r"(valid_row) + : "memory"); } + } else if constexpr (cube_shape::BFractal == BLayout::CubeM16) { + if constexpr (cube_shape::ValidCol > 0 && cube_shape::ValidRow > 0) { +asm volatile( + "BSTART.TLSU TSTORE, %D[DataType]\n" + "B.DATR M162ND.normal, Null\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.IOT %[Src], mask=1111, last\n" + "B.IOR [%[Base],%[RowStrideBytes]], []\n" + : + : [Base] "r"(dst.data()), [Src] "Tr"(src.data()), + [RowStrideBytes] "r"(dst.GetStrideBytes(3)), + [DataType] "i"(type_traits::TypeCode), + [VCOL] "i"(cube_shape::ValidCol), [VROW] "i"(cube_shape::ValidRow) + : "memory"); } + else if constexpr (cube_shape::ValidCol > 0 && cube_shape::ValidRow < 0) { +asm volatile( + "BSTART.TLSU TSTORE, %D[DataType]\n" + "B.DATR M162ND.normal, Null\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.IOT %[Src], mask=1111, last\n" + "B.IOR [%[Base],%[RowStrideBytes]], []\n" + : + : [Base] "r"(dst.data()), [Src] "Tr"(src.data()), + [RowStrideBytes] "r"(dst.GetStrideBytes(3)), + [DataType] "i"(type_traits::TypeCode), + [VCOL] "i"(cube_shape::ValidCol), [VROW] "r"(valid_row) + : "memory"); } + else if constexpr (cube_shape::ValidCol < 0 && cube_shape::ValidRow > 0) { +asm volatile( + "BSTART.TLSU TSTORE, %D[DataType]\n" + "B.DATR M162ND.normal, Null\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.IOT %[Src], mask=1111, last\n" + "B.IOR [%[Base],%[RowStrideBytes]], []\n" + : + : [Base] "r"(dst.data()), [Src] "Tr"(src.data()), + [RowStrideBytes] "r"(dst.GetStrideBytes(3)), + [DataType] "i"(type_traits::TypeCode), + [VCOL] "r"(valid_col), [VROW] "i"(cube_shape::ValidRow) + : "memory"); } + else { +asm volatile( + "BSTART.TLSU TSTORE, %D[DataType]\n" + "B.DATR M162ND.normal, Null\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.IOT %[Src], mask=1111, last\n" + "B.IOR [%[Base],%[RowStrideBytes]], []\n" + : + : [Base] "r"(dst.data()), [Src] "Tr"(src.data()), + [RowStrideBytes] "r"(dst.GetStrideBytes(3)), + [DataType] "i"(type_traits::TypeCode), + [VCOL] "r"(valid_col), [VROW] "r"(valid_row) + : "memory"); } + } else { // CubeN8 + if constexpr (cube_shape::ValidCol > 0 && cube_shape::ValidRow > 0) { +asm volatile( + "BSTART.TLSU TSTORE, %D[DataType]\n" + "B.DATR N82ND.normal, Null\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.IOT %[Src], mask=1111, last\n" + "B.IOR [%[Base],%[RowStrideBytes]], []\n" + : + : [Base] "r"(dst.data()), [Src] "Tr"(src.data()), + [RowStrideBytes] "r"(dst.GetStrideBytes(3)), + [DataType] "i"(type_traits::TypeCode), + [VCOL] "i"(cube_shape::ValidCol), [VROW] "i"(cube_shape::ValidRow) + : "memory"); } + else if constexpr (cube_shape::ValidCol > 0 && cube_shape::ValidRow < 0) { +asm volatile( + "BSTART.TLSU TSTORE, %D[DataType]\n" + "B.DATR N82ND.normal, Null\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.IOT %[Src], mask=1111, last\n" + "B.IOR [%[Base],%[RowStrideBytes]], []\n" + : + : [Base] "r"(dst.data()), [Src] "Tr"(src.data()), + [RowStrideBytes] "r"(dst.GetStrideBytes(3)), + [DataType] "i"(type_traits::TypeCode), + [VCOL] "i"(cube_shape::ValidCol), [VROW] "r"(valid_row) + : "memory"); } + else if constexpr (cube_shape::ValidCol < 0 && cube_shape::ValidRow > 0) { +asm volatile( + "BSTART.TLSU TSTORE, %D[DataType]\n" + "B.DATR N82ND.normal, Null\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.IOT %[Src], mask=1111, last\n" + "B.IOR [%[Base],%[RowStrideBytes]], []\n" + : + : [Base] "r"(dst.data()), [Src] "Tr"(src.data()), + [RowStrideBytes] "r"(dst.GetStrideBytes(3)), + [DataType] "i"(type_traits::TypeCode), + [VCOL] "r"(valid_col), [VROW] "i"(cube_shape::ValidRow) + : "memory"); } + else { +asm volatile( + "BSTART.TLSU TSTORE, %D[DataType]\n" + "B.DATR N82ND.normal, Null\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.IOT %[Src], mask=1111, last\n" + "B.IOR [%[Base],%[RowStrideBytes]], []\n" + : + : [Base] "r"(dst.data()), [Src] "Tr"(src.data()), + [RowStrideBytes] "r"(dst.GetStrideBytes(3)), + [DataType] "i"(type_traits::TypeCode), + [VCOL] "r"(valid_col), [VROW] "r"(valid_row) + : "memory"); } + } +} + +// Unified transport entry points. CUBE Tiles require the explicit ND<->CELL +// layout conversion implemented by TLOAD_CUBE/TSTORE_CUBE; dispatching from +// the Tile layout keeps the common kernel spelling uniform without removing // the explicit expert interfaces. template requires(cube_shape::IsCubeLayout) @@ -2358,7 +3039,8 @@ PTO_SHARED_INLINE void TSTORE(gm_shape &dst, const SharedTileT &src) { "Shared TSTORE source size must be 128 B..256 KB (SizeCode=1..12)"); const size_t valid_col = src.GetValidCol(); const size_t valid_row = src.GetValidRow(); - asm volatile( + if constexpr (SharedTileT::ValidCol > 0 && SharedTileT::ValidRow > 0) { +asm volatile( "BSTART.TLSU TSTORE, %D[SrcType]\n" "B.DIM zero, %c[VCOL], ->lb0\n" "B.DIM zero, %c[VROW], ->lb1\n" @@ -2368,10 +3050,55 @@ PTO_SHARED_INLINE void TSTORE(gm_shape &dst, const SharedTileT &src) { : : [d0] "r"(dst.data()), [Shared] "Sr"(src.handle()), [SrcType] "i"(type_traits::TypeCode), - [VCOL] "ri"(valid_col), [VROW] "ri"(valid_row), + [VCOL] "i"(SharedTileT::ValidCol), [VROW] "i"(SharedTileT::ValidRow), [COL] "i"(LocalType::Cols), [GmStride] "r"(dst.GetStrideBytes(3)) - : "memory"); + : "memory"); } + else if constexpr (SharedTileT::ValidCol > 0 && SharedTileT::ValidRow < 0) { +asm volatile( + "BSTART.TLSU TSTORE, %D[SrcType]\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + "B.IOS %S[Shared], mask=1111\n" + "B.IOR [%[d0],%[GmStride]], []\n" + : + : [d0] "r"(dst.data()), [Shared] "Sr"(src.handle()), + [SrcType] "i"(type_traits::TypeCode), + [VCOL] "i"(SharedTileT::ValidCol), [VROW] "r"(valid_row), + [COL] "i"(LocalType::Cols), + [GmStride] "r"(dst.GetStrideBytes(3)) + : "memory"); } + else if constexpr (SharedTileT::ValidCol < 0 && SharedTileT::ValidRow > 0) { +asm volatile( + "BSTART.TLSU TSTORE, %D[SrcType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + "B.IOS %S[Shared], mask=1111\n" + "B.IOR [%[d0],%[GmStride]], []\n" + : + : [d0] "r"(dst.data()), [Shared] "Sr"(src.handle()), + [SrcType] "i"(type_traits::TypeCode), + [VCOL] "r"(valid_col), [VROW] "i"(SharedTileT::ValidRow), + [COL] "i"(LocalType::Cols), + [GmStride] "r"(dst.GetStrideBytes(3)) + : "memory"); } + else { +asm volatile( + "BSTART.TLSU TSTORE, %D[SrcType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + "B.IOS %S[Shared], mask=1111\n" + "B.IOR [%[d0],%[GmStride]], []\n" + : + : [d0] "r"(dst.data()), [Shared] "Sr"(src.handle()), + [SrcType] "i"(type_traits::TypeCode), + [VCOL] "r"(valid_col), [VROW] "r"(valid_row), + [COL] "i"(LocalType::Cols), + [GmStride] "r"(dst.GetStrideBytes(3)) + : "memory"); } } // TSTORE.SPART: Shared Tile -> GM on an explicit nonzero PE subset @@ -2392,7 +3119,8 @@ PTO_SHARED_INLINE void TSTORE_PART(gm_shape &dst, const SharedTileT &src) { "Shared TSTORE.SPART source size must be 128 B..256 KB (SizeCode=1..12)"); const size_t valid_col = src.GetValidCol(); const size_t valid_row = src.GetValidRow(); - asm volatile( + if constexpr (SharedTileT::ValidCol > 0 && SharedTileT::ValidRow > 0) { +asm volatile( "BSTART.TLSU TSTORE.SPART, %D[SrcType]\n" "B.DIM zero, %c[VCOL], ->lb0\n" "B.DIM zero, %c[VROW], ->lb1\n" @@ -2403,35 +3131,101 @@ PTO_SHARED_INLINE void TSTORE_PART(gm_shape &dst, const SharedTileT &src) { : [d0] "r"(dst.data()), [Shared] "Sr"(src.handle()), [PEMask] "i"(PEMask), [SrcType] "i"(type_traits::TypeCode), - [VCOL] "ri"(valid_col), [VROW] "ri"(valid_row), + [VCOL] "i"(SharedTileT::ValidCol), [VROW] "i"(SharedTileT::ValidRow), [COL] "i"(LocalType::Cols), [GmStride] "r"(dst.GetStrideBytes(3)) - : "memory"); -} - -// TPREFETCH: request GM lines into cache without a Tile destination (PTO -// 0.58.1 TLSU function 3). Implicit PE -// participation 1111, no B.IOT/B.IOS members. Omitted LB0/LB1 default to one -// and omitted LB2 to the resolved ValidCol; we pass the caller's valid shape -// and the GM row length (logical elements) through B.DIM, and the GM base + -// logical row stride through B.IOR. -template -void TPREFETCH(const gm_shape &src, uint32_t valid_col, uint32_t valid_row) { - const size_t rowStride = src.GetStride(3); - const size_t physicalCol = - gm_shape::Cols == DYNAMIC ? rowStride : gm_shape::Cols; - asm volatile( - "BSTART.TLSU TPREFETCH, %D[DataType]\n" + : "memory"); } + else if constexpr (SharedTileT::ValidCol > 0 && SharedTileT::ValidRow < 0) { +asm volatile( + "BSTART.TLSU TSTORE.SPART, %D[SrcType]\n" "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.DIM %[Col], 0, ->lb2\n" - "B.IOR [%[Base], %[Stride]], []\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + PTO_PE_MASK_ASM("B.IOS %S[Shared], mask=", "\n") + "B.IOR [%[d0],%[GmStride]], []\n" : - : [Base] "r"(src.data()), [Stride] "r"(rowStride), - [DataType] "i"(type_traits::TypeCode), - [VCOL] "ri"(valid_col), [VROW] "ri"(valid_row), - [Col] "r"(physicalCol) - : "memory"); + : [d0] "r"(dst.data()), [Shared] "Sr"(src.handle()), + [PEMask] "i"(PEMask), + [SrcType] "i"(type_traits::TypeCode), + [VCOL] "i"(SharedTileT::ValidCol), [VROW] "r"(valid_row), + [COL] "i"(LocalType::Cols), + [GmStride] "r"(dst.GetStrideBytes(3)) + : "memory"); } + else if constexpr (SharedTileT::ValidCol < 0 && SharedTileT::ValidRow > 0) { +asm volatile( + "BSTART.TLSU TSTORE.SPART, %D[SrcType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + PTO_PE_MASK_ASM("B.IOS %S[Shared], mask=", "\n") + "B.IOR [%[d0],%[GmStride]], []\n" + : + : [d0] "r"(dst.data()), [Shared] "Sr"(src.handle()), + [PEMask] "i"(PEMask), + [SrcType] "i"(type_traits::TypeCode), + [VCOL] "r"(valid_col), [VROW] "i"(SharedTileT::ValidRow), + [COL] "i"(LocalType::Cols), + [GmStride] "r"(dst.GetStrideBytes(3)) + : "memory"); } + else { +asm volatile( + "BSTART.TLSU TSTORE.SPART, %D[SrcType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[COL], ->lb2\n" + PTO_PE_MASK_ASM("B.IOS %S[Shared], mask=", "\n") + "B.IOR [%[d0],%[GmStride]], []\n" + : + : [d0] "r"(dst.data()), [Shared] "Sr"(src.handle()), + [PEMask] "i"(PEMask), + [SrcType] "i"(type_traits::TypeCode), + [VCOL] "r"(valid_col), [VROW] "r"(valid_row), + [COL] "i"(LocalType::Cols), + [GmStride] "r"(dst.GetStrideBytes(3)) + : "memory"); } +} + +// TPREFETCH: request GM lines into cache without a Tile destination (PTO +// 0.58.1 TLSU function 3). Implicit PE +// participation 1111, no B.IOT/B.IOS members. Omitted LB0/LB1 default to one +// and omitted LB2 to the resolved ValidCol; we pass the caller's valid shape +// and the GM row length (logical elements) through B.DIM, and the GM base + +// logical row stride through B.IOR. +template +void TPREFETCH(const gm_shape &src, uint32_t valid_col, uint32_t valid_row) { + const size_t rowStride = src.GetStride(3); + const size_t physicalCol = + gm_shape::Cols == DYNAMIC ? rowStride : gm_shape::Cols; + // LB2 carries the GM row length. When the GM shape is statically known it + // lowers to an immediate (C.B.DIMI); only dynamic GM shapes keep the + // register form. LB0/LB1 are runtime function arguments by contract. + if constexpr (gm_shape::Cols != DYNAMIC) { + asm volatile( + "BSTART.TLSU TPREFETCH, %D[DataType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Base], %[Stride]], []\n" + : + : [Base] "r"(src.data()), [Stride] "r"(rowStride), + [DataType] "i"(type_traits::TypeCode), + [VCOL] "r"(valid_col), [VROW] "r"(valid_row), + [Col] "i"(gm_shape::Cols) + : "memory"); + } else { + asm volatile( + "BSTART.TLSU TPREFETCH, %D[DataType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM %[Col], 0, ->lb2\n" + "B.IOR [%[Base], %[Stride]], []\n" + : + : [Base] "r"(src.data()), [Stride] "r"(rowStride), + [DataType] "i"(type_traits::TypeCode), + [VCOL] "r"(valid_col), [VROW] "r"(valid_row), + [Col] "r"(physicalCol) + : "memory"); + } } // MGATHER_CAS: atomic compare-and-swap at byte displacements (PTO ISA 0.58.3 @@ -2475,7 +3269,8 @@ void MGATHER_CAS(DstTile &observedOld, uint64_t base, DstTile::Rows == ExpectedTile::Rows && DstTile::Cols == ExpectedTile::Cols, "MGATHER_CAS tiles must match the resolved ValidRow x ValidCol"); - asm volatile( + if constexpr (DstTile::ValidCol > 0 && DstTile::ValidRow > 0) { +asm volatile( "BSTART.TLSU MGATHER.CAS, %D[DataType]\n" "B.DIM zero, %c[VCOL], ->lb0\n" "B.DIM zero, %c[VROW], ->lb1\n" @@ -2488,10 +3283,64 @@ void MGATHER_CAS(DstTile &observedOld, uint64_t base, [Rep] "Tr"(replacement.data()), [Base] "r"(base), [DataType] "i"(type_traits::TypeCode), - [VCOL] "ri"(validCol), [VROW] "ri"(validRow), + [VCOL] "i"(DstTile::ValidCol), [VROW] "i"(DstTile::ValidRow), [Col] "i"(DstTile::Cols), [DstSize] "i"(DstTile::TilesizeCode) - : "memory"); + : "memory"); } + else if constexpr (DstTile::ValidCol > 0 && DstTile::ValidRow < 0) { +asm volatile( + "BSTART.TLSU MGATHER.CAS, %D[DataType]\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOT %[Idx], %[Exp], mask=1111\n" + "B.IOT %[Rep], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + "B.IOR [%[Base]], []\n" + : [Dst] "=&Tr"(observedOld.data()) + : [Idx] "Tr"(byteDisplacements.data()), [Exp] "Tr"(expected.data()), + [Rep] "Tr"(replacement.data()), + [Base] "r"(base), + [DataType] "i"(type_traits::TypeCode), + [VCOL] "i"(DstTile::ValidCol), [VROW] "r"(validRow), + [Col] "i"(DstTile::Cols), + [DstSize] "i"(DstTile::TilesizeCode) + : "memory"); } + else if constexpr (DstTile::ValidCol < 0 && DstTile::ValidRow > 0) { +asm volatile( + "BSTART.TLSU MGATHER.CAS, %D[DataType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOT %[Idx], %[Exp], mask=1111\n" + "B.IOT %[Rep], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + "B.IOR [%[Base]], []\n" + : [Dst] "=&Tr"(observedOld.data()) + : [Idx] "Tr"(byteDisplacements.data()), [Exp] "Tr"(expected.data()), + [Rep] "Tr"(replacement.data()), + [Base] "r"(base), + [DataType] "i"(type_traits::TypeCode), + [VCOL] "r"(validCol), [VROW] "i"(DstTile::ValidRow), + [Col] "i"(DstTile::Cols), + [DstSize] "i"(DstTile::TilesizeCode) + : "memory"); } + else { +asm volatile( + "BSTART.TLSU MGATHER.CAS, %D[DataType]\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOT %[Idx], %[Exp], mask=1111\n" + "B.IOT %[Rep], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + "B.IOR [%[Base]], []\n" + : [Dst] "=&Tr"(observedOld.data()) + : [Idx] "Tr"(byteDisplacements.data()), [Exp] "Tr"(expected.data()), + [Rep] "Tr"(replacement.data()), + [Base] "r"(base), + [DataType] "i"(type_traits::TypeCode), + [VCOL] "r"(validCol), [VROW] "r"(validRow), + [Col] "i"(DstTile::Cols), + [DstSize] "i"(DstTile::TilesizeCode) + : "memory"); } } // Low-level v5 GMOV. All four PEs must reach the same dynamic instance; @@ -2679,11 +3528,11 @@ namespace pto_matmul_detail { "B.DATR %D[DataTypeB], byte0, Null, RNE, NOSAT\n" \ ".endif\n" EXTRA_ATTRS \ "B.DIM %[M], 0, ->lb0\n" \ - "B.DIM %[N], 0, ->lb1\n" \ - "B.DIM %[K], 0, ->lb2\n" + "B.DIM zero, %c[N], ->lb1\n" \ + "B.DIM zero, %c[K], ->lb2\n" #define PTO_MATMUL_COMMON_INPUTS(DstType, AType, BType, MValue, NValue, KValue) \ - [M] "r"(MValue), [N] "r"(NValue), [K] "r"(KValue), \ + [M] "r"(MValue), [N] "i"(NValue), [K] "i"(KValue), \ [CCTRL] "i"(static_cast(Attr.CubeCtrl)), \ [DataTypeA] "i"(type_traits::TypeCode), \ [DataTypeB] "i"(type_traits::TypeCode), \ @@ -7074,73 +7923,63 @@ void TADD(tile_shape &dst, tile_shape &src0, tile_shape &src1) { "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { - asm volatile( + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( "BSTART.TEPL 0, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src0.GetValidCol()), - "ri"(src0.GetValidRow()), + "i"(tile_shape::ValidCol), + [src0____dimrow] "r"(src0.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TSUB: dst = src0 - src1 -template -void TSUB(tile_shape &dst, tile_shape &src0, tile_shape &src1) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { asm volatile( - "BSTART.TEPL 1, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 0, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape::ValidCol), + [src0____dimcol] "r"(src0.GetValidCol()), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else { asm volatile( - "BSTART.TEPL 1, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 0, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src0.GetValidCol()), - "ri"(src0.GetValidRow()), + [src0____dimcol] "r"(src0.GetValidCol()), + [src0____dimrow] "r"(src0.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } } -// TMUL: dst = src0 * src1 +// TSUB: dst = src0 - src1 template -void TMUL(tile_shape &dst, tile_shape &src0, tile_shape &src1) { +void TSUB(tile_shape &dst, tile_shape &src0, tile_shape &src1) { if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - asm volatile( - "BSTART.TEPL 2, %D1\n" + asm volatile( + "BSTART.TEPL 1, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" @@ -7154,75 +7993,63 @@ void TMUL(tile_shape &dst, tile_shape &src0, tile_shape &src1) { "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { - const size_t valid_col = src0.GetValidCol(); - const size_t valid_row = src0.GetValidRow(); - asm volatile( - "BSTART.TEPL 2, %D1\n" + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 1, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(valid_col), - "ri"(valid_row), + "i"(tile_shape::ValidCol), + [src0____dimrow] "r"(src0.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TDIV: dst = src0 / src1 -template -void TDIV(tile_shape &dst, tile_shape &src0, tile_shape &src1) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { asm volatile( - "BSTART.TEPL 3, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 1, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape::ValidCol), + [src0____dimcol] "r"(src0.GetValidCol()), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else { asm volatile( - "BSTART.TEPL 3, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 1, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src0.GetValidCol()), - "ri"(src0.GetValidRow()), + [src0____dimcol] "r"(src0.GetValidCol()), + [src0____dimrow] "r"(src0.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } } -// TREM: dst = rem(src0, src1) +// TMUL: dst = src0 * src1 template -void TREM(tile_shape &dst, tile_shape &src0, tile_shape &src1) { +void TMUL(tile_shape &dst, tile_shape &src0, tile_shape &src1) { if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - asm volatile( - "BSTART.TEPL 4, %D1\n" + asm volatile( + "BSTART.TEPL 2, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" @@ -7236,81 +8063,67 @@ void TREM(tile_shape &dst, tile_shape &src0, tile_shape &src1) { "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + const size_t valid_col = src0.GetValidCol(); asm volatile( - "BSTART.TEPL 4, %D1\n" + "BSTART.TEPL 2, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[valid_col], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src0.GetValidCol()), - "ri"(src0.GetValidRow()), + "i"(tile_shape::ValidCol), + [valid_col] "r"(src0.GetValidCol()), "i"(tile_shape::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TFMOD: dst = fmod(src0, src1) -template -void TFMOD(tile_shape &dst, tile_shape &src0, tile_shape &src1) { - static_assert(pto_dependent_false_v, - "TFMOD is retired in PTO ISA 0.58.3; no active replacement. Remove the call or migrate to the active surface."); -} - - -// TAND: dst = src0 & src1 -template -void TAND(tile_shape &dst, tile_shape &src0, tile_shape &src1) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - asm volatile( - "BSTART.TEPL 6, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + const size_t valid_col = src0.GetValidCol(); + asm volatile( + "BSTART.TEPL 2, %D1\n" + "B.DIM %[valid_col], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape::ValidCol), + [valid_col] "r"(src0.GetValidCol()), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else { + const size_t valid_col = src0.GetValidCol(); + const size_t valid_row = src0.GetValidRow(); asm volatile( - "BSTART.TEPL 6, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 2, %D1\n" + "B.DIM %[valid_col], 0, ->lb0\n" + "B.DIM %[valid_row], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src0.GetValidCol()), - "ri"(src0.GetValidRow()), + [valid_col] "r"(valid_col), + [valid_row] "r"(valid_row), "i"(tile_shape::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } } -// TOR: dst = src0 | src1 +// TDIV: dst = src0 / src1 template -void TOR(tile_shape &dst, tile_shape &src0, tile_shape &src1) { +void TDIV(tile_shape &dst, tile_shape &src0, tile_shape &src1) { if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { asm volatile( - "BSTART.TEPL 7, %D1\n" + "BSTART.TEPL 3, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" @@ -7324,73 +8137,63 @@ void TOR(tile_shape &dst, tile_shape &src0, tile_shape &src1) { "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { - asm volatile( - "BSTART.TEPL 7, %D1\n" + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 3, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src0.GetValidCol()), - "ri"(src0.GetValidRow()), + "i"(tile_shape::ValidCol), + [src0____dimrow] "r"(src0.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TXOR: dst = src0 ^ src1 -template -void TXOR(tile_shape &dst, tile_shape &src0, tile_shape &src1) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { asm volatile( - "BSTART.TEPL 8, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 3, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape::ValidCol), + [src0____dimcol] "r"(src0.GetValidCol()), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else { asm volatile( - "BSTART.TEPL 8, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 3, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src0.GetValidCol()), - "ri"(src0.GetValidRow()), + [src0____dimcol] "r"(src0.GetValidCol()), + [src0____dimrow] "r"(src0.GetValidRow()), "i"(tile_shape::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } } -// TSHL: dst = src0 << src1 +// TREM: dst = rem(src0, src1) template -void TSHL(tile_shape &dst, tile_shape &src0, tile_shape &src1) { +void TREM(tile_shape &dst, tile_shape &src0, tile_shape &src1) { if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { asm volatile( - "BSTART.TEPL 9, %D1\n" + "BSTART.TEPL 4, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" @@ -7404,33 +8207,71 @@ void TSHL(tile_shape &dst, tile_shape &src0, tile_shape &src1) { "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { - asm volatile( - "BSTART.TEPL 9, %D1\n" + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 4, %D1\n" "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src0____dimrow] "r"(src0.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 4, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src0.GetValidCol()), - "ri"(src0.GetValidRow()), + [src0____dimcol] "r"(src0.GetValidCol()), + "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } else { + asm volatile( + "BSTART.TEPL 4, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src0____dimcol] "r"(src0.GetValidCol()), + [src0____dimrow] "r"(src0.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } -// TSHR: dst = src0 >> src1 +// TFMOD: dst = fmod(src0, src1) template -void TSHR(tile_shape &dst, tile_shape &src0, tile_shape &src1) { +void TFMOD(tile_shape &dst, tile_shape &src0, tile_shape &src1) { + static_assert(pto_dependent_false_v, + "TFMOD is retired in PTO ISA 0.58.3; no active replacement. Remove the call or migrate to the active surface."); +} + + +// TAND: dst = src0 & src1 +template +void TAND(tile_shape &dst, tile_shape &src0, tile_shape &src1) { if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { asm volatile( - "BSTART.TEPL 10, %D1\n" + "BSTART.TEPL 6, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" @@ -7444,33 +8285,63 @@ void TSHR(tile_shape &dst, tile_shape &src0, tile_shape &src1) { "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { - asm volatile( - "BSTART.TEPL 10, %D1\n" + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 6, %D1\n" "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src0____dimrow] "r"(src0.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 6, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src0.GetValidCol()), - "ri"(src0.GetValidRow()), + [src0____dimcol] "r"(src0.GetValidCol()), + "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } else { + asm volatile( + "BSTART.TEPL 6, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src0____dimcol] "r"(src0.GetValidCol()), + [src0____dimrow] "r"(src0.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } -// TMAX: dst = max(src0, src1) +// TOR: dst = src0 | src1 template -void TMAX(tile_shape &dst, tile_shape &src0, tile_shape &src1) { +void TOR(tile_shape &dst, tile_shape &src0, tile_shape &src1) { if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { asm volatile( - "BSTART.TEPL 11, %D1\n" + "BSTART.TEPL 7, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" @@ -7484,33 +8355,63 @@ void TMAX(tile_shape &dst, tile_shape &src0, tile_shape &src1) { "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { - asm volatile( - "BSTART.TEPL 11, %D1\n" + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 7, %D1\n" "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src0____dimrow] "r"(src0.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 7, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src0.GetValidCol()), - "ri"(src0.GetValidRow()), + [src0____dimcol] "r"(src0.GetValidCol()), + "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } else { + asm volatile( + "BSTART.TEPL 7, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src0____dimcol] "r"(src0.GetValidCol()), + [src0____dimrow] "r"(src0.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } -// TMIN: dst = min(src0, src1) +// TXOR: dst = src0 ^ src1 template -void TMIN(tile_shape &dst, tile_shape &src0, tile_shape &src1) { +void TXOR(tile_shape &dst, tile_shape &src0, tile_shape &src1) { if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { asm volatile( - "BSTART.TEPL 12, %D1\n" + "BSTART.TEPL 8, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" @@ -7524,830 +8425,4642 @@ void TMIN(tile_shape &dst, tile_shape &src0, tile_shape &src1) { "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { - asm volatile( - "BSTART.TEPL 12, %D1\n" + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 8, %D1\n" "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src0____dimrow] "r"(src0.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 8, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src0.GetValidCol()), - "ri"(src0.GetValidRow()), + [src0____dimcol] "r"(src0.GetValidCol()), + "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } else { + asm volatile( + "BSTART.TEPL 8, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src0____dimcol] "r"(src0.GetValidCol()), + [src0____dimrow] "r"(src0.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } -// TCMP: compare src0 and src1, write packed predicate. The comparison mode is -// a compile-time template parameter encoded into B.DATR CMode[31:29]; the -// zero-arg form is kept as a deprecated EQ default (PTO 0.58). -template -void TCMP(tile_shape_out &dst, tile_shape_in &src0, tile_shape_in &src1) { - static_assert(is_valid_cmp_mode(Mode), "TCMP requires a valid CmpMode"); - static_assert(tile_shape_in::Rows == tile_shape_out::Rows && - tile_shape_in::Cols == tile_shape_out::Cols, - "TCMP output shape must match input shape"); - if constexpr (Mode == CmpMode::EQ) { - asm volatile( - "BSTART.TEPL 13, %D[TCode]\n" - "B.DATR Zero, EQ\n" - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.DIM zero, %c[Cols], ->lb2\n" - "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" - "" - : [D] "=Tr"(dst.data()) - : [TCode] "i"(type_traits::TypeCode), - [VCOL] "ri"(src0.GetValidCol()), - [VROW] "ri"(src0.GetValidRow()), - [Cols] "i"(tile_shape_in::Cols), - [S0] "Tr"(src0.data()), - [S1] "Tr"(src1.data()), - [TSize] "i"(tile_type_traits::TilesizeCode) - ); - } else if constexpr (Mode == CmpMode::NE) { - asm volatile( - "BSTART.TEPL 13, %D[TCode]\n" - "B.DATR Zero, NE\n" - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.DIM zero, %c[Cols], ->lb2\n" - "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" - "" - : [D] "=Tr"(dst.data()) - : [TCode] "i"(type_traits::TypeCode), - [VCOL] "ri"(src0.GetValidCol()), - [VROW] "ri"(src0.GetValidRow()), - [Cols] "i"(tile_shape_in::Cols), - [S0] "Tr"(src0.data()), - [S1] "Tr"(src1.data()), - [TSize] "i"(tile_type_traits::TilesizeCode) - ); - } else if constexpr (Mode == CmpMode::LT) { +// TSHL: dst = src0 << src1 +template +void TSHL(tile_shape &dst, tile_shape &src0, tile_shape &src1) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { asm volatile( - "BSTART.TEPL 13, %D[TCode]\n" - "B.DATR Zero, LT\n" - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.DIM zero, %c[Cols], ->lb2\n" - "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" - "" - : [D] "=Tr"(dst.data()) - : [TCode] "i"(type_traits::TypeCode), - [VCOL] "ri"(src0.GetValidCol()), - [VROW] "ri"(src0.GetValidRow()), - [Cols] "i"(tile_shape_in::Cols), - [S0] "Tr"(src0.data()), - [S1] "Tr"(src1.data()), - [TSize] "i"(tile_type_traits::TilesizeCode) - ); - } else if constexpr (Mode == CmpMode::GT) { + "BSTART.TEPL 9, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { asm volatile( - "BSTART.TEPL 13, %D[TCode]\n" - "B.DATR Zero, GT\n" - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.DIM zero, %c[Cols], ->lb2\n" - "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" - "" - : [D] "=Tr"(dst.data()) - : [TCode] "i"(type_traits::TypeCode), - [VCOL] "ri"(src0.GetValidCol()), - [VROW] "ri"(src0.GetValidRow()), - [Cols] "i"(tile_shape_in::Cols), - [S0] "Tr"(src0.data()), - [S1] "Tr"(src1.data()), - [TSize] "i"(tile_type_traits::TilesizeCode) - ); - } else if constexpr (Mode == CmpMode::LE) { + "BSTART.TEPL 9, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src0____dimrow] "r"(src0.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { asm volatile( - "BSTART.TEPL 13, %D[TCode]\n" - "B.DATR Zero, LE\n" - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.DIM zero, %c[Cols], ->lb2\n" - "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" - "" - : [D] "=Tr"(dst.data()) - : [TCode] "i"(type_traits::TypeCode), - [VCOL] "ri"(src0.GetValidCol()), - [VROW] "ri"(src0.GetValidRow()), - [Cols] "i"(tile_shape_in::Cols), - [S0] "Tr"(src0.data()), - [S1] "Tr"(src1.data()), - [TSize] "i"(tile_type_traits::TilesizeCode) - ); - } else if constexpr (Mode == CmpMode::GE) { - asm volatile( - "BSTART.TEPL 13, %D[TCode]\n" - "B.DATR Zero, GE\n" - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.DIM zero, %c[Cols], ->lb2\n" - "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" - "" - : [D] "=Tr"(dst.data()) - : [TCode] "i"(type_traits::TypeCode), - [VCOL] "ri"(src0.GetValidCol()), - [VROW] "ri"(src0.GetValidRow()), - [Cols] "i"(tile_shape_in::Cols), - [S0] "Tr"(src0.data()), - [S1] "Tr"(src1.data()), - [TSize] "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// Deprecated EQ-default form retained for old callers. -template -void TCMP(tile_shape_out &dst, tile_shape_in &src0, tile_shape_in &src1) { - TCMP(dst, src0, src1); -} - -// TPRELU: parametric ReLU with per-element slope -template -void TPRELU(tile_shape &dst, tile_shape &src0, tile_shape &src1) { - static_assert(pto_dependent_false_v, - "TPRELU is retired in PTO ISA 0.58.3; no active replacement. Remove the call or migrate to the active surface."); -} - - -// TSEL: select true_src where mask is set, otherwise preserve the prior dst. -// The false source is an explicit ISA operand, so keep dst live as an input -// and bind it with a second B.IOT before publishing the destination. -template -void TSEL(tile_shape &dst, tile_shape &mask, tile_shape &true_src) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - asm volatile( - "BSTART.TEPL 26, %D2\n" - "B.DIM zero, %c3, ->lb0\n" - "B.DIM zero, %c4, ->lb1\n" - "B.DIM zero, %c5, ->lb2\n" - "B.IOT %6, %7, mask=1111\n" - "B.IOT %1, mask=1111, last, ->%0<%Z8>\n" - "" - : [Dst] "=Tr"(dst.data()) - : [Prior] "0"(dst.data()), - [DataType] "i"(type_traits::TypeCode), - [ValidCol] "i"(tile_shape::ValidCol), - [ValidRow] "i"(tile_shape::ValidRow), - [Cols] "i"(tile_shape::Cols), - [Mask] "Tr"(mask.data()), - [True] "Tr"(true_src.data()), - [TileSize] "i"(tile_type_traits::TilesizeCode) - ); - } else { - asm volatile( - "BSTART.TEPL 26, %D2\n" - "B.DIM %3, 0, ->lb0\n" - "B.DIM %4, 0, ->lb1\n" - "B.DIM zero, %c5, ->lb2\n" - "B.IOT %6, %7, mask=1111\n" - "B.IOT %1, mask=1111, last, ->%0<%Z8>\n" - "" - : [Dst] "=Tr"(dst.data()) - : [Prior] "0"(dst.data()), - [DataType] "i"(type_traits::TypeCode), - [ValidCol] "ri"(mask.GetValidCol()), - [ValidRow] "ri"(mask.GetValidRow()), - [Cols] "i"(tile_shape::Cols), - [Mask] "Tr"(mask.data()), - [True] "Tr"(true_src.data()), - [TileSize] "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TABS: dst = |src| -template -void TABS(tile_shape_out &dst, const tile_shape_in &src) { - static_assert(std::is_same_v, - "TABS source and destination dtypes must match"); - static_assert(tile_shape_out::Rows == tile_shape_in::Rows && - tile_shape_out::Cols == tile_shape_in::Cols && - tile_shape_out::BFractal == tile_shape_in::BFractal && - tile_shape_out::SFractal == tile_shape_in::SFractal, - "TABS source and destination descriptors must match"); - if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { - asm volatile( - "BSTART.TEPL 15, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 9, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(tile_shape_in::ValidCol), - "i"(tile_shape_in::ValidRow), - "i"(tile_shape_in::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) - ); - } else { + : "i"(type_traits::TypeCode), + [src0____dimcol] "r"(src0.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { asm volatile( - "BSTART.TEPL 15, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 9, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), - "i"(tile_shape_in::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) - ); - } + : "i"(type_traits::TypeCode), + [src0____dimcol] "r"(src0.GetValidCol()), + [src0____dimrow] "r"(src0.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } -// TNOT: dst = ~src +// TSHR: dst = src0 >> src1 template -void TNOT(tile_shape &dst, tile_shape &src) { +void TSHR(tile_shape &dst, tile_shape &src0, tile_shape &src1) { if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - asm volatile( - "BSTART.TEPL 16, %D1\n" + asm volatile( + "BSTART.TEPL 10, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { - asm volatile( - "BSTART.TEPL 16, %D1\n" + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 10, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), + "i"(tile_shape::ValidCol), + [src0____dimrow] "r"(src0.GetValidRow()), "i"(tile_shape::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TNEG: dst = -src -template -void TNEG(tile_shape &dst, tile_shape &src) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - asm volatile( - "BSTART.TEPL 17, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 10, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape::ValidCol), + [src0____dimcol] "r"(src0.GetValidCol()), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else { asm volatile( - "BSTART.TEPL 17, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 10, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), + [src0____dimcol] "r"(src0.GetValidCol()), + [src0____dimrow] "r"(src0.GetValidRow()), "i"(tile_shape::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } } -// TEXP: dst = exp(src) +// TMAX: dst = max(src0, src1) template -void TEXP(tile_shape &dst, tile_shape &src) { +void TMAX(tile_shape &dst, tile_shape &src0, tile_shape &src1) { if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - asm volatile( - "BSTART.TEPL 18, %D1\n" + asm volatile( + "BSTART.TEPL 11, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { - asm volatile( - "BSTART.TEPL 18, %D1\n" + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 11, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), + "i"(tile_shape::ValidCol), + [src0____dimrow] "r"(src0.GetValidRow()), "i"(tile_shape::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TLOG: dst = log(src) -template -void TLOG(tile_shape &dst, tile_shape &src) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - asm volatile( - "BSTART.TEPL 19, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 11, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape::ValidCol), + [src0____dimcol] "r"(src0.GetValidCol()), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else { asm volatile( - "BSTART.TEPL 19, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 11, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), + [src0____dimcol] "r"(src0.GetValidCol()), + [src0____dimrow] "r"(src0.GetValidRow()), "i"(tile_shape::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } } -// TRECIP: dst = 1/src +// TMIN: dst = min(src0, src1) template -void TRECIP(tile_shape &dst, tile_shape &src) { - const size_t valid_col = src.GetValidCol(); - const size_t valid_row = src.GetValidRow(); - asm volatile( - "BSTART.TEPL 20, %D1\n" +void TMIN(tile_shape &dst, tile_shape &src0, tile_shape &src1) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 12, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "" - : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "ri"(valid_col), - "ri"(valid_row), - "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) - ); -} - -// TSQRT: dst = sqrt(src) -template -void TSQRT(tile_shape &dst, tile_shape &src) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - asm volatile( - "BSTART.TEPL 21, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" - "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) - ); - } else { - asm volatile( - "BSTART.TEPL 21, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" - "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "" - : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), - "i"(tile_shape::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TRSQRT: dst = 1/sqrt(src) -template -void TRSQRT(tile_shape &dst, tile_shape &src) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - asm volatile( - "BSTART.TEPL 22, %D1\n" + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 12, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), - "i"(tile_shape::ValidRow), - "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) - ); - } else { - asm volatile( - "BSTART.TEPL 22, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" - "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "" - : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), + [src0____dimrow] "r"(src0.GetValidRow()), "i"(tile_shape::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TRELU: dst = max(src, 0) -template -void TRELU(tile_shape &dst, tile_shape &src) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - asm volatile( - "BSTART.TEPL 23, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 12, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape::ValidCol), + [src0____dimcol] "r"(src0.GetValidCol()), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else { asm volatile( - "BSTART.TEPL 23, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 12, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), + [src0____dimcol] "r"(src0.GetValidCol()), + [src0____dimrow] "r"(src0.GetValidRow()), "i"(tile_shape::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TADDC: dst = src0 + src1 + src2 -template -void TADDC(tile_shape &dst, tile_shape &src0, tile_shape &src1, tile_shape &src2) { - static_assert(pto_dependent_false_v, - "TADDC is retired in PTO ISA 0.58.3; no active replacement. Remove the call or migrate to the active surface."); -} - - -// TSUBC: dst = src0 - src1 + src2 -template -void TSUBC(tile_shape &dst, tile_shape &src0, tile_shape &src1, tile_shape &src2) { - static_assert(pto_dependent_false_v, - "TSUBC is retired in PTO ISA 0.58.3; no active replacement. Remove the call or migrate to the active surface."); -} - - -// TCVT: elementwise type conversion (opcode 27, already has TCVT_T) -// Use TCVT_T(dst, src) for this; TCVT is aliased below for convenience. -template -void TCVT(tile_shape_out &dst, tile_shape_in &src) { - TCVT_T(dst, src); -} -//===--- TEPL Mode 1: tile-scalar elementwise ops (BSTART.TEPL) ---===// -// opcode = Mode(1) * 32 + Function = 32 + Function. - -// TADDS: dst = src + scalar -template -void TADDS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; - const size_t valid_col = src.GetValidCol(); - const size_t valid_row = src.GetValidRow(); - asm volatile( - "BSTART.TEPL 32, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" - "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7],[]\n" - "" - : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "ri"(valid_col), - "ri"(valid_row), - "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); -} - -// TSUBS: dst = src - scalar -template -void TSUBS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; - asm volatile( - "BSTART.TEPL 33, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" - "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7],[]\n" - "" - : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(tile_shape::ValidCol), - "i"(tile_shape::ValidRow), - "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } else { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; - asm volatile( - "BSTART.TEPL 33, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" - "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7],[]\n" - "" - : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), - "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } -} - -// TMULS: dst = src * scalar -template -void TMULS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; - const size_t valid_col = src.GetValidCol(); - const size_t valid_row = src.GetValidRow(); - asm volatile( - "BSTART.TEPL 34, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" - "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7],[]\n" - "" - : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "ri"(valid_col), - "ri"(valid_row), - "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); + ); } } -// TDIVS: dst = src / scalar -template -void TDIVS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; - asm volatile( - "BSTART.TEPL 35, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" - "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7],[]\n" - "" - : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(tile_shape::ValidCol), - "i"(tile_shape::ValidRow), - "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } else { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; - asm volatile( - "BSTART.TEPL 35, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" - "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" +// TCMP: compare src0 and src1, write packed predicate. The comparison mode is +// a compile-time template parameter encoded into B.DATR CMode[31:29]; the +// zero-arg form is kept as a deprecated EQ default (PTO 0.58). +template +void TCMP(tile_shape_out &dst, tile_shape_in &src0, tile_shape_in &src1) { + static_assert(is_valid_cmp_mode(Mode), "TCMP requires a valid CmpMode"); + static_assert(tile_shape_in::Rows == tile_shape_out::Rows && + tile_shape_in::Cols == tile_shape_out::Cols, + "TCMP output shape must match input shape"); + if constexpr (Mode == CmpMode::EQ) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, EQ\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, EQ\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "r"(src0.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, EQ\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src0.GetValidCol()), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + else { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, EQ\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src0.GetValidCol()), + [VROW] "r"(src0.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + } else if constexpr (Mode == CmpMode::NE) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, NE\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, NE\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "r"(src0.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, NE\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src0.GetValidCol()), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + else { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, NE\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src0.GetValidCol()), + [VROW] "r"(src0.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + } else if constexpr (Mode == CmpMode::LT) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, LT\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, LT\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "r"(src0.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, LT\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src0.GetValidCol()), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + else { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, LT\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src0.GetValidCol()), + [VROW] "r"(src0.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + } else if constexpr (Mode == CmpMode::GT) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, GT\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, GT\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "r"(src0.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, GT\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src0.GetValidCol()), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + else { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, GT\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src0.GetValidCol()), + [VROW] "r"(src0.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + } else if constexpr (Mode == CmpMode::LE) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, LE\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, LE\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "r"(src0.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, LE\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src0.GetValidCol()), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + else { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, LE\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src0.GetValidCol()), + [VROW] "r"(src0.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + } else if constexpr (Mode == CmpMode::GE) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, GE\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, GE\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "r"(src0.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, GE\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src0.GetValidCol()), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + else { +asm volatile( + "BSTART.TEPL 13, %D[TCode]\n" + "B.DATR Zero, GE\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S0], %[S1], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src0.GetValidCol()), + [VROW] "r"(src0.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S0] "Tr"(src0.data()), + [S1] "Tr"(src1.data()), + [TSize] "i"(tile_type_traits::TilesizeCode) + ); } + } +} + +// Deprecated EQ-default form retained for old callers. +template +void TCMP(tile_shape_out &dst, tile_shape_in &src0, tile_shape_in &src1) { + TCMP(dst, src0, src1); +} + +// TPRELU: parametric ReLU with per-element slope +template +void TPRELU(tile_shape &dst, tile_shape &src0, tile_shape &src1) { + static_assert(pto_dependent_false_v, + "TPRELU is retired in PTO ISA 0.58.3; no active replacement. Remove the call or migrate to the active surface."); +} + + +// TSEL: select true_src where mask is set, otherwise preserve the prior dst. +// The false source is an explicit ISA operand, so keep dst live as an input +// and bind it with a second B.IOT before publishing the destination. +template +void TSEL(tile_shape &dst, tile_shape &mask, tile_shape &true_src) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 26, %D2\n" + "B.DIM zero, %c3, ->lb0\n" + "B.DIM zero, %c4, ->lb1\n" + "B.DIM zero, %c5, ->lb2\n" + "B.IOT %6, %7, mask=1111\n" + "B.IOT %1, mask=1111, last, ->%0<%Z8>\n" + "" + : [Dst] "=Tr"(dst.data()) + : [Prior] "0"(dst.data()), + [DataType] "i"(type_traits::TypeCode), + [ValidCol] "i"(tile_shape::ValidCol), + [ValidRow] "i"(tile_shape::ValidRow), + [Cols] "i"(tile_shape::Cols), + [Mask] "Tr"(mask.data()), + [True] "Tr"(true_src.data()), + [TileSize] "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 26, %D2\n" + "B.DIM zero, %c3, ->lb0\n" + "B.DIM %[mask____dimrow], 0, ->lb1\n" + "B.DIM zero, %c5, ->lb2\n" + "B.IOT %6, %7, mask=1111\n" + "B.IOT %1, mask=1111, last, ->%0<%Z8>\n" + "" + : [Dst] "=Tr"(dst.data()) + : [Prior] "0"(dst.data()), + [DataType] "i"(type_traits::TypeCode), + [ValidCol] "i"(tile_shape::ValidCol), + [mask____dimrow] "r"(mask.GetValidRow()), + [Cols] "i"(tile_shape::Cols), + [Mask] "Tr"(mask.data()), + [True] "Tr"(true_src.data()), + [TileSize] "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 26, %D2\n" + "B.DIM %[mask____dimcol], 0, ->lb0\n" + "B.DIM zero, %c4, ->lb1\n" + "B.DIM zero, %c5, ->lb2\n" + "B.IOT %6, %7, mask=1111\n" + "B.IOT %1, mask=1111, last, ->%0<%Z8>\n" + "" + : [Dst] "=Tr"(dst.data()) + : [Prior] "0"(dst.data()), + [DataType] "i"(type_traits::TypeCode), + [mask____dimcol] "r"(mask.GetValidCol()), + [ValidRow] "i"(tile_shape::ValidRow), + [Cols] "i"(tile_shape::Cols), + [Mask] "Tr"(mask.data()), + [True] "Tr"(true_src.data()), + [TileSize] "i"(tile_type_traits::TilesizeCode) + ); } else { + asm volatile( + "BSTART.TEPL 26, %D2\n" + "B.DIM %[mask____dimcol], 0, ->lb0\n" + "B.DIM %[mask____dimrow], 0, ->lb1\n" + "B.DIM zero, %c5, ->lb2\n" + "B.IOT %6, %7, mask=1111\n" + "B.IOT %1, mask=1111, last, ->%0<%Z8>\n" + "" + : [Dst] "=Tr"(dst.data()) + : [Prior] "0"(dst.data()), + [DataType] "i"(type_traits::TypeCode), + [mask____dimcol] "r"(mask.GetValidCol()), + [mask____dimrow] "r"(mask.GetValidRow()), + [Cols] "i"(tile_shape::Cols), + [Mask] "Tr"(mask.data()), + [True] "Tr"(true_src.data()), + [TileSize] "i"(tile_type_traits::TilesizeCode) + ); } +} + +// TABS: dst = |src| +template +void TABS(tile_shape_out &dst, const tile_shape_in &src) { + static_assert(std::is_same_v, + "TABS source and destination dtypes must match"); + static_assert(tile_shape_out::Rows == tile_shape_in::Rows && + tile_shape_out::Cols == tile_shape_in::Cols && + tile_shape_out::BFractal == tile_shape_in::BFractal && + tile_shape_out::SFractal == tile_shape_in::SFractal, + "TABS source and destination descriptors must match"); + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 15, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_in::ValidCol), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 15, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_in::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 15, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { + asm volatile( + "BSTART.TEPL 15, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } +} + +// TNOT: dst = ~src +template +void TNOT(tile_shape &dst, tile_shape &src) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 16, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 16, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 16, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { + asm volatile( + "BSTART.TEPL 16, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } +} + +// TNEG: dst = -src +template +void TNEG(tile_shape &dst, tile_shape &src) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 17, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 17, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 17, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { + asm volatile( + "BSTART.TEPL 17, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } +} + +// TEXP: dst = exp(src) +template +void TEXP(tile_shape &dst, tile_shape &src) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 18, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 18, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 18, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { + asm volatile( + "BSTART.TEPL 18, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } +} + +// TLOG: dst = log(src) +template +void TLOG(tile_shape &dst, tile_shape &src) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 19, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 19, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 19, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { + asm volatile( + "BSTART.TEPL 19, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } +} + +// TRECIP: dst = 1/src +template +void TRECIP(tile_shape &dst, tile_shape &src) { + const size_t valid_col = src.GetValidCol(); + const size_t valid_row = src.GetValidRow(); + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 20, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } + else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 20, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[dyn_row], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), [dyn_row] "r"(valid_row), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } + else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 20, %D1\n" + "B.DIM %[dyn_col], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [dyn_col] "r"(valid_col), "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } + else { +asm volatile( + "BSTART.TEPL 20, %D1\n" + "B.DIM %[dyn_col], 0, ->lb0\n" + "B.DIM %[dyn_row], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [dyn_col] "r"(valid_col), [dyn_row] "r"(valid_row), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } +} + +// TSQRT: dst = sqrt(src) +template +void TSQRT(tile_shape &dst, tile_shape &src) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 21, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 21, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 21, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { + asm volatile( + "BSTART.TEPL 21, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } +} + +// TRSQRT: dst = 1/sqrt(src) +template +void TRSQRT(tile_shape &dst, tile_shape &src) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 22, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 22, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 22, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { + asm volatile( + "BSTART.TEPL 22, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } +} + +// TRELU: dst = max(src, 0) +template +void TRELU(tile_shape &dst, tile_shape &src) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 23, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 23, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 23, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { + asm volatile( + "BSTART.TEPL 23, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } +} + +// TADDC: dst = src0 + src1 + src2 +template +void TADDC(tile_shape &dst, tile_shape &src0, tile_shape &src1, tile_shape &src2) { + static_assert(pto_dependent_false_v, + "TADDC is retired in PTO ISA 0.58.3; no active replacement. Remove the call or migrate to the active surface."); +} + + +// TSUBC: dst = src0 - src1 + src2 +template +void TSUBC(tile_shape &dst, tile_shape &src0, tile_shape &src1, tile_shape &src2) { + static_assert(pto_dependent_false_v, + "TSUBC is retired in PTO ISA 0.58.3; no active replacement. Remove the call or migrate to the active surface."); +} + + +// TCVT: elementwise type conversion (opcode 27, already has TCVT_T) +// Use TCVT_T(dst, src) for this; TCVT is aliased below for convenience. +template +void TCVT(tile_shape_out &dst, tile_shape_in &src) { + TCVT_T(dst, src); +} +//===--- TEPL Mode 1: tile-scalar elementwise ops (BSTART.TEPL) ---===// +// opcode = Mode(1) * 32 + Function = 32 + Function. + +// TADDS: dst = src + scalar +template +void TADDS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + const size_t valid_col = src.GetValidCol(); + const size_t valid_row = src.GetValidRow(); + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 32, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } + else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 32, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[dyn_row], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), [dyn_row] "r"(valid_row), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } + else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 32, %D1\n" + "B.DIM %[dyn_col], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [dyn_col] "r"(valid_col), "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } + else { +asm volatile( + "BSTART.TEPL 32, %D1\n" + "B.DIM %[dyn_col], 0, ->lb0\n" + "B.DIM %[dyn_row], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [dyn_col] "r"(valid_col), [dyn_row] "r"(valid_row), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } +} + +// TSUBS: dst = src - scalar +template +void TSUBS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 33, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 33, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 33, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 33, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } +} + +// TMULS: dst = src * scalar +template +void TMULS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + const size_t valid_col = src.GetValidCol(); + const size_t valid_row = src.GetValidRow(); + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 34, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } + else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 34, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[dyn_row], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), [dyn_row] "r"(valid_row), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } + else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 34, %D1\n" + "B.DIM %[dyn_col], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [dyn_col] "r"(valid_col), "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } + else { +asm volatile( + "BSTART.TEPL 34, %D1\n" + "B.DIM %[dyn_col], 0, ->lb0\n" + "B.DIM %[dyn_row], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [dyn_col] "r"(valid_col), [dyn_row] "r"(valid_row), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } +} + +// TDIVS: dst = src / scalar +template +void TDIVS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 35, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 35, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 35, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 35, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } +} + +// TREMS: dst = rem(src, scalar) +template +void TREMS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 36, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 36, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 36, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 36, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } +} + +// TFMODS: dst = fmod(src, scalar) +template +void TFMODS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { + static_assert(pto_dependent_false_v, + "TFMODS is retired in PTO ISA 0.58.3; no active replacement. Remove the call or migrate to the active surface."); +} + + +// TANDS: dst = src & scalar +template +void TANDS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 38, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 38, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 38, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 38, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } +} + +// TORS: dst = src | scalar +template +void TORS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 39, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 39, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 39, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 39, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } +} + +// TXORS: dst = src ^ scalar +template +void TXORS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 40, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 40, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 40, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 40, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } +} + +// TSHLS: dst = src << scalar +template +void TSHLS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 41, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 41, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 41, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 41, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } +} + +// TSHRS: dst = src >> scalar +template +void TSHRS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 42, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 42, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 42, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 42, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } +} + +// TMAXS: dst = max(src, scalar) +template +void TMAXS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 43, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 43, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 43, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 43, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } +} + +// TMINS: dst = min(src, scalar) +template +void TMINS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 44, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 44, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 44, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 44, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } +} + +// TCMPS: compare src with scalar. The comparison mode is a compile-time +// template parameter encoded into B.DATR CMode[31:29]; scalar travels via the +// canonical B.IOR slot, never as a Tile source (PTO 0.58). +template +void TCMPS(tile_shape_out &dst, tile_shape_in &src, + typename tile_shape_in::DType s) { + static_assert(is_valid_cmp_mode(Mode), "TCMPS requires a valid CmpMode"); + static_assert(tile_shape_in::Rows == tile_shape_out::Rows && + tile_shape_in::Cols == tile_shape_out::Cols, + "TCMPS output shape must match input shape"); + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape_in::DType sv = s; + if constexpr (Mode == CmpMode::EQ) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, EQ\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, EQ\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "r"(src.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, EQ\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src.GetValidCol()), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + else { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, EQ\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src.GetValidCol()), + [VROW] "r"(src.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + } else if constexpr (Mode == CmpMode::NE) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, NE\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, NE\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "r"(src.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, NE\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src.GetValidCol()), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + else { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, NE\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src.GetValidCol()), + [VROW] "r"(src.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + } else if constexpr (Mode == CmpMode::LT) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, LT\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, LT\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "r"(src.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, LT\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src.GetValidCol()), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + else { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, LT\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src.GetValidCol()), + [VROW] "r"(src.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + } else if constexpr (Mode == CmpMode::GT) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, GT\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, GT\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "r"(src.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, GT\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src.GetValidCol()), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + else { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, GT\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src.GetValidCol()), + [VROW] "r"(src.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + } else if constexpr (Mode == CmpMode::LE) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, LE\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, LE\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "r"(src.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, LE\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src.GetValidCol()), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + else { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, LE\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src.GetValidCol()), + [VROW] "r"(src.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + } else if constexpr (Mode == CmpMode::GE) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, GE\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, GE\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), + [VROW] "r"(src.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, GE\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src.GetValidCol()), + [VROW] "i"(tile_shape_in::ValidRow), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + else { +asm volatile( + "BSTART.TEPL 45, %D[TCode]\n" + "B.DATR Zero, GE\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Cols], ->lb2\n" + "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" + "B.IOR [%[Scalar]],[]\n" + "" + : [D] "=Tr"(dst.data()) + : [TCode] "i"(type_traits::TypeCode), + [VCOL] "r"(src.GetValidCol()), + [VROW] "r"(src.GetValidRow()), + [Cols] "i"(tile_shape_in::Cols), + [S] "Tr"(src.data()), + [TSize] "i"(tile_type_traits::TilesizeCode), + [Scalar] "r"(sv) + ); } + } +} + +// Deprecated EQ-default form retained for old callers. +template +void TCMPS(tile_shape_out &dst, tile_shape_in &src, + typename tile_shape_in::DType s) { + TCMPS(dst, src, s); +} + +// TLRELU: leaky ReLU with scalar slope +template +void TLRELU(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { + static_assert(pto_dependent_false_v, + "TLRELU is retired in PTO ISA 0.58.3; no active replacement. Remove the call or migrate to the active surface."); +} + + +// TAXPY: AXPY-style fused update (DavinciOO ext) +template +void TAXPY(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { + static_assert(pto_dependent_false_v, + "TAXPY is retired in PTO ISA 0.58.3; no active replacement. Remove the call or migrate to the active surface."); +} + + +// TADDSC: dst = src0 + scalar + src1 +template +void TADDSC(tile_shape &dst, tile_shape &src0, typename tile_shape::DType s, tile_shape &src1) { + static_assert(pto_dependent_false_v, + "TADDSC is retired in PTO ISA 0.58.3; no active replacement. Remove the call or migrate to the active surface."); +} + + +// TSUBSC: dst = src0 - scalar + src1 +template +void TSUBSC(tile_shape &dst, tile_shape &src0, typename tile_shape::DType s, tile_shape &src1) { + static_assert(pto_dependent_false_v, + "TSUBSC is retired in PTO ISA 0.58.3; no active replacement. Remove the call or migrate to the active surface."); +} + + +// TSELS: select between src tile and scalar using mask +template +void TSELS(tile_shape &dst, tile_shape &src0, typename tile_shape::DType s, tile_shape &src1) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 58, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "B.IOR [%8],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } + else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 58, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "B.IOR [%8],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), [src0____dimrow] "r"(src0.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } + else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 58, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "B.IOR [%8],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src0____dimcol] "r"(src0.GetValidCol()), "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } + else { +asm volatile( + "BSTART.TEPL 58, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "B.IOR [%8],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src0____dimcol] "r"(src0.GetValidCol()), [src0____dimrow] "r"(src0.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } +} + +// TEXPANDS: broadcast scalar into dst tile +template +void TEXPANDS(tile_shape &dst, typename tile_shape::DType s) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 59, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT mask=1111, last, ->%0<%Z5>\n" + "B.IOR [%6],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 59, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT mask=1111, last, ->%0<%Z5>\n" + "B.IOR [%6],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape::Cols), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 59, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT mask=1111, last, ->%0<%Z5>\n" + "B.IOR [%6],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } else { + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType sv = s; + asm volatile( + "BSTART.TEPL 59, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT mask=1111, last, ->%0<%Z5>\n" + "B.IOR [%6],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape::Cols), + "i"(tile_type_traits::TilesizeCode), + "r"(sv) + ); } +} + + +//===--- TEPL Mode 0 extension: TFMA (fused multiply-add, opcode 28) ---===// + +// TFMA: dst = src0 * src1 + src2 (fused element-wise multiply-add) +template +void TFMA(tile_shape &dst, tile_shape &src0, tile_shape &src1, tile_shape &src2) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 28, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111\n" + "B.IOT %7, mask=1111, last, ->%0<%Z8>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "Tr"(src2.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 28, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111\n" + "B.IOT %7, mask=1111, last, ->%0<%Z8>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src0____dimrow] "r"(src0.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "Tr"(src2.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 28, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111\n" + "B.IOT %7, mask=1111, last, ->%0<%Z8>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src0____dimcol] "r"(src0.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "Tr"(src2.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { + asm volatile( + "BSTART.TEPL 28, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111\n" + "B.IOT %7, mask=1111, last, ->%0<%Z8>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src0____dimcol] "r"(src0.GetValidCol()), + [src0____dimrow] "r"(src0.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "Tr"(src2.data()), + "i"(tile_type_traits::TilesizeCode) + ); } +} + +//===--- TEPL Mode 3: complex ops (opcode = 96 + Function) ---===// + +// TEXTRACT: extract sub-tile (indexRow, indexCol via B.IOR) +template +void TEXTRACT(tile_shape_out &dst, tile_shape_in &src, int32_t indexRow, int32_t indexCol) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { + // Anti-fold: keep compile-time-constant indices (e.g. 0) off the zero + // register so B.IOR [zero,...]/[...,zero] still matches an instruction. + volatile int32_t irv = indexRow; + volatile int32_t icv = indexCol; + asm volatile( + "BSTART.TEPL 98, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7,%8],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_in::ValidCol), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(irv), + "r"(icv) + ); } else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { + // Anti-fold: keep compile-time-constant indices (e.g. 0) off the zero + // register so B.IOR [zero,...]/[...,zero] still matches an instruction. + volatile int32_t irv = indexRow; + volatile int32_t icv = indexCol; + asm volatile( + "BSTART.TEPL 98, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7,%8],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_in::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(irv), + "r"(icv) + ); } else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { + // Anti-fold: keep compile-time-constant indices (e.g. 0) off the zero + // register so B.IOR [zero,...]/[...,zero] still matches an instruction. + volatile int32_t irv = indexRow; + volatile int32_t icv = indexCol; + asm volatile( + "BSTART.TEPL 98, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7,%8],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(irv), + "r"(icv) + ); } else { + // Anti-fold: keep compile-time-constant indices (e.g. 0) off the zero + // register so B.IOR [zero,...]/[...,zero] still matches an instruction. + volatile int32_t irv = indexRow; + volatile int32_t icv = indexCol; + asm volatile( + "BSTART.TEPL 98, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7,%8],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(irv), + "r"(icv) + ); } +} + +// TINSERT: insert sub-tile (indexRow, indexCol via B.IOR) +template +void TINSERT(tile_shape_out &dst, tile_shape_in &src, int32_t indexRow, int32_t indexCol) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { + // Anti-fold: keep compile-time-constant indices (e.g. 0) off the zero + // register so B.IOR [zero,...]/[...,zero] still matches an instruction. + volatile int32_t irv = indexRow; + volatile int32_t icv = indexCol; + asm volatile( + "BSTART.TEPL 99, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7,%8],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_in::ValidCol), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(irv), + "r"(icv) + ); } else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { + // Anti-fold: keep compile-time-constant indices (e.g. 0) off the zero + // register so B.IOR [zero,...]/[...,zero] still matches an instruction. + volatile int32_t irv = indexRow; + volatile int32_t icv = indexCol; + asm volatile( + "BSTART.TEPL 99, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7,%8],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_in::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(irv), + "r"(icv) + ); } else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { + // Anti-fold: keep compile-time-constant indices (e.g. 0) off the zero + // register so B.IOR [zero,...]/[...,zero] still matches an instruction. + volatile int32_t irv = indexRow; + volatile int32_t icv = indexCol; + asm volatile( + "BSTART.TEPL 99, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7,%8],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(irv), + "r"(icv) + ); } else { + // Anti-fold: keep compile-time-constant indices (e.g. 0) off the zero + // register so B.IOR [zero,...]/[...,zero] still matches an instruction. + volatile int32_t irv = indexRow; + volatile int32_t icv = indexCol; + asm volatile( + "BSTART.TEPL 99, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7,%8],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "r"(irv), + "r"(icv) + ); } +} + +// TIMG2COL: GM feature-map image-to-column materialization. The source-only +// B.IOR pair carries GMBase followed by the three packed parameter GPRs. +template + requires(tile_shape_out::Loc == Location::Left && + (tile_shape_out::BFractal == BLayout::CubeM16 || + tile_shape_out::BFractal == BLayout::CubeM32)) +void TIMG2COL(tile_shape_out &dst, gm_shape &src, TIMG2COLParams params) { + static_assert(tile_shape_out::ValidRow != 0 && + tile_shape_out::ValidCol != 0, + "TIMG2COL valid dimensions must be nonzero"); + static_assert(type_traits::TypeCode == __type_fp32 || + type_traits::TypeCode == __type_fp16 || + type_traits::TypeCode == __type_bf16 || + type_traits::TypeCode == __type_int32 || + type_traits::TypeCode == __type_int16 || + type_traits::TypeCode == __type_int8 || + type_traits::TypeCode == __type_uint32 || + type_traits::TypeCode == __type_uint16 || + type_traits::TypeCode == __type_uint8, + "TIMG2COL DataType is not supported by the ASL contract"); + volatile uint64_t param0 = params.param0; + volatile uint64_t param1 = params.param1; + volatile uint64_t param2 = params.param2; + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { +asm volatile( + "BSTART.TIMG2COL %D[DataType]\n" + "B.DATR %c[Layout], DTYPE_NONE, Zero\n" + "B.DIM zero, %c[ValidCol], ->lb0\n" + "B.DIM zero, %c[ValidRow], ->lb1\n" + "B.DIM zero, %c[TotalCol], ->lb2\n" + "B.IOR [%[GMBase], zero, zero], []\n" + "B.IOR [%[Param0], %[Param1], %[Param2]], []\n" + "B.IOT mask=1111, last, ->%[Dst]<%Z[TileSize]>\n" + : [Dst] "=Tr"(dst.data()) + : [GMBase] "r"(src.data()), + [DataType] "i"(type_traits::TypeCode), + [Layout] "i"(tile_shape_out::BFractal == BLayout::CubeM16 ? + LayoutCvtEnum::ND2M16 : LayoutCvtEnum::ND2M32), + [ValidCol] "i"(tile_shape_out::ValidCol), + [ValidRow] "i"(tile_shape_out::ValidRow), + [TotalCol] "i"(tile_shape_out::Cols), + [Param0] "r"(param0), [Param1] "r"(param1), [Param2] "r"(param2), + [TileSize] "i"(tile_type_traits::TilesizeCode) + : "memory"); } + else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { +asm volatile( + "BSTART.TIMG2COL %D[DataType]\n" + "B.DATR %c[Layout], DTYPE_NONE, Zero\n" + "B.DIM zero, %c[ValidCol], ->lb0\n" + "B.DIM %[ValidRow], 0, ->lb1\n" + "B.DIM zero, %c[TotalCol], ->lb2\n" + "B.IOR [%[GMBase], zero, zero], []\n" + "B.IOR [%[Param0], %[Param1], %[Param2]], []\n" + "B.IOT mask=1111, last, ->%[Dst]<%Z[TileSize]>\n" + : [Dst] "=Tr"(dst.data()) + : [GMBase] "r"(src.data()), + [DataType] "i"(type_traits::TypeCode), + [Layout] "i"(tile_shape_out::BFractal == BLayout::CubeM16 ? + LayoutCvtEnum::ND2M16 : LayoutCvtEnum::ND2M32), + [ValidCol] "i"(tile_shape_out::ValidCol), + [ValidRow] "r"(dst.GetValidRow()), + [TotalCol] "i"(tile_shape_out::Cols), + [Param0] "r"(param0), [Param1] "r"(param1), [Param2] "r"(param2), + [TileSize] "i"(tile_type_traits::TilesizeCode) + : "memory"); } + else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { +asm volatile( + "BSTART.TIMG2COL %D[DataType]\n" + "B.DATR %c[Layout], DTYPE_NONE, Zero\n" + "B.DIM %[ValidCol], 0, ->lb0\n" + "B.DIM zero, %c[ValidRow], ->lb1\n" + "B.DIM zero, %c[TotalCol], ->lb2\n" + "B.IOR [%[GMBase], zero, zero], []\n" + "B.IOR [%[Param0], %[Param1], %[Param2]], []\n" + "B.IOT mask=1111, last, ->%[Dst]<%Z[TileSize]>\n" + : [Dst] "=Tr"(dst.data()) + : [GMBase] "r"(src.data()), + [DataType] "i"(type_traits::TypeCode), + [Layout] "i"(tile_shape_out::BFractal == BLayout::CubeM16 ? + LayoutCvtEnum::ND2M16 : LayoutCvtEnum::ND2M32), + [ValidCol] "r"(dst.GetValidCol()), + [ValidRow] "i"(tile_shape_out::ValidRow), + [TotalCol] "i"(tile_shape_out::Cols), + [Param0] "r"(param0), [Param1] "r"(param1), [Param2] "r"(param2), + [TileSize] "i"(tile_type_traits::TilesizeCode) + : "memory"); } + else { +asm volatile( + "BSTART.TIMG2COL %D[DataType]\n" + "B.DATR %c[Layout], DTYPE_NONE, Zero\n" + "B.DIM %[ValidCol], 0, ->lb0\n" + "B.DIM %[ValidRow], 0, ->lb1\n" + "B.DIM zero, %c[TotalCol], ->lb2\n" + "B.IOR [%[GMBase], zero, zero], []\n" + "B.IOR [%[Param0], %[Param1], %[Param2]], []\n" + "B.IOT mask=1111, last, ->%[Dst]<%Z[TileSize]>\n" + : [Dst] "=Tr"(dst.data()) + : [GMBase] "r"(src.data()), + [DataType] "i"(type_traits::TypeCode), + [Layout] "i"(tile_shape_out::BFractal == BLayout::CubeM16 ? + LayoutCvtEnum::ND2M16 : LayoutCvtEnum::ND2M32), + [ValidCol] "r"(dst.GetValidCol()), + [ValidRow] "r"(dst.GetValidRow()), + [TotalCol] "i"(tile_shape_out::Cols), + [Param0] "r"(param0), [Param1] "r"(param1), [Param2] "r"(param2), + [TileSize] "i"(tile_type_traits::TilesizeCode) + : "memory"); } +} + +template +void TIMG2COL(tile_shape_out &dst, gm_shape &src, + uint64_t param0, uint64_t param1, uint64_t param2) { + TIMG2COL(dst, src, TIMG2COLParams{param0, param1, param2}); +} + +// TFILLPAD: copy valid region and fill padding +template +void TFILLPAD(tile_shape_out &dst, tile_shape_in &src) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 101, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_in::ValidCol), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 101, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_in::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 101, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { + asm volatile( + "BSTART.TEPL 101, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } +} + +// TCI: contiguous integer sequence generation +template +void TCI(tile_shape &dst, T s) { + static_assert(std::is_same::value, + "TCI destination and start must have the same type"); + static_assert(descending == 0 || descending == 1, + "TCI direction must be ascending (0) or descending (1)"); + static_assert(tile_shape::Loc == Location::Vec, + "TCI requires a Local vector tile"); + static_assert(tile_shape::isRowMajor && !tile_shape::isBoxedLayout, + "TCI requires an unboxed RowMajor tile"); + static_assert(tile_shape::ValidRow == 1, + "TCI requires ValidRow == 1"); + static_assert(tile_shape::ValidCol > 0 && + tile_shape::Cols >= tile_shape::ValidCol, + "TCI requires 0 < ValidCol <= Cols"); + static_assert(std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value, + "TCI supports only S32, S16, U32, and U16"); + // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero + // register so B.IOR [zero],[] still matches an instruction. + volatile typename tile_shape::DType startValue = s; + volatile uint32_t directionValue = descending; + if constexpr (tile_shape::ValidCol > 0) { +asm volatile( + "BSTART.TEPL 102, %D[DataType]\n" + "B.DIM zero, %c[ValidCol], ->lb0\n" + "B.DIM zero, %c[PhysicalCol], ->lb2\n" + "B.IOR [%[Start],%[Direction]],[]\n" + "B.IOT mask=1111, last, ->%[Dst]<%Z[TileSize]>\n" + : [Dst] "=Tr"(dst.data()) + : [DataType] "i"(type_traits::TypeCode), + [ValidCol] "i"(tile_shape::ValidCol), + [PhysicalCol] "i"(tile_shape::Cols), + [TileSize] "i"(tile_type_traits::TilesizeCode), + [Start] "r"(startValue), + [Direction] "r"(directionValue) + ); } + else { +asm volatile( + "BSTART.TEPL 102, %D[DataType]\n" + "B.DIM %[ValidCol], 0, ->lb0\n" + "B.DIM zero, %c[PhysicalCol], ->lb2\n" + "B.IOR [%[Start],%[Direction]],[]\n" + "B.IOT mask=1111, last, ->%[Dst]<%Z[TileSize]>\n" + : [Dst] "=Tr"(dst.data()) + : [DataType] "i"(type_traits::TypeCode), + [ValidCol] "r"(dst.GetValidCol()), + [PhysicalCol] "i"(tile_shape::Cols), + [TileSize] "i"(tile_type_traits::TilesizeCode), + [Start] "r"(startValue), + [Direction] "r"(directionValue) + ); } +} + +// TTRI: triangular mask generation +template +void TTRI(tile_shape &dst) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 103, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT mask=1111, last, ->%0<%Z5>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 103, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT mask=1111, last, ->%0<%Z5>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape::Cols), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 103, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT mask=1111, last, ->%0<%Z5>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + "i"(tile_shape::ValidRow), + "i"(tile_shape::Cols), + "i"(tile_type_traits::TilesizeCode) + ); } else { + asm volatile( + "BSTART.TEPL 103, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT mask=1111, last, ->%0<%Z5>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape::Cols), + "i"(tile_type_traits::TilesizeCode) + ); } +} + +// TRANDOM: counter-based random tile generation +template +void TRANDOM(tile_shape &dst, typename tile_shape::DType s) { + static_assert(pto_dependent_false_v, + "TRANDOM is retired in PTO ISA 0.58.3; no active replacement. Remove the call or migrate to the active surface."); +} + + +// TQUANT: FP32 -> S8/U8 quantization with B.DATR RMode/Sat and B.IOR +// multiplier/zero-point (PTO ISA 0.58.3 TEPL Mode3 Fn10 / selector 0x06A). +// RMode and Sat are B.DATR fields: RMode accepts a numeric immediate (the +// parser maps mnemonic or numeric), Sat only the NOSAT/SAT token, so Sat is +// selected with if constexpr. The multiplier travels as its raw FP32 bits in +// a GPR and zeroPoint in another. +template +void TQUANT(tile_shape_out &dst, tile_shape_in &src, float multiplier = 1.0f, + int32_t zeroPoint = 0) { + static_assert(is_valid_round_mode(Mode), "TQUANT invalid rounding mode"); + static_assert(type_traits::TypeCode == + __type_fp32, + "TQUANT source must be FP32"); + static_assert(type_traits::TypeCode == + __type_int8 || + type_traits::TypeCode == + __type_uint8, + "TQUANT destination must be S8 or U8"); + static_assert(tile_shape_out::Loc == Location::Vec && + tile_shape_in::Loc == Location::Vec && + tile_shape_out::isRowMajor && + tile_shape_in::isRowMajor && + !tile_shape_out::isBoxedLayout && + !tile_shape_in::isBoxedLayout, + "TQUANT operands must be Local RowMajor numeric Tiles"); + static_assert( + tile_shape_out::Rows == tile_shape_in::Rows && + tile_shape_out::Cols == tile_shape_in::Cols && + (tile_shape_out::ValidRow == DYNAMIC || + tile_shape_in::ValidRow == DYNAMIC || + tile_shape_out::ValidRow == tile_shape_in::ValidRow) && + (tile_shape_out::ValidCol == DYNAMIC || + tile_shape_in::ValidCol == DYNAMIC || + tile_shape_out::ValidCol == tile_shape_in::ValidCol), + "TQUANT source and destination logical shapes must match"); + uint32_t multiplierBits; + __builtin_memcpy(&multiplierBits, &multiplier, sizeof(multiplier)); + volatile uint32_t mult = multiplierBits; + volatile int32_t zp = zeroPoint; + if constexpr (Mode == RoundMode::RNE && Saturate) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 106, %D[SType]\n" + // LLVM currently names encoded RMode zero RNONE. PTO ISA 0.58.3 defines + // that encoding as the operation default, which is RNE for TQUANT. + "B.DATR %D[__pto_DstType], RNONE, sat\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"(type_traits::TypeCode), + [__pto_DstType] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), [VROW] "i"(tile_shape_in::ValidRow), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 106, %D[SType]\n" + // LLVM currently names encoded RMode zero RNONE. PTO ISA 0.58.3 defines + // that encoding as the operation default, which is RNE for TQUANT. + "B.DATR %D[__pto_DstType], RNONE, sat\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"(type_traits::TypeCode), + [__pto_DstType] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), [VROW] "r"(src.GetValidRow()), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 106, %D[SType]\n" + // LLVM currently names encoded RMode zero RNONE. PTO ISA 0.58.3 defines + // that encoding as the operation default, which is RNE for TQUANT. + "B.DATR %D[__pto_DstType], RNONE, sat\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"(type_traits::TypeCode), + [__pto_DstType] "i"(type_traits::TypeCode), + [VCOL] "r"(src.GetValidCol()), [VROW] "i"(tile_shape_in::ValidRow), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + else { +asm volatile( + "BSTART.TEPL 106, %D[SType]\n" + // LLVM currently names encoded RMode zero RNONE. PTO ISA 0.58.3 defines + // that encoding as the operation default, which is RNE for TQUANT. + "B.DATR %D[__pto_DstType], RNONE, sat\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"(type_traits::TypeCode), + [__pto_DstType] "i"(type_traits::TypeCode), + [VCOL] "r"(src.GetValidCol()), [VROW] "r"(src.GetValidRow()), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + } else if constexpr (Mode == RoundMode::RNE) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 106, %D[SType]\n" + "B.DATR %D[__pto_DstType], RNONE\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"(type_traits::TypeCode), + [__pto_DstType] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), [VROW] "i"(tile_shape_in::ValidRow), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 106, %D[SType]\n" + "B.DATR %D[__pto_DstType], RNONE\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"(type_traits::TypeCode), + [__pto_DstType] "i"(type_traits::TypeCode), + [VCOL] "i"(tile_shape_in::ValidCol), [VROW] "r"(src.GetValidRow()), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 106, %D[SType]\n" + "B.DATR %D[__pto_DstType], RNONE\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"(type_traits::TypeCode), + [__pto_DstType] "i"(type_traits::TypeCode), + [VCOL] "r"(src.GetValidCol()), [VROW] "i"(tile_shape_in::ValidRow), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + else { +asm volatile( + "BSTART.TEPL 106, %D[SType]\n" + "B.DATR %D[__pto_DstType], RNONE\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"(type_traits::TypeCode), + [__pto_DstType] "i"(type_traits::TypeCode), + [VCOL] "r"(src.GetValidCol()), [VROW] "r"(src.GetValidRow()), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + } else if constexpr (Saturate) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 106, %D[SType]\n" + PTO_RMODE_DATR_ASM(", sat\n") + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"(type_traits::TypeCode), + [__pto_DstType] "i"(type_traits::TypeCode), + [RMode] "i"(static_cast(Mode)), + [VCOL] "i"(tile_shape_in::ValidCol), [VROW] "i"(tile_shape_in::ValidRow), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 106, %D[SType]\n" + PTO_RMODE_DATR_ASM(", sat\n") + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"(type_traits::TypeCode), + [__pto_DstType] "i"(type_traits::TypeCode), + [RMode] "i"(static_cast(Mode)), + [VCOL] "i"(tile_shape_in::ValidCol), [VROW] "r"(src.GetValidRow()), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 106, %D[SType]\n" + PTO_RMODE_DATR_ASM(", sat\n") + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"(type_traits::TypeCode), + [__pto_DstType] "i"(type_traits::TypeCode), + [RMode] "i"(static_cast(Mode)), + [VCOL] "r"(src.GetValidCol()), [VROW] "i"(tile_shape_in::ValidRow), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + else { +asm volatile( + "BSTART.TEPL 106, %D[SType]\n" + PTO_RMODE_DATR_ASM(", sat\n") + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"(type_traits::TypeCode), + [__pto_DstType] "i"(type_traits::TypeCode), + [RMode] "i"(static_cast(Mode)), + [VCOL] "r"(src.GetValidCol()), [VROW] "r"(src.GetValidRow()), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + } else { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 106, %D[SType]\n" + PTO_RMODE_DATR_ASM("\n") + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"(type_traits::TypeCode), + [__pto_DstType] "i"(type_traits::TypeCode), + [RMode] "i"(static_cast(Mode)), + [VCOL] "i"(tile_shape_in::ValidCol), [VROW] "i"(tile_shape_in::ValidRow), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 106, %D[SType]\n" + PTO_RMODE_DATR_ASM("\n") + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"(type_traits::TypeCode), + [__pto_DstType] "i"(type_traits::TypeCode), + [RMode] "i"(static_cast(Mode)), + [VCOL] "i"(tile_shape_in::ValidCol), [VROW] "r"(src.GetValidRow()), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 106, %D[SType]\n" + PTO_RMODE_DATR_ASM("\n") + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"(type_traits::TypeCode), + [__pto_DstType] "i"(type_traits::TypeCode), + [RMode] "i"(static_cast(Mode)), + [VCOL] "r"(src.GetValidCol()), [VROW] "i"(tile_shape_in::ValidRow), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + else { +asm volatile( + "BSTART.TEPL 106, %D[SType]\n" + PTO_RMODE_DATR_ASM("\n") + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"(type_traits::TypeCode), + [__pto_DstType] "i"(type_traits::TypeCode), + [RMode] "i"(static_cast(Mode)), + [VCOL] "r"(src.GetValidCol()), [VROW] "r"(src.GetValidRow()), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + } +} + +// TDEQUANT: S8/U8 -> FP32 dequantization (TEPL Mode3 Fn11 / 0x06B). +// B.DATR carries only FP32 + RMode (Sat is always false per spec). +template +void TDEQUANT(tile_shape_out &dst, tile_shape_in &src, float multiplier = 1.0f, + int32_t zeroPoint = 0) { + static_assert(is_valid_round_mode(Mode), "TDEQUANT invalid rounding mode"); + static_assert(type_traits::TypeCode == + __type_fp32, + "TDEQUANT destination must be FP32"); + static_assert(type_traits::TypeCode == + __type_int8 || + type_traits::TypeCode == + __type_uint8, + "TDEQUANT source must be S8 or U8"); + static_assert(tile_shape_out::Loc == Location::Vec && + tile_shape_in::Loc == Location::Vec && + tile_shape_out::isRowMajor && + tile_shape_in::isRowMajor && + !tile_shape_out::isBoxedLayout && + !tile_shape_in::isBoxedLayout, + "TDEQUANT operands must be Local RowMajor numeric Tiles"); + static_assert( + tile_shape_out::Rows == tile_shape_in::Rows && + tile_shape_out::Cols == tile_shape_in::Cols && + (tile_shape_out::ValidRow == DYNAMIC || + tile_shape_in::ValidRow == DYNAMIC || + tile_shape_out::ValidRow == tile_shape_in::ValidRow) && + (tile_shape_out::ValidCol == DYNAMIC || + tile_shape_in::ValidCol == DYNAMIC || + tile_shape_out::ValidCol == tile_shape_in::ValidCol), + "TDEQUANT source and destination logical shapes must match"); + uint32_t multiplierBits; + __builtin_memcpy(&multiplierBits, &multiplier, sizeof(multiplier)); + volatile uint32_t mult = multiplierBits; + volatile int32_t zp = zeroPoint; + if constexpr (Mode == RoundMode::RNE) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 107, %D[SType]\n" + "B.DATR FP32, RNONE\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"( + type_traits::TypeCode == __type_int8 + ? __type_int8 : __type_uint8), + [VCOL] "i"(tile_shape_in::ValidCol), [VROW] "i"(tile_shape_in::ValidRow), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 107, %D[SType]\n" + "B.DATR FP32, RNONE\n" + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"( + type_traits::TypeCode == __type_int8 + ? __type_int8 : __type_uint8), + [VCOL] "i"(tile_shape_in::ValidCol), [VROW] "r"(src.GetValidRow()), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 107, %D[SType]\n" + "B.DATR FP32, RNONE\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"( + type_traits::TypeCode == __type_int8 + ? __type_int8 : __type_uint8), + [VCOL] "r"(src.GetValidCol()), [VROW] "i"(tile_shape_in::ValidRow), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + else { +asm volatile( + "BSTART.TEPL 107, %D[SType]\n" + "B.DATR FP32, RNONE\n" + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"( + type_traits::TypeCode == __type_int8 + ? __type_int8 : __type_uint8), + [VCOL] "r"(src.GetValidCol()), [VROW] "r"(src.GetValidRow()), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + } else { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 107, %D[SType]\n" + PTO_RMODE_DATR_ASM("\n") + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"( + type_traits::TypeCode == __type_int8 + ? __type_int8 : __type_uint8), + [__pto_DstType] "i"(__type_fp32), + [RMode] "i"(static_cast(Mode)), + [VCOL] "i"(tile_shape_in::ValidCol), [VROW] "i"(tile_shape_in::ValidRow), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { +asm volatile( + "BSTART.TEPL 107, %D[SType]\n" + PTO_RMODE_DATR_ASM("\n") + "B.DIM zero, %c[VCOL], ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"( + type_traits::TypeCode == __type_int8 + ? __type_int8 : __type_uint8), + [__pto_DstType] "i"(__type_fp32), + [RMode] "i"(static_cast(Mode)), + [VCOL] "i"(tile_shape_in::ValidCol), [VROW] "r"(src.GetValidRow()), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { +asm volatile( + "BSTART.TEPL 107, %D[SType]\n" + PTO_RMODE_DATR_ASM("\n") + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM zero, %c[VROW], ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"( + type_traits::TypeCode == __type_int8 + ? __type_int8 : __type_uint8), + [__pto_DstType] "i"(__type_fp32), + [RMode] "i"(static_cast(Mode)), + [VCOL] "r"(src.GetValidCol()), [VROW] "i"(tile_shape_in::ValidRow), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + else { +asm volatile( + "BSTART.TEPL 107, %D[SType]\n" + PTO_RMODE_DATR_ASM("\n") + "B.DIM %[VCOL], 0, ->lb0\n" + "B.DIM %[VROW], 0, ->lb1\n" + "B.DIM zero, %c[Col], ->lb2\n" + "B.IOR [%[Mult], %[ZP]], []\n" + "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Src] "Tr"(src.data()), + [SType] "i"( + type_traits::TypeCode == __type_int8 + ? __type_int8 : __type_uint8), + [__pto_DstType] "i"(__type_fp32), + [RMode] "i"(static_cast(Mode)), + [VCOL] "r"(src.GetValidCol()), [VROW] "r"(src.GetValidRow()), + [Col] "i"(tile_shape_in::Cols), + [Mult] "r"(mult), [ZP] "r"(zp), + [DstSize] "i"(tile_shape_out::TilesizeCode) + ); } + } +} + +// TDEQUANT: profile-defined dequantization +template +void TDEQUANT(tile_shape_out &dst, tile_shape_in &src) { + TDEQUANT(dst, src); +} + +// TSORT: stable per-row group sort producing sorted values (FP16/FP32) and +// group-local original indices (U32). PTO ISA 0.58.3: TEPL Mode 3 Function 12 +// (selector 0x06c). Each row sorts its columns in `sortWidth`-width groups +// from column 0; the last group may be short. ascending when descending is +// false, descending otherwise. sortWidth must be 1..64 (0/LB0 omitted -> 32). +// +// Encoding-carrier note: the LinxV5 backend has no canonical BSTART.SFU +// TSORT mnemonic yet, so the TEPL 108 (TSORT32) carrier is used here. The +// bundle shape below matches the normative contract: only LB0, a single +// B.IOR with the descending flag, a source+value-dest B.IOT, then a +// destination-only index B.IOT. The two destinations use their own +// TileSizeCode (value FP16/FP32 vs index U32 differ in bytes). +template +void TSORT(ValueDstTile &valueDst, IndexDstTile &indexDst, + SourceTile &source, uint32_t sortWidth = 32, + bool descending = false) { + static_assert(std::is_same_v, + "TSORT value destination dtype must match source dtype"); + static_assert(type_traits::TypeCode == + __type_fp16 || + type_traits::TypeCode == + __type_fp32, + "TSORT value source/destination must be FP16 or FP32"); + static_assert(type_traits::TypeCode == + __type_uint32, + "TSORT index destination must be U32"); + static_assert(ValueDstTile::Rows == SourceTile::Rows && + ValueDstTile::Cols == SourceTile::Cols && + IndexDstTile::Rows == SourceTile::Rows && + IndexDstTile::Cols == SourceTile::Cols && + (ValueDstTile::ValidRow == DYNAMIC || + SourceTile::ValidRow == DYNAMIC || + ValueDstTile::ValidRow == SourceTile::ValidRow) && + (ValueDstTile::ValidCol == DYNAMIC || + SourceTile::ValidCol == DYNAMIC || + ValueDstTile::ValidCol == SourceTile::ValidCol) && + (IndexDstTile::ValidRow == DYNAMIC || + SourceTile::ValidRow == DYNAMIC || + IndexDstTile::ValidRow == SourceTile::ValidRow) && + (IndexDstTile::ValidCol == DYNAMIC || + SourceTile::ValidCol == DYNAMIC || + IndexDstTile::ValidCol == SourceTile::ValidCol), + "TSORT value/index destinations must match source logical " + "and valid shapes"); + static_assert(ValueDstTile::Loc == Location::Vec && + IndexDstTile::Loc == Location::Vec && + SourceTile::Loc == Location::Vec && + ValueDstTile::isRowMajor && IndexDstTile::isRowMajor && + SourceTile::isRowMajor && + !ValueDstTile::isBoxedLayout && + !IndexDstTile::isBoxedLayout && + !SourceTile::isBoxedLayout, + "TSORT operands must be Local RowMajor VEC Tiles"); + + // Anti-fold: keep the 0/1 descending flag off the zero register so the + // B.IOR binder still carries a real GPR (B.IOR [zero],[] does not match). + volatile uint32_t descendingValue = descending ? 1u : 0u; + asm volatile( + "BSTART.TEPL 108, %D[DataType]\n" + "B.DIM %[SortWidth], 0, ->lb0\n" + "B.IOR [%[Descending]], []\n" + "B.IOT %[Source], mask=1111, ->%[ValueDst]<%Z[ValueTileSize]>\n" + "B.IOT mask=1111, last, ->%[IndexDst]<%Z[IndexTileSize]>\n" + : [ValueDst] "=&Tr"(valueDst.data()), + [IndexDst] "=&Tr"(indexDst.data()) + : [Source] "Tr"(source.data()), + [DataType] "i"(type_traits::TypeCode), + [SortWidth] "r"(sortWidth), + [Descending] "r"(descendingValue), + // Use each destination's logical TilesizeCode: FP16 value (e.g. 2 KB + // for 32x32) and U32 index (4 KB) differ, and B.IOT must carry the + // per-PE logical size, not the uniform 4 KB storage carrier. + [ValueTileSize] "i"(ValueDstTile::TilesizeCode), + [IndexTileSize] "i"(IndexDstTile::TilesizeCode) ); - } } -// TREMS: dst = rem(src, scalar) -template -void TREMS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; +// Deprecated single-output sort: does not match the PTO ISA 0.58.3 TSORT +// contract (no U32 index destination, legacy LB1/LB2 shape bundle). Kept as +// a migration diagnostic that fails at instantiation. +template +void TSORT32(tile_shape_out &dst, tile_shape_in &src) { + static_assert(pto_dependent_false_v, + "TSORT32 is removed; use TSORT(valueDst, indexDst, source, " + "sortWidth, descending), which emits the PTO ISA 0.58.3 " + "value+index dual-output bundle"); +} + +// TMRGSORT: merge two sorted single-row sources into one destination +// (PTO ISA 0.58.3 TEPL Mode 3 Function 13 / selector 0x06D; canonical +// BSTART.SFU TMRGSORT). No B.DIM: the block carries only B.IOR RegSrc0 +// (0/1 ascending/descending) and one TwoSrc_Dst B.IOT with . +template +void TMRGSORT(DstTile &dst, LeftTile &left, RightTile &right, + bool descending = false) { + static_assert(std::is_same_v && + std::is_same_v, + "TMRGSORT dst/left/right must share one dtype"); + static_assert(type_traits::TypeCode == + __type_fp16 || + type_traits::TypeCode == + __type_fp32, + "TMRGSORT dtype must be FP16 or FP32"); + static_assert(DstTile::Loc == Location::Vec && + LeftTile::Loc == Location::Vec && + RightTile::Loc == Location::Vec && + DstTile::isRowMajor && LeftTile::isRowMajor && + RightTile::isRowMajor && !DstTile::isBoxedLayout && + !LeftTile::isBoxedLayout && !RightTile::isBoxedLayout, + "TMRGSORT operands must be Local RowMajor numeric Tiles"); + static_assert(DstTile::Rows == 1 && LeftTile::Rows == 1 && + RightTile::Rows == 1, + "TMRGSORT operands must be single-row Tiles"); + static_assert((LeftTile::ValidCol == DYNAMIC || LeftTile::ValidCol > 0) && + (RightTile::ValidCol == DYNAMIC || + RightTile::ValidCol > 0), + "TMRGSORT sources must be non-empty"); + static_assert( + (DstTile::ValidCol == DYNAMIC || LeftTile::ValidCol == DYNAMIC || + RightTile::ValidCol == DYNAMIC || + (DstTile::ValidCol == LeftTile::ValidCol + RightTile::ValidCol && + DstTile::Cols >= LeftTile::ValidCol + RightTile::ValidCol && + (DstTile::Cols & (DstTile::Cols - 1)) == 0 && + DstTile::Cols / 2 < LeftTile::ValidCol + RightTile::ValidCol)), + "TMRGSORT destination must contain the combined source columns"); + // Anti-fold: keep the 0/1 flag off the zero register (B.IOR [zero],[] does + // not match). + volatile uint32_t descendingValue = descending ? 1u : 0u; asm volatile( - "BSTART.TEPL 36, %D1\n" + "BSTART.TEPL 109, %D[DataType]\n" + "B.IOR [%[Descending]], []\n" + "B.IOT %[Left], %[Right], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" + : [Dst] "=&Tr"(dst.data()) + : [Left] "Tr"(left.data()), [Right] "Tr"(right.data()), + [Descending] "r"(descendingValue), + [DataType] "i"(type_traits::TypeCode), + [DstSize] "i"(DstTile::TilesizeCode) + ); +} + +// TTRANS: tile transpose +template +void TTRANS(tile_shape_out &dst, tile_shape_in &src) { + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 110, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(tile_shape::ValidCol), - "i"(tile_shape::ValidRow), - "i"(tile_shape::Cols), + : "i"(type_traits::TypeCode), + "i"(tile_shape_in::ValidCol), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } else { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { asm volatile( - "BSTART.TEPL 36, %D1\n" + "BSTART.TEPL 110, %D1\n" "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_in::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 110, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), - "i"(tile_shape::Cols), + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } -} - -// TFMODS: dst = fmod(src, scalar) -template -void TFMODS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { - static_assert(pto_dependent_false_v, - "TFMODS is retired in PTO ISA 0.58.3; no active replacement. Remove the call or migrate to the active surface."); + "i"(tile_type_traits::TilesizeCode) + ); } else { + asm volatile( + "BSTART.TEPL 110, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } - -// TANDS: dst = src & scalar -template -void TANDS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; +// TGATHER: index/mask tile gather +template +void TGATHER(tile_shape_out &dst, tile_shape_in &src, tile_shape_off &off) { + if constexpr (tile_shape_off::ValidCol > 0 && tile_shape_off::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 111, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_off::ValidCol), + "i"(tile_shape_off::ValidRow), + "i"(tile_shape_off::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "Tr"(off.data()) + ); } else if constexpr (tile_shape_off::ValidCol > 0 && tile_shape_off::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 111, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[off____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOR [%7],[]\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_off::ValidCol), + [off____dimrow] "r"(off.GetValidRow()), + "i"(tile_shape_off::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode), + "Tr"(off.data()) + ); } else if constexpr (tile_shape_off::ValidCol < 0 && tile_shape_off::ValidRow > 0) { asm volatile( - "BSTART.TEPL 38, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 111, %D1\n" + "B.DIM %[off____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(tile_shape::ValidCol), - "i"(tile_shape::ValidRow), - "i"(tile_shape::Cols), + : "i"(type_traits::TypeCode), + [off____dimcol] "r"(off.GetValidCol()), + "i"(tile_shape_off::ValidRow), + "i"(tile_shape_off::Cols), "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } else { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; + "i"(tile_type_traits::TilesizeCode), + "Tr"(off.data()) + ); } else { asm volatile( - "BSTART.TEPL 38, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 111, %D1\n" + "B.DIM %[off____dimcol], 0, ->lb0\n" + "B.DIM %[off____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), - "i"(tile_shape::Cols), + : "i"(type_traits::TypeCode), + [off____dimcol] "r"(off.GetValidCol()), + [off____dimrow] "r"(off.GetValidRow()), + "i"(tile_shape_off::Cols), "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } + "i"(tile_type_traits::TilesizeCode), + "Tr"(off.data()) + ); } } -// TORS: dst = src | scalar -template -void TORS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; +// TSCATTER: index tile scatter +template +void TSCATTER(tile_shape_out &dst, tile_shape_in &src, tile_shape_off &off) { + if constexpr (tile_shape_off::ValidCol > 0 && tile_shape_off::ValidRow > 0) { asm volatile( - "BSTART.TEPL 39, %D1\n" + "BSTART.TEPL 112, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" @@ -8355,534 +13068,287 @@ void TORS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(tile_shape::ValidCol), - "i"(tile_shape::ValidRow), - "i"(tile_shape::Cols), + : "i"(type_traits::TypeCode), + "i"(tile_shape_off::ValidCol), + "i"(tile_shape_off::ValidRow), + "i"(tile_shape_off::Cols), "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } else { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; + "i"(tile_type_traits::TilesizeCode), + "Tr"(off.data()) + ); } else if constexpr (tile_shape_off::ValidCol > 0 && tile_shape_off::ValidRow < 0) { asm volatile( - "BSTART.TEPL 39, %D1\n" + "BSTART.TEPL 112, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[off____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), - "i"(tile_shape::Cols), + : "i"(type_traits::TypeCode), + "i"(tile_shape_off::ValidCol), + [off____dimrow] "r"(off.GetValidRow()), + "i"(tile_shape_off::Cols), "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } -} - -// TXORS: dst = src ^ scalar -template -void TXORS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; + "i"(tile_type_traits::TilesizeCode), + "Tr"(off.data()) + ); } else if constexpr (tile_shape_off::ValidCol < 0 && tile_shape_off::ValidRow > 0) { asm volatile( - "BSTART.TEPL 40, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 112, %D1\n" + "B.DIM %[off____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(tile_shape::ValidCol), - "i"(tile_shape::ValidRow), - "i"(tile_shape::Cols), + : "i"(type_traits::TypeCode), + [off____dimcol] "r"(off.GetValidCol()), + "i"(tile_shape_off::ValidRow), + "i"(tile_shape_off::Cols), "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } else { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; + "i"(tile_type_traits::TilesizeCode), + "Tr"(off.data()) + ); } else { asm volatile( - "BSTART.TEPL 40, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 112, %D1\n" + "B.DIM %[off____dimcol], 0, ->lb0\n" + "B.DIM %[off____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), - "i"(tile_shape::Cols), + : "i"(type_traits::TypeCode), + [off____dimcol] "r"(off.GetValidCol()), + [off____dimrow] "r"(off.GetValidRow()), + "i"(tile_shape_off::Cols), "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } + "i"(tile_type_traits::TilesizeCode), + "Tr"(off.data()) + ); } } -// TSHLS: dst = src << scalar +// TPARTADD: partial-valid add (moved from Mode 0) template -void TSHLS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { +void TPARTADD(tile_shape &dst, tile_shape &src0, tile_shape &src1) { if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; asm volatile( - "BSTART.TEPL 41, %D1\n" + "BSTART.TEPL 113, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7],[]\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } else { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { asm volatile( - "BSTART.TEPL 41, %D1\n" + "BSTART.TEPL 113, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7],[]\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), + "i"(tile_shape::ValidCol), + [src0____dimrow] "r"(src0.GetValidRow()), "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } -} - -// TSHRS: dst = src >> scalar -template -void TSHRS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { asm volatile( - "BSTART.TEPL 42, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 113, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7],[]\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape::ValidCol), + [src0____dimcol] "r"(src0.GetValidCol()), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } else { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { asm volatile( - "BSTART.TEPL 42, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 113, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7],[]\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), + [src0____dimcol] "r"(src0.GetValidCol()), + [src0____dimrow] "r"(src0.GetValidRow()), "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } -// TMAXS: dst = max(src, scalar) +// TPARTMUL: partial-valid multiply (moved from Mode 0) template -void TMAXS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { +void TPARTMUL(tile_shape &dst, tile_shape &src0, tile_shape &src1) { if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; asm volatile( - "BSTART.TEPL 43, %D1\n" + "BSTART.TEPL 114, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7],[]\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } else { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { asm volatile( - "BSTART.TEPL 43, %D1\n" + "BSTART.TEPL 114, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7],[]\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), + "i"(tile_shape::ValidCol), + [src0____dimrow] "r"(src0.GetValidRow()), "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } -} - -// TMINS: dst = min(src, scalar) -template -void TMINS(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; - asm volatile( - "BSTART.TEPL 44, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 114, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7],[]\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape::ValidCol), + [src0____dimcol] "r"(src0.GetValidCol()), "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } else { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { asm volatile( - "BSTART.TEPL 44, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 114, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7],[]\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), + [src0____dimcol] "r"(src0.GetValidCol()), + [src0____dimrow] "r"(src0.GetValidRow()), "i"(tile_shape::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } -} - -// TCMPS: compare src with scalar. The comparison mode is a compile-time -// template parameter encoded into B.DATR CMode[31:29]; scalar travels via the -// canonical B.IOR slot, never as a Tile source (PTO 0.58). -template -void TCMPS(tile_shape_out &dst, tile_shape_in &src, - typename tile_shape_in::DType s) { - static_assert(is_valid_cmp_mode(Mode), "TCMPS requires a valid CmpMode"); - static_assert(tile_shape_in::Rows == tile_shape_out::Rows && - tile_shape_in::Cols == tile_shape_out::Cols, - "TCMPS output shape must match input shape"); - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape_in::DType sv = s; - if constexpr (Mode == CmpMode::EQ) { - asm volatile( - "BSTART.TEPL 45, %D[TCode]\n" - "B.DATR Zero, EQ\n" - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.DIM zero, %c[Cols], ->lb2\n" - "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" - "B.IOR [%[Scalar]],[]\n" - "" - : [D] "=Tr"(dst.data()) - : [TCode] "i"(type_traits::TypeCode), - [VCOL] "ri"(src.GetValidCol()), - [VROW] "ri"(src.GetValidRow()), - [Cols] "i"(tile_shape_in::Cols), - [S] "Tr"(src.data()), - [TSize] "i"(tile_type_traits::TilesizeCode), - [Scalar] "r"(sv) - ); - } else if constexpr (Mode == CmpMode::NE) { - asm volatile( - "BSTART.TEPL 45, %D[TCode]\n" - "B.DATR Zero, NE\n" - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.DIM zero, %c[Cols], ->lb2\n" - "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" - "B.IOR [%[Scalar]],[]\n" - "" - : [D] "=Tr"(dst.data()) - : [TCode] "i"(type_traits::TypeCode), - [VCOL] "ri"(src.GetValidCol()), - [VROW] "ri"(src.GetValidRow()), - [Cols] "i"(tile_shape_in::Cols), - [S] "Tr"(src.data()), - [TSize] "i"(tile_type_traits::TilesizeCode), - [Scalar] "r"(sv) - ); - } else if constexpr (Mode == CmpMode::LT) { - asm volatile( - "BSTART.TEPL 45, %D[TCode]\n" - "B.DATR Zero, LT\n" - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.DIM zero, %c[Cols], ->lb2\n" - "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" - "B.IOR [%[Scalar]],[]\n" - "" - : [D] "=Tr"(dst.data()) - : [TCode] "i"(type_traits::TypeCode), - [VCOL] "ri"(src.GetValidCol()), - [VROW] "ri"(src.GetValidRow()), - [Cols] "i"(tile_shape_in::Cols), - [S] "Tr"(src.data()), - [TSize] "i"(tile_type_traits::TilesizeCode), - [Scalar] "r"(sv) - ); - } else if constexpr (Mode == CmpMode::GT) { - asm volatile( - "BSTART.TEPL 45, %D[TCode]\n" - "B.DATR Zero, GT\n" - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.DIM zero, %c[Cols], ->lb2\n" - "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" - "B.IOR [%[Scalar]],[]\n" - "" - : [D] "=Tr"(dst.data()) - : [TCode] "i"(type_traits::TypeCode), - [VCOL] "ri"(src.GetValidCol()), - [VROW] "ri"(src.GetValidRow()), - [Cols] "i"(tile_shape_in::Cols), - [S] "Tr"(src.data()), - [TSize] "i"(tile_type_traits::TilesizeCode), - [Scalar] "r"(sv) - ); - } else if constexpr (Mode == CmpMode::LE) { - asm volatile( - "BSTART.TEPL 45, %D[TCode]\n" - "B.DATR Zero, LE\n" - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.DIM zero, %c[Cols], ->lb2\n" - "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" - "B.IOR [%[Scalar]],[]\n" - "" - : [D] "=Tr"(dst.data()) - : [TCode] "i"(type_traits::TypeCode), - [VCOL] "ri"(src.GetValidCol()), - [VROW] "ri"(src.GetValidRow()), - [Cols] "i"(tile_shape_in::Cols), - [S] "Tr"(src.data()), - [TSize] "i"(tile_type_traits::TilesizeCode), - [Scalar] "r"(sv) - ); - } else if constexpr (Mode == CmpMode::GE) { - asm volatile( - "BSTART.TEPL 45, %D[TCode]\n" - "B.DATR Zero, GE\n" - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.DIM zero, %c[Cols], ->lb2\n" - "B.IOT %[S], mask=1111, last, ->%[D]<%Z[TSize]>\n" - "B.IOR [%[Scalar]],[]\n" - "" - : [D] "=Tr"(dst.data()) - : [TCode] "i"(type_traits::TypeCode), - [VCOL] "ri"(src.GetValidCol()), - [VROW] "ri"(src.GetValidRow()), - [Cols] "i"(tile_shape_in::Cols), - [S] "Tr"(src.data()), - [TSize] "i"(tile_type_traits::TilesizeCode), - [Scalar] "r"(sv) - ); - } -} - -// Deprecated EQ-default form retained for old callers. -template -void TCMPS(tile_shape_out &dst, tile_shape_in &src, - typename tile_shape_in::DType s) { - TCMPS(dst, src, s); -} - -// TLRELU: leaky ReLU with scalar slope -template -void TLRELU(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { - static_assert(pto_dependent_false_v, - "TLRELU is retired in PTO ISA 0.58.3; no active replacement. Remove the call or migrate to the active surface."); -} - - -// TAXPY: AXPY-style fused update (DavinciOO ext) -template -void TAXPY(tile_shape &dst, tile_shape &src, typename tile_shape::DType s) { - static_assert(pto_dependent_false_v, - "TAXPY is retired in PTO ISA 0.58.3; no active replacement. Remove the call or migrate to the active surface."); -} - - -// TADDSC: dst = src0 + scalar + src1 -template -void TADDSC(tile_shape &dst, tile_shape &src0, typename tile_shape::DType s, tile_shape &src1) { - static_assert(pto_dependent_false_v, - "TADDSC is retired in PTO ISA 0.58.3; no active replacement. Remove the call or migrate to the active surface."); -} - - -// TSUBSC: dst = src0 - scalar + src1 -template -void TSUBSC(tile_shape &dst, tile_shape &src0, typename tile_shape::DType s, tile_shape &src1) { - static_assert(pto_dependent_false_v, - "TSUBSC is retired in PTO ISA 0.58.3; no active replacement. Remove the call or migrate to the active surface."); + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } - -// TSELS: select between src tile and scalar using mask +// TPARTMAX: partial-valid max (moved from Mode 0) template -void TSELS(tile_shape &dst, tile_shape &src0, typename tile_shape::DType s, tile_shape &src1) { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; +void TPARTMAX(tile_shape &dst, tile_shape &src0, tile_shape &src1) { + if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { asm volatile( - "BSTART.TEPL 58, %D1\n" + "BSTART.TEPL 115, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" - "B.IOR [%8],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src0.GetValidCol()), - "ri"(src0.GetValidRow()), + "i"(tile_shape::ValidCol), + "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), "Tr"(src0.data()), "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); -} - -// TEXPANDS: broadcast scalar into dst tile -template -void TEXPANDS(tile_shape &dst, typename tile_shape::DType s) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { asm volatile( - "BSTART.TEPL 59, %D1\n" + "BSTART.TEPL 115, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT mask=1111, last, ->%0<%Z5>\n" - "B.IOR [%6],[]\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), "i"(tile_shape::ValidCol), - "i"(tile_shape::ValidRow), + [src0____dimrow] "r"(src0.GetValidRow()), "i"(tile_shape::Cols), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } else { - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType sv = s; + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { asm volatile( - "BSTART.TEPL 59, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 115, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT mask=1111, last, ->%0<%Z5>\n" - "B.IOR [%6],[]\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + [src0____dimcol] "r"(src0.GetValidCol()), + "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), - "i"(tile_type_traits::TilesizeCode), - "r"(sv) - ); - } -} - - -//===--- TEPL Mode 0 extension: TFMA (fused multiply-add, opcode 28) ---===// + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { + asm volatile( + "BSTART.TEPL 115, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src0____dimcol] "r"(src0.GetValidCol()), + [src0____dimrow] "r"(src0.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } +} -// TFMA: dst = src0 * src1 + src2 (fused element-wise multiply-add) +// TPARTMIN: partial-valid min (moved from Mode 0) template -void TFMA(tile_shape &dst, tile_shape &src0, tile_shape &src1, tile_shape &src2) { +void TPARTMIN(tile_shape &dst, tile_shape &src0, tile_shape &src1) { if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { asm volatile( - "BSTART.TEPL 28, %D1\n" + "BSTART.TEPL 116, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, %6, mask=1111\n" - "B.IOT %7, mask=1111, last, ->%0<%Z8>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), @@ -8891,48 +13357,77 @@ void TFMA(tile_shape &dst, tile_shape &src0, tile_shape &src1, tile_shape &src2) "i"(tile_shape::Cols), "Tr"(src0.data()), "Tr"(src1.data()), - "Tr"(src2.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow < 0) { asm volatile( - "BSTART.TEPL 28, %D1\n" + "BSTART.TEPL 116, %D1\n" "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape::ValidCol), + [src0____dimrow] "r"(src0.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape::ValidCol < 0 && tile_shape::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 116, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, %6, mask=1111\n" - "B.IOT %7, mask=1111, last, ->%0<%Z8>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(src0.GetValidCol()), - "ri"(src0.GetValidRow()), + [src0____dimcol] "r"(src0.GetValidCol()), + "i"(tile_shape::ValidRow), "i"(tile_shape::Cols), "Tr"(src0.data()), "Tr"(src1.data()), - "Tr"(src2.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } else { + asm volatile( + "BSTART.TEPL 116, %D1\n" + "B.DIM %[src0____dimcol], 0, ->lb0\n" + "B.DIM %[src0____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src0____dimcol] "r"(src0.GetValidCol()), + [src0____dimrow] "r"(src0.GetValidRow()), + "i"(tile_shape::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } +//===--- TEPL Mode 2: reduction/broadcast ops (BSTART.TEPL) ---===// +// opcode = Mode(2) * 32 + Function = 64 + Function. -//===--- TEPL Mode 3: complex ops (opcode = 96 + Function) ---===// - -// TEXTRACT: extract sub-tile (indexRow, indexCol via B.IOR) +// TROWSUM: row sum reduction template -void TEXTRACT(tile_shape_out &dst, tile_shape_in &src, int32_t indexRow, int32_t indexCol) { +void TROWSUM(tile_shape_out &dst, tile_shape_in &src) { + // ASL (row reduction): B.DIM describes the SOURCE geometry + // (ValidCol/ValidRow/Col); the destination is rule-derived: one + // column, ValidRow = source.ValidRow. + static_assert(tile_shape_out::ValidCol == DYNAMIC || (tile_shape_out::ValidCol == 1 && tile_shape_out::Cols == 1), + "TROWSUM destination must be a single-column tile (N x 1)"); + static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_in::ValidRow == DYNAMIC || tile_shape_out::ValidRow == tile_shape_in::ValidRow, + "TROWSUM destination valid rows must equal the source valid rows"); if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { - // Anti-fold: keep compile-time-constant indices (e.g. 0) off the zero - // register so B.IOR [zero,...]/[...,zero] still matches an instruction. - volatile int32_t irv = indexRow; - volatile int32_t icv = indexCol; asm volatile( - "BSTART.TEPL 98, %D1\n" + "BSTART.TEPL 64, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7,%8],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), @@ -8940,145 +13435,72 @@ void TEXTRACT(tile_shape_out &dst, tile_shape_in &src, int32_t indexRow, int32_t "i"(tile_shape_in::ValidRow), "i"(tile_shape_in::Cols), "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(irv), - "r"(icv) - ); - } else { - // Anti-fold: keep compile-time-constant indices (e.g. 0) off the zero - // register so B.IOR [zero,...]/[...,zero] still matches an instruction. - volatile int32_t irv = indexRow; - volatile int32_t icv = indexCol; + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { + const size_t valid_col = src.GetValidCol(); asm volatile( - "BSTART.TEPL 98, %D1\n" + "BSTART.TEPL 64, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[valid_col], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7,%8],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), + "i"(tile_shape_in::ValidCol), + [valid_col] "r"(src.GetValidCol()), "i"(tile_shape_in::Cols), "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(irv), - "r"(icv) - ); - } -} - -// TINSERT: insert sub-tile (indexRow, indexCol via B.IOR) -template -void TINSERT(tile_shape_out &dst, tile_shape_in &src, int32_t indexRow, int32_t indexCol) { - if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { - // Anti-fold: keep compile-time-constant indices (e.g. 0) off the zero - // register so B.IOR [zero,...]/[...,zero] still matches an instruction. - volatile int32_t irv = indexRow; - volatile int32_t icv = indexCol; + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { + const size_t valid_col = src.GetValidCol(); asm volatile( - "BSTART.TEPL 99, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 64, %D1\n" + "B.DIM %[valid_col], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7,%8],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_in::ValidCol), + [valid_col] "r"(src.GetValidCol()), "i"(tile_shape_in::ValidRow), "i"(tile_shape_in::Cols), "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(irv), - "r"(icv) - ); - } else { - // Anti-fold: keep compile-time-constant indices (e.g. 0) off the zero - // register so B.IOR [zero,...]/[...,zero] still matches an instruction. - volatile int32_t irv = indexRow; - volatile int32_t icv = indexCol; + "i"(tile_type_traits::TilesizeCode) + ); } else { + const size_t valid_col = src.GetValidCol(); + const size_t valid_row = src.GetValidRow(); asm volatile( - "BSTART.TEPL 99, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 64, %D1\n" + "B.DIM %[valid_col], 0, ->lb0\n" + "B.DIM %[valid_row], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7,%8],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), + [valid_col] "r"(valid_col), + [valid_row] "r"(valid_row), "i"(tile_shape_in::Cols), "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "r"(irv), - "r"(icv) - ); - } -} - -// TIMG2COL: GM feature-map image-to-column materialization. The source-only -// B.IOR pair carries GMBase followed by the three packed parameter GPRs. -template - requires(tile_shape_out::Loc == Location::Left && - (tile_shape_out::BFractal == BLayout::CubeM16 || - tile_shape_out::BFractal == BLayout::CubeM32)) -void TIMG2COL(tile_shape_out &dst, gm_shape &src, TIMG2COLParams params) { - static_assert(tile_shape_out::ValidRow != 0 && - tile_shape_out::ValidCol != 0, - "TIMG2COL valid dimensions must be nonzero"); - static_assert(type_traits::TypeCode == __type_fp32 || - type_traits::TypeCode == __type_fp16 || - type_traits::TypeCode == __type_bf16 || - type_traits::TypeCode == __type_int32 || - type_traits::TypeCode == __type_int16 || - type_traits::TypeCode == __type_int8 || - type_traits::TypeCode == __type_uint32 || - type_traits::TypeCode == __type_uint16 || - type_traits::TypeCode == __type_uint8, - "TIMG2COL DataType is not supported by the ASL contract"); - volatile uint64_t param0 = params.param0; - volatile uint64_t param1 = params.param1; - volatile uint64_t param2 = params.param2; - asm volatile( - "BSTART.TIMG2COL %D[DataType]\n" - "B.DATR %c[Layout], DTYPE_NONE, Zero\n" - "B.DIM zero, %c[ValidCol], ->lb0\n" - "B.DIM zero, %c[ValidRow], ->lb1\n" - "B.DIM zero, %c[TotalCol], ->lb2\n" - "B.IOR [%[GMBase], zero, zero], []\n" - "B.IOR [%[Param0], %[Param1], %[Param2]], []\n" - "B.IOT mask=1111, last, ->%[Dst]<%Z[TileSize]>\n" - : [Dst] "=Tr"(dst.data()) - : [GMBase] "r"(src.data()), - [DataType] "i"(type_traits::TypeCode), - [Layout] "i"(tile_shape_out::BFractal == BLayout::CubeM16 ? - BLayout::ND2M16 : BLayout::ND2M32), - [ValidCol] "r"(dst.GetValidCol()), - [ValidRow] "r"(dst.GetValidRow()), - [TotalCol] "i"(tile_shape_out::Cols), - [Param0] "r"(param0), [Param1] "r"(param1), [Param2] "r"(param2), - [TileSize] "i"(tile_type_traits::TilesizeCode) - : "memory"); -} - -template -void TIMG2COL(tile_shape_out &dst, gm_shape &src, - uint64_t param0, uint64_t param1, uint64_t param2) { - TIMG2COL(dst, src, TIMG2COLParams{param0, param1, param2}); + "i"(tile_type_traits::TilesizeCode) + ); } } -// TFILLPAD: copy valid region and fill padding +// TROWMAX: row max reduction template -void TFILLPAD(tile_shape_out &dst, tile_shape_in &src) { +void TROWMAX(tile_shape_out &dst, tile_shape_in &src) { + // ASL (row reduction): B.DIM describes the SOURCE geometry + // (ValidCol/ValidRow/Col); the destination is rule-derived: one + // column, ValidRow = source.ValidRow. + static_assert(tile_shape_out::ValidCol == DYNAMIC || (tile_shape_out::ValidCol == 1 && tile_shape_out::Cols == 1), + "TROWMAX destination must be a single-column tile (N x 1)"); + static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_in::ValidRow == DYNAMIC || tile_shape_out::ValidRow == tile_shape_in::ValidRow, + "TROWMAX destination valid rows must equal the source valid rows"); if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { asm volatile( - "BSTART.TEPL 101, %D1\n" + "BSTART.TEPL 65, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" @@ -9088,469 +13510,507 @@ void TFILLPAD(tile_shape_out &dst, tile_shape_in &src) { : "i"(type_traits::TypeCode), "i"(tile_shape_in::ValidCol), "i"(tile_shape_in::ValidRow), - "i"(tile_shape_in::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) - ); - } else { - asm volatile( - "BSTART.TEPL 101, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" - "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "" - : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), - "i"(tile_shape_in::Cols), - "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TCI: contiguous integer sequence generation -template -void TCI(tile_shape &dst, T s) { - static_assert(std::is_same::value, - "TCI destination and start must have the same type"); - static_assert(descending == 0 || descending == 1, - "TCI direction must be ascending (0) or descending (1)"); - static_assert(tile_shape::Loc == Location::Vec, - "TCI requires a Local vector tile"); - static_assert(tile_shape::isRowMajor && !tile_shape::isBoxedLayout, - "TCI requires an unboxed RowMajor tile"); - static_assert(tile_shape::ValidRow == 1, - "TCI requires ValidRow == 1"); - static_assert(tile_shape::ValidCol > 0 && - tile_shape::Cols >= tile_shape::ValidCol, - "TCI requires 0 < ValidCol <= Cols"); - static_assert(std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value, - "TCI supports only S32, S16, U32, and U16"); - // Anti-fold: keep a compile-time-constant scalar (e.g. 0) off the zero - // register so B.IOR [zero],[] still matches an instruction. - volatile typename tile_shape::DType startValue = s; - volatile uint32_t directionValue = descending; - asm volatile( - "BSTART.TEPL 102, %D[DataType]\n" - "B.DIM zero, %c[ValidCol], ->lb0\n" - "B.DIM zero, %c[PhysicalCol], ->lb2\n" - "B.IOR [%[Start],%[Direction]],[]\n" - "B.IOT mask=1111, last, ->%[Dst]<%Z[TileSize]>\n" - : [Dst] "=Tr"(dst.data()) - : [DataType] "i"(type_traits::TypeCode), - [ValidCol] "ri"(dst.GetValidCol()), - [PhysicalCol] "i"(tile_shape::Cols), - [TileSize] "i"(tile_type_traits::TilesizeCode), - [Start] "r"(startValue), - [Direction] "r"(directionValue) - ); -} - -// TTRI: triangular mask generation -template -void TTRI(tile_shape &dst) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { asm volatile( - "BSTART.TEPL 103, %D1\n" + "BSTART.TEPL 65, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT mask=1111, last, ->%0<%Z5>\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(tile_shape::ValidCol), - "i"(tile_shape::ValidRow), - "i"(tile_shape::Cols), - "i"(tile_type_traits::TilesizeCode) - ); - } else { + : "i"(type_traits::TypeCode), + "i"(tile_shape_in::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { asm volatile( - "BSTART.TEPL 103, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 65, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT mask=1111, last, ->%0<%Z5>\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), - "i"(tile_shape::Cols), - "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TRANDOM: counter-based random tile generation -template -void TRANDOM(tile_shape &dst, typename tile_shape::DType s) { - static_assert(pto_dependent_false_v, - "TRANDOM is retired in PTO ISA 0.58.3; no active replacement. Remove the call or migrate to the active surface."); -} - - -// TQUANT: FP32 -> S8/U8 quantization with B.DATR RMode/Sat and B.IOR -// multiplier/zero-point (PTO ISA 0.58.3 TEPL Mode3 Fn10 / selector 0x06A). -// RMode and Sat are B.DATR fields: RMode accepts a numeric immediate (the -// parser maps mnemonic or numeric), Sat only the NOSAT/SAT token, so Sat is -// selected with if constexpr. The multiplier travels as its raw FP32 bits in -// a GPR and zeroPoint in another. -template -void TQUANT(tile_shape_out &dst, tile_shape_in &src, float multiplier = 1.0f, - int32_t zeroPoint = 0) { - static_assert(is_valid_round_mode(Mode), "TQUANT invalid rounding mode"); - static_assert(type_traits::TypeCode == - __type_fp32, - "TQUANT source must be FP32"); - static_assert(type_traits::TypeCode == - __type_int8 || - type_traits::TypeCode == - __type_uint8, - "TQUANT destination must be S8 or U8"); - static_assert(tile_shape_out::Loc == Location::Vec && - tile_shape_in::Loc == Location::Vec && - tile_shape_out::isRowMajor && - tile_shape_in::isRowMajor && - !tile_shape_out::isBoxedLayout && - !tile_shape_in::isBoxedLayout, - "TQUANT operands must be Local RowMajor numeric Tiles"); - static_assert( - tile_shape_out::Rows == tile_shape_in::Rows && - tile_shape_out::Cols == tile_shape_in::Cols && - (tile_shape_out::ValidRow == DYNAMIC || - tile_shape_in::ValidRow == DYNAMIC || - tile_shape_out::ValidRow == tile_shape_in::ValidRow) && - (tile_shape_out::ValidCol == DYNAMIC || - tile_shape_in::ValidCol == DYNAMIC || - tile_shape_out::ValidCol == tile_shape_in::ValidCol), - "TQUANT source and destination logical shapes must match"); - uint32_t multiplierBits; - __builtin_memcpy(&multiplierBits, &multiplier, sizeof(multiplier)); - volatile uint32_t mult = multiplierBits; - volatile int32_t zp = zeroPoint; - if constexpr (Mode == RoundMode::RNE && Saturate) { - asm volatile( - "BSTART.TEPL 106, %D[SType]\n" - // LLVM currently names encoded RMode zero RNONE. PTO ISA 0.58.3 defines - // that encoding as the operation default, which is RNE for TQUANT. - "B.DATR %D[__pto_DstType], RNONE, sat\n" - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.DIM zero, %c[Col], ->lb2\n" - "B.IOR [%[Mult], %[ZP]], []\n" - "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" - : [Dst] "=&Tr"(dst.data()) - : [Src] "Tr"(src.data()), - [SType] "i"(type_traits::TypeCode), - [__pto_DstType] "i"(type_traits::TypeCode), - [VCOL] "ri"(src.GetValidCol()), [VROW] "ri"(src.GetValidRow()), - [Col] "i"(tile_shape_in::Cols), - [Mult] "r"(mult), [ZP] "r"(zp), - [DstSize] "i"(tile_shape_out::TilesizeCode) - ); - } else if constexpr (Mode == RoundMode::RNE) { - asm volatile( - "BSTART.TEPL 106, %D[SType]\n" - "B.DATR %D[__pto_DstType], RNONE\n" - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.DIM zero, %c[Col], ->lb2\n" - "B.IOR [%[Mult], %[ZP]], []\n" - "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" - : [Dst] "=&Tr"(dst.data()) - : [Src] "Tr"(src.data()), - [SType] "i"(type_traits::TypeCode), - [__pto_DstType] "i"(type_traits::TypeCode), - [VCOL] "ri"(src.GetValidCol()), [VROW] "ri"(src.GetValidRow()), - [Col] "i"(tile_shape_in::Cols), - [Mult] "r"(mult), [ZP] "r"(zp), - [DstSize] "i"(tile_shape_out::TilesizeCode) - ); - } else if constexpr (Saturate) { - asm volatile( - "BSTART.TEPL 106, %D[SType]\n" - PTO_RMODE_DATR_ASM(", sat\n") - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.DIM zero, %c[Col], ->lb2\n" - "B.IOR [%[Mult], %[ZP]], []\n" - "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" - : [Dst] "=&Tr"(dst.data()) - : [Src] "Tr"(src.data()), - [SType] "i"(type_traits::TypeCode), - [__pto_DstType] "i"(type_traits::TypeCode), - [RMode] "i"(static_cast(Mode)), - [VCOL] "ri"(src.GetValidCol()), [VROW] "ri"(src.GetValidRow()), - [Col] "i"(tile_shape_in::Cols), - [Mult] "r"(mult), [ZP] "r"(zp), - [DstSize] "i"(tile_shape_out::TilesizeCode) - ); - } else { - asm volatile( - "BSTART.TEPL 106, %D[SType]\n" - PTO_RMODE_DATR_ASM("\n") - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.DIM zero, %c[Col], ->lb2\n" - "B.IOR [%[Mult], %[ZP]], []\n" - "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" - : [Dst] "=&Tr"(dst.data()) - : [Src] "Tr"(src.data()), - [SType] "i"(type_traits::TypeCode), - [__pto_DstType] "i"(type_traits::TypeCode), - [RMode] "i"(static_cast(Mode)), - [VCOL] "ri"(src.GetValidCol()), [VROW] "ri"(src.GetValidRow()), - [Col] "i"(tile_shape_in::Cols), - [Mult] "r"(mult), [ZP] "r"(zp), - [DstSize] "i"(tile_shape_out::TilesizeCode) - ); - } + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { + asm volatile( + "BSTART.TEPL 65, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } -// TDEQUANT: S8/U8 -> FP32 dequantization (TEPL Mode3 Fn11 / 0x06B). -// B.DATR carries only FP32 + RMode (Sat is always false per spec). -template -void TDEQUANT(tile_shape_out &dst, tile_shape_in &src, float multiplier = 1.0f, - int32_t zeroPoint = 0) { - static_assert(is_valid_round_mode(Mode), "TDEQUANT invalid rounding mode"); - static_assert(type_traits::TypeCode == - __type_fp32, - "TDEQUANT destination must be FP32"); - static_assert(type_traits::TypeCode == - __type_int8 || - type_traits::TypeCode == - __type_uint8, - "TDEQUANT source must be S8 or U8"); - static_assert(tile_shape_out::Loc == Location::Vec && - tile_shape_in::Loc == Location::Vec && - tile_shape_out::isRowMajor && - tile_shape_in::isRowMajor && - !tile_shape_out::isBoxedLayout && - !tile_shape_in::isBoxedLayout, - "TDEQUANT operands must be Local RowMajor numeric Tiles"); - static_assert( - tile_shape_out::Rows == tile_shape_in::Rows && - tile_shape_out::Cols == tile_shape_in::Cols && - (tile_shape_out::ValidRow == DYNAMIC || - tile_shape_in::ValidRow == DYNAMIC || - tile_shape_out::ValidRow == tile_shape_in::ValidRow) && - (tile_shape_out::ValidCol == DYNAMIC || - tile_shape_in::ValidCol == DYNAMIC || - tile_shape_out::ValidCol == tile_shape_in::ValidCol), - "TDEQUANT source and destination logical shapes must match"); - uint32_t multiplierBits; - __builtin_memcpy(&multiplierBits, &multiplier, sizeof(multiplier)); - volatile uint32_t mult = multiplierBits; - volatile int32_t zp = zeroPoint; - if constexpr (Mode == RoundMode::RNE) { - asm volatile( - "BSTART.TEPL 107, %D[SType]\n" - "B.DATR FP32, RNONE\n" - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.DIM zero, %c[Col], ->lb2\n" - "B.IOR [%[Mult], %[ZP]], []\n" - "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" - : [Dst] "=&Tr"(dst.data()) - : [Src] "Tr"(src.data()), - [SType] "i"( - type_traits::TypeCode == __type_int8 - ? __type_int8 : __type_uint8), - [VCOL] "ri"(src.GetValidCol()), [VROW] "ri"(src.GetValidRow()), - [Col] "i"(tile_shape_in::Cols), - [Mult] "r"(mult), [ZP] "r"(zp), - [DstSize] "i"(tile_shape_out::TilesizeCode) - ); - } else { - asm volatile( - "BSTART.TEPL 107, %D[SType]\n" - PTO_RMODE_DATR_ASM("\n") - "B.DIM zero, %c[VCOL], ->lb0\n" - "B.DIM zero, %c[VROW], ->lb1\n" - "B.DIM zero, %c[Col], ->lb2\n" - "B.IOR [%[Mult], %[ZP]], []\n" - "B.IOT %[Src], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" - : [Dst] "=&Tr"(dst.data()) - : [Src] "Tr"(src.data()), - [SType] "i"( - type_traits::TypeCode == __type_int8 - ? __type_int8 : __type_uint8), - [__pto_DstType] "i"(__type_fp32), - [RMode] "i"(static_cast(Mode)), - [VCOL] "ri"(src.GetValidCol()), [VROW] "ri"(src.GetValidRow()), - [Col] "i"(tile_shape_in::Cols), - [Mult] "r"(mult), [ZP] "r"(zp), - [DstSize] "i"(tile_shape_out::TilesizeCode) - ); - } +// TROWMIN: row min reduction +template +void TROWMIN(tile_shape_out &dst, tile_shape_in &src) { + // ASL (row reduction): B.DIM describes the SOURCE geometry + // (ValidCol/ValidRow/Col); the destination is rule-derived: one + // column, ValidRow = source.ValidRow. + static_assert(tile_shape_out::ValidCol == DYNAMIC || (tile_shape_out::ValidCol == 1 && tile_shape_out::Cols == 1), + "TROWMIN destination must be a single-column tile (N x 1)"); + static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_in::ValidRow == DYNAMIC || tile_shape_out::ValidRow == tile_shape_in::ValidRow, + "TROWMIN destination valid rows must equal the source valid rows"); + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 66, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_in::ValidCol), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 66, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_in::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 66, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { + asm volatile( + "BSTART.TEPL 66, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } -// TDEQUANT: profile-defined dequantization +// TROWPROD: row product reduction template -void TDEQUANT(tile_shape_out &dst, tile_shape_in &src) { - TDEQUANT(dst, src); +void TROWPROD(tile_shape_out &dst, tile_shape_in &src) { + // ASL (row reduction): B.DIM describes the SOURCE geometry + // (ValidCol/ValidRow/Col); the destination is rule-derived: one + // column, ValidRow = source.ValidRow. + static_assert(tile_shape_out::ValidCol == DYNAMIC || (tile_shape_out::ValidCol == 1 && tile_shape_out::Cols == 1), + "TROWPROD destination must be a single-column tile (N x 1)"); + static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_in::ValidRow == DYNAMIC || tile_shape_out::ValidRow == tile_shape_in::ValidRow, + "TROWPROD destination valid rows must equal the source valid rows"); + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 67, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_in::ValidCol), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 67, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_in::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 67, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { + asm volatile( + "BSTART.TEPL 67, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } -// TSORT: stable per-row group sort producing sorted values (FP16/FP32) and -// group-local original indices (U32). PTO ISA 0.58.3: TEPL Mode 3 Function 12 -// (selector 0x06c). Each row sorts its columns in `sortWidth`-width groups -// from column 0; the last group may be short. ascending when descending is -// false, descending otherwise. sortWidth must be 1..64 (0/LB0 omitted -> 32). -// -// Encoding-carrier note: the LinxV5 backend has no canonical BSTART.SFU -// TSORT mnemonic yet, so the TEPL 108 (TSORT32) carrier is used here. The -// bundle shape below matches the normative contract: only LB0, a single -// B.IOR with the descending flag, a source+value-dest B.IOT, then a -// destination-only index B.IOT. The two destinations use their own -// TileSizeCode (value FP16/FP32 vs index U32 differ in bytes). -template -void TSORT(ValueDstTile &valueDst, IndexDstTile &indexDst, - SourceTile &source, uint32_t sortWidth = 32, - bool descending = false) { - static_assert(std::is_same_v, - "TSORT value destination dtype must match source dtype"); - static_assert(type_traits::TypeCode == - __type_fp16 || - type_traits::TypeCode == - __type_fp32, - "TSORT value source/destination must be FP16 or FP32"); - static_assert(type_traits::TypeCode == - __type_uint32, - "TSORT index destination must be U32"); - static_assert(ValueDstTile::Rows == SourceTile::Rows && - ValueDstTile::Cols == SourceTile::Cols && - IndexDstTile::Rows == SourceTile::Rows && - IndexDstTile::Cols == SourceTile::Cols && - (ValueDstTile::ValidRow == DYNAMIC || - SourceTile::ValidRow == DYNAMIC || - ValueDstTile::ValidRow == SourceTile::ValidRow) && - (ValueDstTile::ValidCol == DYNAMIC || - SourceTile::ValidCol == DYNAMIC || - ValueDstTile::ValidCol == SourceTile::ValidCol) && - (IndexDstTile::ValidRow == DYNAMIC || - SourceTile::ValidRow == DYNAMIC || - IndexDstTile::ValidRow == SourceTile::ValidRow) && - (IndexDstTile::ValidCol == DYNAMIC || - SourceTile::ValidCol == DYNAMIC || - IndexDstTile::ValidCol == SourceTile::ValidCol), - "TSORT value/index destinations must match source logical " - "and valid shapes"); - static_assert(ValueDstTile::Loc == Location::Vec && - IndexDstTile::Loc == Location::Vec && - SourceTile::Loc == Location::Vec && - ValueDstTile::isRowMajor && IndexDstTile::isRowMajor && - SourceTile::isRowMajor && - !ValueDstTile::isBoxedLayout && - !IndexDstTile::isBoxedLayout && - !SourceTile::isBoxedLayout, - "TSORT operands must be Local RowMajor VEC Tiles"); +// TROWEXPAND: broadcast first element of each row +template +void TROWEXPAND(tile_shape_out &dst, tile_shape_in &src) { + // ASL (expansion): row expansion broadcasts a one-column source; the + // destination geometry comes from the destination B.DIM, not the source. + static_assert(tile_shape_in::ValidCol == DYNAMIC || tile_shape_in::ValidCol == 1, + "TROWEXPAND source must be a one-column broadcast tile"); + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 68, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_out::ValidCol), + "i"(tile_shape_out::ValidRow), + "i"(tile_shape_out::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 68, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 68, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + "i"(tile_shape_out::ValidRow), + "i"(tile_shape_out::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { + asm volatile( + "BSTART.TEPL 68, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } +} - // Anti-fold: keep the 0/1 descending flag off the zero register so the - // B.IOR binder still carries a real GPR (B.IOR [zero],[] does not match). - volatile uint32_t descendingValue = descending ? 1u : 0u; +// TROWARGMAX: row argmax (DavinciOO ext) +template +void TROWARGMAX(tile_shape_out &dst, tile_shape_in &src) { + // ASL (row reduction): B.DIM describes the SOURCE geometry + // (ValidCol/ValidRow/Col); the destination is rule-derived: one + // column, ValidRow = source.ValidRow. + static_assert(tile_shape_out::ValidCol == DYNAMIC || (tile_shape_out::ValidCol == 1 && tile_shape_out::Cols == 1), + "TROWARGMAX destination must be a single-column tile (N x 1)"); + static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_in::ValidRow == DYNAMIC || tile_shape_out::ValidRow == tile_shape_in::ValidRow, + "TROWARGMAX destination valid rows must equal the source valid rows"); + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 76, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_in::ValidCol), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 76, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_in::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 76, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { asm volatile( - "BSTART.TEPL 108, %D[DataType]\n" - "B.DIM %[SortWidth], 0, ->lb0\n" - "B.IOR [%[Descending]], []\n" - "B.IOT %[Source], mask=1111, ->%[ValueDst]<%Z[ValueTileSize]>\n" - "B.IOT mask=1111, last, ->%[IndexDst]<%Z[IndexTileSize]>\n" - : [ValueDst] "=&Tr"(valueDst.data()), - [IndexDst] "=&Tr"(indexDst.data()) - : [Source] "Tr"(source.data()), - [DataType] "i"(type_traits::TypeCode), - [SortWidth] "r"(sortWidth), - [Descending] "r"(descendingValue), - // Use each destination's logical TilesizeCode: FP16 value (e.g. 2 KB - // for 32x32) and U32 index (4 KB) differ, and B.IOT must carry the - // per-PE logical size, not the uniform 4 KB storage carrier. - [ValueTileSize] "i"(ValueDstTile::TilesizeCode), - [IndexTileSize] "i"(IndexDstTile::TilesizeCode) - ); + "BSTART.TEPL 76, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } -// Deprecated single-output sort: does not match the PTO ISA 0.58.3 TSORT -// contract (no U32 index destination, legacy LB1/LB2 shape bundle). Kept as -// a migration diagnostic that fails at instantiation. +// TROWARGMIN: row argmin (DavinciOO ext) template -void TSORT32(tile_shape_out &dst, tile_shape_in &src) { - static_assert(pto_dependent_false_v, - "TSORT32 is removed; use TSORT(valueDst, indexDst, source, " - "sortWidth, descending), which emits the PTO ISA 0.58.3 " - "value+index dual-output bundle"); +void TROWARGMIN(tile_shape_out &dst, tile_shape_in &src) { + // ASL (row reduction): B.DIM describes the SOURCE geometry + // (ValidCol/ValidRow/Col); the destination is rule-derived: one + // column, ValidRow = source.ValidRow. + static_assert(tile_shape_out::ValidCol == DYNAMIC || (tile_shape_out::ValidCol == 1 && tile_shape_out::Cols == 1), + "TROWARGMIN destination must be a single-column tile (N x 1)"); + static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_in::ValidRow == DYNAMIC || tile_shape_out::ValidRow == tile_shape_in::ValidRow, + "TROWARGMIN destination valid rows must equal the source valid rows"); + if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 77, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_in::ValidCol), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 77, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_in::ValidCol), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_in::ValidCol < 0 && tile_shape_in::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 77, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + "i"(tile_shape_in::ValidRow), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { + asm volatile( + "BSTART.TEPL 77, %D1\n" + "B.DIM %[src____dimcol], 0, ->lb0\n" + "B.DIM %[src____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [src____dimcol] "r"(src.GetValidCol()), + [src____dimrow] "r"(src.GetValidRow()), + "i"(tile_shape_in::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } -// TMRGSORT: merge two sorted single-row sources into one destination -// (PTO ISA 0.58.3 TEPL Mode 3 Function 13 / selector 0x06D; canonical -// BSTART.SFU TMRGSORT). No B.DIM: the block carries only B.IOR RegSrc0 -// (0/1 ascending/descending) and one TwoSrc_Dst B.IOT with . -template -void TMRGSORT(DstTile &dst, LeftTile &left, RightTile &right, - bool descending = false) { - static_assert(std::is_same_v && - std::is_same_v, - "TMRGSORT dst/left/right must share one dtype"); - static_assert(type_traits::TypeCode == - __type_fp16 || - type_traits::TypeCode == - __type_fp32, - "TMRGSORT dtype must be FP16 or FP32"); - static_assert(DstTile::Loc == Location::Vec && - LeftTile::Loc == Location::Vec && - RightTile::Loc == Location::Vec && - DstTile::isRowMajor && LeftTile::isRowMajor && - RightTile::isRowMajor && !DstTile::isBoxedLayout && - !LeftTile::isBoxedLayout && !RightTile::isBoxedLayout, - "TMRGSORT operands must be Local RowMajor numeric Tiles"); - static_assert(DstTile::Rows == 1 && LeftTile::Rows == 1 && - RightTile::Rows == 1, - "TMRGSORT operands must be single-row Tiles"); - static_assert((LeftTile::ValidCol == DYNAMIC || LeftTile::ValidCol > 0) && - (RightTile::ValidCol == DYNAMIC || - RightTile::ValidCol > 0), - "TMRGSORT sources must be non-empty"); - static_assert( - (DstTile::ValidCol == DYNAMIC || LeftTile::ValidCol == DYNAMIC || - RightTile::ValidCol == DYNAMIC || - (DstTile::ValidCol == LeftTile::ValidCol + RightTile::ValidCol && - DstTile::Cols >= LeftTile::ValidCol + RightTile::ValidCol && - (DstTile::Cols & (DstTile::Cols - 1)) == 0 && - DstTile::Cols / 2 < LeftTile::ValidCol + RightTile::ValidCol)), - "TMRGSORT destination must contain the combined source columns"); - // Anti-fold: keep the 0/1 flag off the zero register (B.IOR [zero],[] does - // not match). - volatile uint32_t descendingValue = descending ? 1u : 0u; +// TCOLSUM: col sum reduction +template +void TCOLSUM(tile_shape_out &dst, tile_shape_in &src) { + // ASL (reduction-and-expansion): column-axis reduction destination has + // exactly one valid row and inherits the source column geometry. + static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_out::ValidRow == 1, + "TCOLSUM destination must have exactly one valid row (1 x N)"); + static_assert(tile_shape_out::ValidCol == DYNAMIC || + (tile_shape_out::ValidCol == tile_shape_in::ValidCol && + tile_shape_out::Cols == tile_shape_in::Cols), + "TCOLSUM destination columns must match the source columns"); + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { asm volatile( - "BSTART.TEPL 109, %D[DataType]\n" - "B.IOR [%[Descending]], []\n" - "B.IOT %[Left], %[Right], mask=1111, last, ->%[Dst]<%Z[DstSize]>\n" - : [Dst] "=&Tr"(dst.data()) - : [Left] "Tr"(left.data()), [Right] "Tr"(right.data()), - [Descending] "r"(descendingValue), - [DataType] "i"(type_traits::TypeCode), - [DstSize] "i"(DstTile::TilesizeCode) - ); + "BSTART.TEPL 80, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_out::ValidCol), + "i"(tile_shape_out::ValidRow), + "i"(tile_shape_out::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 80, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 80, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + "i"(tile_shape_out::ValidRow), + "i"(tile_shape_out::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { + asm volatile( + "BSTART.TEPL 80, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } -// TTRANS: tile transpose +// TCOLMAX: col max reduction template -void TTRANS(tile_shape_out &dst, tile_shape_in &src) { - if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +void TCOLMAX(tile_shape_out &dst, tile_shape_in &src) { + // ASL (reduction-and-expansion): column-axis reduction destination has + // exactly one valid row and inherits the source column geometry. + static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_out::ValidRow == 1, + "TCOLMAX destination must have exactly one valid row (1 x N)"); + static_assert(tile_shape_out::ValidCol == DYNAMIC || + (tile_shape_out::ValidCol == tile_shape_in::ValidCol && + tile_shape_out::Cols == tile_shape_in::Cols), + "TCOLMAX destination columns must match the source columns"); + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { asm volatile( - "BSTART.TEPL 110, %D1\n" + "BSTART.TEPL 81, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" @@ -9558,290 +14018,291 @@ void TTRANS(tile_shape_out &dst, tile_shape_in &src) { "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_in::ValidCol), - "i"(tile_shape_in::ValidRow), - "i"(tile_shape_in::Cols), + "i"(tile_shape_out::ValidCol), + "i"(tile_shape_out::ValidRow), + "i"(tile_shape_out::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { + asm volatile( + "BSTART.TEPL 81, %D1\n" + "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { + asm volatile( + "BSTART.TEPL 81, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM zero, %c3, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + "i"(tile_shape_out::ValidRow), + "i"(tile_shape_out::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else { asm volatile( - "BSTART.TEPL 110, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 81, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), - "i"(tile_shape_in::Cols), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } } -// TGATHER: index/mask tile gather -template -void TGATHER(tile_shape_out &dst, tile_shape_in &src, tile_shape_off &off) { - if constexpr (tile_shape_off::ValidCol > 0 && tile_shape_off::ValidRow > 0) { +// TCOLMIN: col min reduction +template +void TCOLMIN(tile_shape_out &dst, tile_shape_in &src) { + // ASL (reduction-and-expansion): column-axis reduction destination has + // exactly one valid row and inherits the source column geometry. + static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_out::ValidRow == 1, + "TCOLMIN destination must have exactly one valid row (1 x N)"); + static_assert(tile_shape_out::ValidCol == DYNAMIC || + (tile_shape_out::ValidCol == tile_shape_in::ValidCol && + tile_shape_out::Cols == tile_shape_in::Cols), + "TCOLMIN destination columns must match the source columns"); + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { asm volatile( - "BSTART.TEPL 111, %D1\n" + "BSTART.TEPL 82, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_off::ValidCol), - "i"(tile_shape_off::ValidRow), - "i"(tile_shape_off::Cols), + "i"(tile_shape_out::ValidCol), + "i"(tile_shape_out::ValidRow), + "i"(tile_shape_out::Cols), "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "Tr"(off.data()) - ); - } else { + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { asm volatile( - "BSTART.TEPL 111, %D1\n" + "BSTART.TEPL 82, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(off.GetValidCol()), - "ri"(off.GetValidRow()), - "i"(tile_shape_off::Cols), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "Tr"(off.data()) - ); - } -} - -// TSCATTER: index tile scatter -template -void TSCATTER(tile_shape_out &dst, tile_shape_in &src, tile_shape_off &off) { - if constexpr (tile_shape_off::ValidCol > 0 && tile_shape_off::ValidRow > 0) { + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { asm volatile( - "BSTART.TEPL 112, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 82, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_off::ValidCol), - "i"(tile_shape_off::ValidRow), - "i"(tile_shape_off::Cols), + [dst____dimcol] "r"(dst.GetValidCol()), + "i"(tile_shape_out::ValidRow), + "i"(tile_shape_out::Cols), "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "Tr"(off.data()) - ); - } else { + "i"(tile_type_traits::TilesizeCode) + ); } else { asm volatile( - "BSTART.TEPL 112, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 82, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" - "B.IOR [%7],[]\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(off.GetValidCol()), - "ri"(off.GetValidRow()), - "i"(tile_shape_off::Cols), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), "Tr"(src.data()), - "i"(tile_type_traits::TilesizeCode), - "Tr"(off.data()) - ); - } + "i"(tile_type_traits::TilesizeCode) + ); } } -// TPARTADD: partial-valid add (moved from Mode 0) -template -void TPARTADD(tile_shape &dst, tile_shape &src0, tile_shape &src1) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { +// TCOLPROD: col product reduction +template +void TCOLPROD(tile_shape_out &dst, tile_shape_in &src) { + // ASL (reduction-and-expansion): column-axis reduction destination has + // exactly one valid row and inherits the source column geometry. + static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_out::ValidRow == 1, + "TCOLPROD destination must have exactly one valid row (1 x N)"); + static_assert(tile_shape_out::ValidCol == DYNAMIC || + (tile_shape_out::ValidCol == tile_shape_in::ValidCol && + tile_shape_out::Cols == tile_shape_in::Cols), + "TCOLPROD destination columns must match the source columns"); + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { asm volatile( - "BSTART.TEPL 113, %D1\n" + "BSTART.TEPL 83, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(tile_shape::ValidCol), - "i"(tile_shape::ValidRow), - "i"(tile_shape::Cols), - "Tr"(src0.data()), - "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode) - ); - } else { + : "i"(type_traits::TypeCode), + "i"(tile_shape_out::ValidCol), + "i"(tile_shape_out::ValidRow), + "i"(tile_shape_out::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { asm volatile( - "BSTART.TEPL 113, %D1\n" + "BSTART.TEPL 83, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "ri"(src0.GetValidCol()), - "ri"(src0.GetValidRow()), - "i"(tile_shape::Cols), - "Tr"(src0.data()), - "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TPARTMUL: partial-valid multiply (moved from Mode 0) -template -void TPARTMUL(tile_shape &dst, tile_shape &src0, tile_shape &src1) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + : "i"(type_traits::TypeCode), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { asm volatile( - "BSTART.TEPL 114, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 83, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(tile_shape::ValidCol), - "i"(tile_shape::ValidRow), - "i"(tile_shape::Cols), - "Tr"(src0.data()), - "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode) - ); - } else { + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + "i"(tile_shape_out::ValidRow), + "i"(tile_shape_out::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { asm volatile( - "BSTART.TEPL 114, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 83, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "ri"(src0.GetValidCol()), - "ri"(src0.GetValidRow()), - "i"(tile_shape::Cols), - "Tr"(src0.data()), - "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode) - ); - } + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } -// TPARTMAX: partial-valid max (moved from Mode 0) -template -void TPARTMAX(tile_shape &dst, tile_shape &src0, tile_shape &src1) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { +// TCOLEXPAND: broadcast first element of each col +template +void TCOLEXPAND(tile_shape_out &dst, tile_shape_in &src) { + // ASL (expansion): column expansion broadcasts a one-row source; the + // destination geometry comes from the destination B.DIM, not the source. + static_assert(tile_shape_in::ValidRow == DYNAMIC || tile_shape_in::ValidRow == 1, + "TCOLEXPAND source must be a one-row broadcast tile"); + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { asm volatile( - "BSTART.TEPL 115, %D1\n" + "BSTART.TEPL 84, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(tile_shape::ValidCol), - "i"(tile_shape::ValidRow), - "i"(tile_shape::Cols), - "Tr"(src0.data()), - "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode) - ); - } else { + : "i"(type_traits::TypeCode), + "i"(tile_shape_out::ValidCol), + "i"(tile_shape_out::ValidRow), + "i"(tile_shape_out::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { asm volatile( - "BSTART.TEPL 115, %D1\n" + "BSTART.TEPL 84, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "ri"(src0.GetValidCol()), - "ri"(src0.GetValidRow()), - "i"(tile_shape::Cols), - "Tr"(src0.data()), - "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TPARTMIN: partial-valid min (moved from Mode 0) -template -void TPARTMIN(tile_shape &dst, tile_shape &src0, tile_shape &src1) { - if constexpr (tile_shape::ValidCol > 0 && tile_shape::ValidRow > 0) { + : "i"(type_traits::TypeCode), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { asm volatile( - "BSTART.TEPL 116, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 84, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(tile_shape::ValidCol), - "i"(tile_shape::ValidRow), - "i"(tile_shape::Cols), - "Tr"(src0.data()), - "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode) - ); - } else { + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + "i"(tile_shape_out::ValidRow), + "i"(tile_shape_out::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else { asm volatile( - "BSTART.TEPL 116, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 84, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "ri"(src0.GetValidCol()), - "ri"(src0.GetValidRow()), - "i"(tile_shape::Cols), - "Tr"(src0.data()), - "Tr"(src1.data()), - "i"(tile_type_traits::TilesizeCode) - ); - } + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } -//===--- TEPL Mode 2: reduction/broadcast ops (BSTART.TEPL) ---===// -// opcode = Mode(2) * 32 + Function = 64 + Function. -// TROWSUM: row sum reduction +// TCOLARGMAX: col argmax (DavinciOO ext) template -void TROWSUM(tile_shape_out &dst, tile_shape_in &src) { - // ASL (row reduction): B.DIM describes the SOURCE geometry - // (ValidCol/ValidRow/Col); the destination is rule-derived: one - // column, ValidRow = source.ValidRow. - static_assert(tile_shape_out::ValidCol == DYNAMIC || (tile_shape_out::ValidCol == 1 && tile_shape_out::Cols == 1), - "TROWSUM destination must be a single-column tile (N x 1)"); - static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_in::ValidRow == DYNAMIC || tile_shape_out::ValidRow == tile_shape_in::ValidRow, - "TROWSUM destination valid rows must equal the source valid rows"); - if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +void TCOLARGMAX(tile_shape_out &dst, tile_shape_in &src) { + // ASL (reduction-and-expansion): column-axis reduction destination has + // exactly one valid row and inherits the source column geometry. + static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_out::ValidRow == 1, + "TCOLARGMAX destination must have exactly one valid row (1 x N)"); + static_assert(tile_shape_out::ValidCol == DYNAMIC || + (tile_shape_out::ValidCol == tile_shape_in::ValidCol && + tile_shape_out::Cols == tile_shape_in::Cols), + "TCOLARGMAX destination columns must match the source columns"); + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { asm volatile( - "BSTART.TEPL 64, %D1\n" + "BSTART.TEPL 92, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" @@ -9849,91 +14310,73 @@ void TROWSUM(tile_shape_out &dst, tile_shape_in &src) { "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_in::ValidCol), - "i"(tile_shape_in::ValidRow), - "i"(tile_shape_in::Cols), + "i"(tile_shape_out::ValidCol), + "i"(tile_shape_out::ValidRow), + "i"(tile_shape_out::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { - const size_t valid_col = src.GetValidCol(); - const size_t valid_row = src.GetValidRow(); + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { asm volatile( - "BSTART.TEPL 64, %D1\n" + "BSTART.TEPL 92, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(valid_col), - "ri"(valid_row), - "i"(tile_shape_in::Cols), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TROWMAX: row max reduction -template -void TROWMAX(tile_shape_out &dst, tile_shape_in &src) { - // ASL (row reduction): B.DIM describes the SOURCE geometry - // (ValidCol/ValidRow/Col); the destination is rule-derived: one - // column, ValidRow = source.ValidRow. - static_assert(tile_shape_out::ValidCol == DYNAMIC || (tile_shape_out::ValidCol == 1 && tile_shape_out::Cols == 1), - "TROWMAX destination must be a single-column tile (N x 1)"); - static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_in::ValidRow == DYNAMIC || tile_shape_out::ValidRow == tile_shape_in::ValidRow, - "TROWMAX destination valid rows must equal the source valid rows"); - if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { asm volatile( - "BSTART.TEPL 65, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 92, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_in::ValidCol), - "i"(tile_shape_in::ValidRow), - "i"(tile_shape_in::Cols), + [dst____dimcol] "r"(dst.GetValidCol()), + "i"(tile_shape_out::ValidRow), + "i"(tile_shape_out::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else { asm volatile( - "BSTART.TEPL 65, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 92, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), - "i"(tile_shape_in::Cols), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } } -// TROWMIN: row min reduction +// TCOLARGMIN: col argmin (DavinciOO ext) template -void TROWMIN(tile_shape_out &dst, tile_shape_in &src) { - // ASL (row reduction): B.DIM describes the SOURCE geometry - // (ValidCol/ValidRow/Col); the destination is rule-derived: one - // column, ValidRow = source.ValidRow. - static_assert(tile_shape_out::ValidCol == DYNAMIC || (tile_shape_out::ValidCol == 1 && tile_shape_out::Cols == 1), - "TROWMIN destination must be a single-column tile (N x 1)"); - static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_in::ValidRow == DYNAMIC || tile_shape_out::ValidRow == tile_shape_in::ValidRow, - "TROWMIN destination valid rows must equal the source valid rows"); - if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +void TCOLARGMIN(tile_shape_out &dst, tile_shape_in &src) { + // ASL (reduction-and-expansion): column-axis reduction destination has + // exactly one valid row and inherits the source column geometry. + static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_out::ValidRow == 1, + "TCOLARGMIN destination must have exactly one valid row (1 x N)"); + static_assert(tile_shape_out::ValidCol == DYNAMIC || + (tile_shape_out::ValidCol == tile_shape_in::ValidCol && + tile_shape_out::Cols == tile_shape_in::Cols), + "TCOLARGMIN destination columns must match the source columns"); + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { asm volatile( - "BSTART.TEPL 66, %D1\n" + "BSTART.TEPL 93, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" @@ -9941,545 +14384,688 @@ void TROWMIN(tile_shape_out &dst, tile_shape_in &src) { "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_in::ValidCol), - "i"(tile_shape_in::ValidRow), - "i"(tile_shape_in::Cols), + "i"(tile_shape_out::ValidCol), + "i"(tile_shape_out::ValidRow), + "i"(tile_shape_out::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { asm volatile( - "BSTART.TEPL 66, %D1\n" + "BSTART.TEPL 93, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), - "i"(tile_shape_in::Cols), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TROWPROD: row product reduction -template -void TROWPROD(tile_shape_out &dst, tile_shape_in &src) { - // ASL (row reduction): B.DIM describes the SOURCE geometry - // (ValidCol/ValidRow/Col); the destination is rule-derived: one - // column, ValidRow = source.ValidRow. - static_assert(tile_shape_out::ValidCol == DYNAMIC || (tile_shape_out::ValidCol == 1 && tile_shape_out::Cols == 1), - "TROWPROD destination must be a single-column tile (N x 1)"); - static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_in::ValidRow == DYNAMIC || tile_shape_out::ValidRow == tile_shape_in::ValidRow, - "TROWPROD destination valid rows must equal the source valid rows"); - if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { asm volatile( - "BSTART.TEPL 67, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 93, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_in::ValidCol), - "i"(tile_shape_in::ValidRow), - "i"(tile_shape_in::Cols), + [dst____dimcol] "r"(dst.GetValidCol()), + "i"(tile_shape_out::ValidRow), + "i"(tile_shape_out::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else { asm volatile( - "BSTART.TEPL 67, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 93, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), - "i"(tile_shape_in::Cols), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), "Tr"(src.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } } -// TROWEXPAND: broadcast first element of each row -template -void TROWEXPAND(tile_shape_out &dst, tile_shape_in &src) { +// TROWEXPANDADD: row broadcast add +// src1 is a per-row scalar/byte-strip operand whose shape may differ from src0 +// (see pto/TROWEXPANDMUL.md Mode 1/2); only dtype is required to match. +template +void TROWEXPANDADD(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { // ASL (expansion): row expansion broadcasts a one-column source; the // destination geometry comes from the destination B.DIM, not the source. - static_assert(tile_shape_in::ValidCol == DYNAMIC || tile_shape_in::ValidCol == 1, - "TROWEXPAND source must be a one-column broadcast tile"); - if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { + static_assert(tile_shape_in1::ValidCol == DYNAMIC || tile_shape_in1::ValidCol == 1, + "TROWEXPANDADD broadcast source must be a one-column tile"); + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { + static_assert(std::is_same::value, + "TROWEXPANDADD: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDADD: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 68, %D1\n" + "BSTART.TEPL 69, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), + : "i"(type_traits::TypeCode), "i"(tile_shape_out::ValidCol), "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { + static_assert(std::is_same::value, + "TROWEXPANDADD: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDADD: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 68, %D1\n" + "BSTART.TEPL 69, %D1\n" "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { + static_assert(std::is_same::value, + "TROWEXPANDADD: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDADD: src0/dst dtype must match"); + asm volatile( + "BSTART.TEPL 69, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } else { + static_assert(std::is_same::value, + "TROWEXPANDADD: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDADD: src0/dst dtype must match"); + asm volatile( + "BSTART.TEPL 69, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } -// TROWARGMAX: row argmax (DavinciOO ext) -template -void TROWARGMAX(tile_shape_out &dst, tile_shape_in &src) { - // ASL (row reduction): B.DIM describes the SOURCE geometry - // (ValidCol/ValidRow/Col); the destination is rule-derived: one - // column, ValidRow = source.ValidRow. - static_assert(tile_shape_out::ValidCol == DYNAMIC || (tile_shape_out::ValidCol == 1 && tile_shape_out::Cols == 1), - "TROWARGMAX destination must be a single-column tile (N x 1)"); - static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_in::ValidRow == DYNAMIC || tile_shape_out::ValidRow == tile_shape_in::ValidRow, - "TROWARGMAX destination valid rows must equal the source valid rows"); - if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +// TROWEXPANDSUB: row broadcast sub +// src1 is a per-row scalar/byte-strip operand whose shape may differ from src0 +// (see pto/TROWEXPANDMUL.md Mode 1/2); only dtype is required to match. +template +void TROWEXPANDSUB(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { + // ASL (expansion): row expansion broadcasts a one-column source; the + // destination geometry comes from the destination B.DIM, not the source. + static_assert(tile_shape_in1::ValidCol == DYNAMIC || tile_shape_in1::ValidCol == 1, + "TROWEXPANDSUB broadcast source must be a one-column tile"); + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { + static_assert(std::is_same::value, + "TROWEXPANDSUB: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDSUB: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 76, %D1\n" + "BSTART.TEPL 70, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(tile_shape_in::ValidCol), - "i"(tile_shape_in::ValidRow), - "i"(tile_shape_in::Cols), - "Tr"(src.data()), + : "i"(type_traits::TypeCode), + "i"(tile_shape_out::ValidCol), + "i"(tile_shape_out::ValidRow), + "i"(tile_shape_out::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { + static_assert(std::is_same::value, + "TROWEXPANDSUB: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDSUB: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 76, %D1\n" + "BSTART.TEPL 70, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), - "i"(tile_shape_in::Cols), - "Tr"(src.data()), + : "i"(type_traits::TypeCode), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TROWARGMIN: row argmin (DavinciOO ext) -template -void TROWARGMIN(tile_shape_out &dst, tile_shape_in &src) { - // ASL (row reduction): B.DIM describes the SOURCE geometry - // (ValidCol/ValidRow/Col); the destination is rule-derived: one - // column, ValidRow = source.ValidRow. - static_assert(tile_shape_out::ValidCol == DYNAMIC || (tile_shape_out::ValidCol == 1 && tile_shape_out::Cols == 1), - "TROWARGMIN destination must be a single-column tile (N x 1)"); - static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_in::ValidRow == DYNAMIC || tile_shape_out::ValidRow == tile_shape_in::ValidRow, - "TROWARGMIN destination valid rows must equal the source valid rows"); - if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { + static_assert(std::is_same::value, + "TROWEXPANDSUB: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDSUB: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 77, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 70, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(tile_shape_in::ValidCol), - "i"(tile_shape_in::ValidRow), - "i"(tile_shape_in::Cols), - "Tr"(src.data()), + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + "i"(tile_shape_out::ValidRow), + "i"(tile_shape_out::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else { + static_assert(std::is_same::value, + "TROWEXPANDSUB: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDSUB: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 77, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 70, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(src.GetValidCol()), - "i"(src.GetValidRow()), - "i"(tile_shape_in::Cols), - "Tr"(src.data()), + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } } -// TCOLSUM: col sum reduction -template -void TCOLSUM(tile_shape_out &dst, tile_shape_in &src) { - // ASL (reduction-and-expansion): column-axis reduction destination has - // exactly one valid row and inherits the source column geometry. - static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_out::ValidRow == 1, - "TCOLSUM destination must have exactly one valid row (1 x N)"); - static_assert(tile_shape_out::ValidCol == DYNAMIC || - (tile_shape_out::ValidCol == tile_shape_in::ValidCol && - tile_shape_out::Cols == tile_shape_in::Cols), - "TCOLSUM destination columns must match the source columns"); - if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +// TROWEXPANDMUL: row broadcast mul +// src1 is a per-row scalar/byte-strip operand whose shape may differ from src0 +// (see pto/TROWEXPANDMUL.md Mode 1/2); only dtype is required to match. +template +void TROWEXPANDMUL(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { + // ASL (expansion): row expansion broadcasts a one-column source; the + // destination geometry comes from the destination B.DIM, not the source. + static_assert(tile_shape_in1::ValidCol == DYNAMIC || tile_shape_in1::ValidCol == 1, + "TROWEXPANDMUL broadcast source must be a one-column tile"); + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { + static_assert(std::is_same::value, + "TROWEXPANDMUL: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDMUL: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 80, %D1\n" + "BSTART.TEPL 71, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), + : "i"(type_traits::TypeCode), "i"(tile_shape_out::ValidCol), "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { + static_assert(std::is_same::value, + "TROWEXPANDMUL: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDMUL: src0/dst dtype must match"); + const size_t valid_col = src0.GetValidCol(); asm volatile( - "BSTART.TEPL 80, %D1\n" + "BSTART.TEPL 71, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[valid_col], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + : "i"(type_traits::TypeCode), + "i"(tile_shape_out::ValidCol), + [valid_col] "r"(src0.GetValidCol()), "i"(tile_shape_out::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TCOLMAX: col max reduction -template -void TCOLMAX(tile_shape_out &dst, tile_shape_in &src) { - // ASL (reduction-and-expansion): column-axis reduction destination has - // exactly one valid row and inherits the source column geometry. - static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_out::ValidRow == 1, - "TCOLMAX destination must have exactly one valid row (1 x N)"); - static_assert(tile_shape_out::ValidCol == DYNAMIC || - (tile_shape_out::ValidCol == tile_shape_in::ValidCol && - tile_shape_out::Cols == tile_shape_in::Cols), - "TCOLMAX destination columns must match the source columns"); - if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { + static_assert(std::is_same::value, + "TROWEXPANDMUL: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDMUL: src0/dst dtype must match"); + const size_t valid_col = src0.GetValidCol(); asm volatile( - "BSTART.TEPL 81, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 71, %D1\n" + "B.DIM %[valid_col], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), + : "i"(type_traits::TypeCode), + [valid_col] "r"(src0.GetValidCol()), "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else { + static_assert(std::is_same::value, + "TROWEXPANDMUL: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDMUL: src0/dst dtype must match"); + const size_t valid_col = src0.GetValidCol(); + const size_t valid_row = src0.GetValidRow(); asm volatile( - "BSTART.TEPL 81, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 71, %D1\n" + "B.DIM %[valid_col], 0, ->lb0\n" + "B.DIM %[valid_row], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + : "i"(type_traits::TypeCode), + [valid_col] "r"(valid_col), + [valid_row] "r"(valid_row), "i"(tile_shape_out::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } } -// TCOLMIN: col min reduction -template -void TCOLMIN(tile_shape_out &dst, tile_shape_in &src) { - // ASL (reduction-and-expansion): column-axis reduction destination has - // exactly one valid row and inherits the source column geometry. - static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_out::ValidRow == 1, - "TCOLMIN destination must have exactly one valid row (1 x N)"); - static_assert(tile_shape_out::ValidCol == DYNAMIC || - (tile_shape_out::ValidCol == tile_shape_in::ValidCol && - tile_shape_out::Cols == tile_shape_in::Cols), - "TCOLMIN destination columns must match the source columns"); - if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +// TROWEXPANDDIV: row broadcast div +// src1 is a per-row scalar/byte-strip operand whose shape may differ from src0 +// (see pto/TROWEXPANDMUL.md Mode 1/2); only dtype is required to match. +template +void TROWEXPANDDIV(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { + // ASL (expansion): row expansion broadcasts a one-column source; the + // destination geometry comes from the destination B.DIM, not the source. + static_assert(tile_shape_in1::ValidCol == DYNAMIC || tile_shape_in1::ValidCol == 1, + "TROWEXPANDDIV broadcast source must be a one-column tile"); + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { + static_assert(std::is_same::value, + "TROWEXPANDDIV: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDDIV: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 82, %D1\n" + "BSTART.TEPL 72, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), + : "i"(type_traits::TypeCode), "i"(tile_shape_out::ValidCol), "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { + static_assert(std::is_same::value, + "TROWEXPANDDIV: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDDIV: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 82, %D1\n" + "BSTART.TEPL 72, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + : "i"(type_traits::TypeCode), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), "i"(tile_shape_out::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TCOLPROD: col product reduction -template -void TCOLPROD(tile_shape_out &dst, tile_shape_in &src) { - // ASL (reduction-and-expansion): column-axis reduction destination has - // exactly one valid row and inherits the source column geometry. - static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_out::ValidRow == 1, - "TCOLPROD destination must have exactly one valid row (1 x N)"); - static_assert(tile_shape_out::ValidCol == DYNAMIC || - (tile_shape_out::ValidCol == tile_shape_in::ValidCol && - tile_shape_out::Cols == tile_shape_in::Cols), - "TCOLPROD destination columns must match the source columns"); - if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { + static_assert(std::is_same::value, + "TROWEXPANDDIV: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDDIV: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 83, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 72, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else { + static_assert(std::is_same::value, + "TROWEXPANDDIV: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDDIV: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 83, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 72, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), "i"(tile_shape_out::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } } -// TCOLEXPAND: broadcast first element of each col -template -void TCOLEXPAND(tile_shape_out &dst, tile_shape_in &src) { - // ASL (expansion): column expansion broadcasts a one-row source; the +// TROWEXPANDMAX: row broadcast max +// src1 is a per-row scalar/byte-strip operand whose shape may differ from src0 +// (see pto/TROWEXPANDMUL.md Mode 1/2); only dtype is required to match. +template +void TROWEXPANDMAX(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { + // ASL (expansion): row expansion broadcasts a one-column source; the // destination geometry comes from the destination B.DIM, not the source. - static_assert(tile_shape_in::ValidRow == DYNAMIC || tile_shape_in::ValidRow == 1, - "TCOLEXPAND source must be a one-row broadcast tile"); - if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { + static_assert(tile_shape_in1::ValidCol == DYNAMIC || tile_shape_in1::ValidCol == 1, + "TROWEXPANDMAX broadcast source must be a one-column tile"); + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { + static_assert(std::is_same::value, + "TROWEXPANDMAX: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDMAX: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 84, %D1\n" + "BSTART.TEPL 73, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), + : "i"(type_traits::TypeCode), "i"(tile_shape_out::ValidCol), "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { + static_assert(std::is_same::value, + "TROWEXPANDMAX: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDMAX: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 84, %D1\n" + "BSTART.TEPL 73, %D1\n" "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { + static_assert(std::is_same::value, + "TROWEXPANDMAX: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDMAX: src0/dst dtype must match"); + asm volatile( + "BSTART.TEPL 73, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } else { + static_assert(std::is_same::value, + "TROWEXPANDMAX: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDMAX: src0/dst dtype must match"); + asm volatile( + "BSTART.TEPL 73, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } -// TCOLARGMAX: col argmax (DavinciOO ext) -template -void TCOLARGMAX(tile_shape_out &dst, tile_shape_in &src) { - // ASL (reduction-and-expansion): column-axis reduction destination has - // exactly one valid row and inherits the source column geometry. - static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_out::ValidRow == 1, - "TCOLARGMAX destination must have exactly one valid row (1 x N)"); - static_assert(tile_shape_out::ValidCol == DYNAMIC || - (tile_shape_out::ValidCol == tile_shape_in::ValidCol && - tile_shape_out::Cols == tile_shape_in::Cols), - "TCOLARGMAX destination columns must match the source columns"); - if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { +// TROWEXPANDMIN: row broadcast min +// src1 is a per-row scalar/byte-strip operand whose shape may differ from src0 +// (see pto/TROWEXPANDMUL.md Mode 1/2); only dtype is required to match. +template +void TROWEXPANDMIN(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { + // ASL (expansion): row expansion broadcasts a one-column source; the + // destination geometry comes from the destination B.DIM, not the source. + static_assert(tile_shape_in1::ValidCol == DYNAMIC || tile_shape_in1::ValidCol == 1, + "TROWEXPANDMIN broadcast source must be a one-column tile"); + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { + static_assert(std::is_same::value, + "TROWEXPANDMIN: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDMIN: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 92, %D1\n" + "BSTART.TEPL 74, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), + : "i"(type_traits::TypeCode), "i"(tile_shape_out::ValidCol), "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { + static_assert(std::is_same::value, + "TROWEXPANDMIN: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDMIN: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 92, %D1\n" + "BSTART.TEPL 74, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + : "i"(type_traits::TypeCode), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), "i"(tile_shape_out::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TCOLARGMIN: col argmin (DavinciOO ext) -template -void TCOLARGMIN(tile_shape_out &dst, tile_shape_in &src) { - // ASL (reduction-and-expansion): column-axis reduction destination has - // exactly one valid row and inherits the source column geometry. - static_assert(tile_shape_out::ValidRow == DYNAMIC || tile_shape_out::ValidRow == 1, - "TCOLARGMIN destination must have exactly one valid row (1 x N)"); - static_assert(tile_shape_out::ValidCol == DYNAMIC || - (tile_shape_out::ValidCol == tile_shape_in::ValidCol && - tile_shape_out::Cols == tile_shape_in::Cols), - "TCOLARGMIN destination columns must match the source columns"); - if constexpr (tile_shape_in::ValidCol > 0 && tile_shape_in::ValidRow > 0) { + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { + static_assert(std::is_same::value, + "TROWEXPANDMIN: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDMIN: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 93, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 74, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else { + static_assert(std::is_same::value, + "TROWEXPANDMIN: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TROWEXPANDMIN: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 93, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 74, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" - "B.IOT %5, mask=1111, last, ->%0<%Z6>\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) - : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), "i"(tile_shape_out::Cols), - "Tr"(src.data()), + "Tr"(src0.data()), + "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } } -// TROWEXPANDADD: row broadcast add +// TROWEXPANDEXPDIF: row exp(src0-src1) // src1 is a per-row scalar/byte-strip operand whose shape may differ from src0 // (see pto/TROWEXPANDMUL.md Mode 1/2); only dtype is required to match. template -void TROWEXPANDADD(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { +void TROWEXPANDEXPDIF(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { // ASL (expansion): row expansion broadcasts a one-column source; the // destination geometry comes from the destination B.DIM, not the source. static_assert(tile_shape_in1::ValidCol == DYNAMIC || tile_shape_in1::ValidCol == 1, - "TROWEXPANDADD broadcast source must be a one-column tile"); - if constexpr (tile_shape_in0::ValidCol > 0 && tile_shape_in0::ValidRow > 0) { + "TROWEXPANDEXPDIF broadcast source must be a one-column tile"); + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { static_assert(std::is_same::value, - "TROWEXPANDADD: src0/src1 dtype must match"); + "TROWEXPANDEXPDIF: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TROWEXPANDADD: src0/dst dtype must match"); + "TROWEXPANDEXPDIF: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 69, %D1\n" + "BSTART.TEPL 75, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" @@ -10493,111 +15079,94 @@ void TROWEXPANDADD(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { static_assert(std::is_same::value, - "TROWEXPANDADD: src0/src1 dtype must match"); + "TROWEXPANDEXPDIF: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TROWEXPANDADD: src0/dst dtype must match"); + "TROWEXPANDEXPDIF: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 69, %D1\n" + "BSTART.TEPL 75, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TROWEXPANDSUB: row broadcast sub -// src1 is a per-row scalar/byte-strip operand whose shape may differ from src0 -// (see pto/TROWEXPANDMUL.md Mode 1/2); only dtype is required to match. -template -void TROWEXPANDSUB(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { - // ASL (expansion): row expansion broadcasts a one-column source; the - // destination geometry comes from the destination B.DIM, not the source. - static_assert(tile_shape_in1::ValidCol == DYNAMIC || tile_shape_in1::ValidCol == 1, - "TROWEXPANDSUB broadcast source must be a one-column tile"); - if constexpr (tile_shape_in0::ValidCol > 0 && tile_shape_in0::ValidRow > 0) { + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { static_assert(std::is_same::value, - "TROWEXPANDSUB: src0/src1 dtype must match"); + "TROWEXPANDEXPDIF: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TROWEXPANDSUB: src0/dst dtype must match"); + "TROWEXPANDEXPDIF: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 70, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 75, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), + [dst____dimcol] "r"(dst.GetValidCol()), "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else { static_assert(std::is_same::value, - "TROWEXPANDSUB: src0/src1 dtype must match"); + "TROWEXPANDEXPDIF: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TROWEXPANDSUB: src0/dst dtype must match"); + "TROWEXPANDEXPDIF: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 70, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 75, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } } -// TROWEXPANDMUL: row broadcast mul -// src1 is a per-row scalar/byte-strip operand whose shape may differ from src0 -// (see pto/TROWEXPANDMUL.md Mode 1/2); only dtype is required to match. +// TCOLEXPANDADD: col broadcast add +// src1 is a per-column scalar/byte-strip operand whose shape may differ from src0 +// (col-broadcast analog of pto/TROWEXPANDMUL.md Mode 1/2); only dtype must match. template -void TROWEXPANDMUL(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { - // ASL (expansion): row expansion broadcasts a one-column source; the +void TCOLEXPANDADD(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { + // ASL (expansion): column expansion broadcasts a one-row source; the // destination geometry comes from the destination B.DIM, not the source. - static_assert(tile_shape_in1::ValidCol == DYNAMIC || tile_shape_in1::ValidCol == 1, - "TROWEXPANDMUL broadcast source must be a one-column tile"); - if constexpr (tile_shape_in0::ValidCol > 0 && tile_shape_in0::ValidRow > 0) { + static_assert(tile_shape_in1::ValidRow == DYNAMIC || tile_shape_in1::ValidRow == 1, + "TCOLEXPANDADD broadcast source must be a one-row tile"); + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { static_assert(std::is_same::value, - "TROWEXPANDMUL: src0/src1 dtype must match"); + "TCOLEXPANDADD: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TROWEXPANDMUL: src0/dst dtype must match"); + "TCOLEXPANDADD: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 71, %D1\n" + "BSTART.TEPL 85, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" @@ -10611,113 +15180,94 @@ void TROWEXPANDMUL(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { static_assert(std::is_same::value, - "TROWEXPANDMUL: src0/src1 dtype must match"); + "TCOLEXPANDADD: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TROWEXPANDMUL: src0/dst dtype must match"); - const size_t valid_col = src0.GetValidCol(); - const size_t valid_row = src0.GetValidRow(); + "TCOLEXPANDADD: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 71, %D1\n" + "BSTART.TEPL 85, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(valid_col), - "ri"(valid_row), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TROWEXPANDDIV: row broadcast div -// src1 is a per-row scalar/byte-strip operand whose shape may differ from src0 -// (see pto/TROWEXPANDMUL.md Mode 1/2); only dtype is required to match. -template -void TROWEXPANDDIV(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { - // ASL (expansion): row expansion broadcasts a one-column source; the - // destination geometry comes from the destination B.DIM, not the source. - static_assert(tile_shape_in1::ValidCol == DYNAMIC || tile_shape_in1::ValidCol == 1, - "TROWEXPANDDIV broadcast source must be a one-column tile"); - if constexpr (tile_shape_in0::ValidCol > 0 && tile_shape_in0::ValidRow > 0) { + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { static_assert(std::is_same::value, - "TROWEXPANDDIV: src0/src1 dtype must match"); + "TCOLEXPANDADD: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TROWEXPANDDIV: src0/dst dtype must match"); + "TCOLEXPANDADD: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 72, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 85, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), + [dst____dimcol] "r"(dst.GetValidCol()), "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else { static_assert(std::is_same::value, - "TROWEXPANDDIV: src0/src1 dtype must match"); + "TCOLEXPANDADD: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TROWEXPANDDIV: src0/dst dtype must match"); + "TCOLEXPANDADD: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 72, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 85, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } } -// TROWEXPANDMAX: row broadcast max -// src1 is a per-row scalar/byte-strip operand whose shape may differ from src0 -// (see pto/TROWEXPANDMUL.md Mode 1/2); only dtype is required to match. +// TCOLEXPANDSUB: col broadcast sub +// src1 is a per-column scalar/byte-strip operand whose shape may differ from src0 +// (col-broadcast analog of pto/TROWEXPANDMUL.md Mode 1/2); only dtype must match. template -void TROWEXPANDMAX(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { - // ASL (expansion): row expansion broadcasts a one-column source; the +void TCOLEXPANDSUB(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { + // ASL (expansion): column expansion broadcasts a one-row source; the // destination geometry comes from the destination B.DIM, not the source. - static_assert(tile_shape_in1::ValidCol == DYNAMIC || tile_shape_in1::ValidCol == 1, - "TROWEXPANDMAX broadcast source must be a one-column tile"); - if constexpr (tile_shape_in0::ValidCol > 0 && tile_shape_in0::ValidRow > 0) { + static_assert(tile_shape_in1::ValidRow == DYNAMIC || tile_shape_in1::ValidRow == 1, + "TCOLEXPANDSUB broadcast source must be a one-row tile"); + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { static_assert(std::is_same::value, - "TROWEXPANDMAX: src0/src1 dtype must match"); + "TCOLEXPANDSUB: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TROWEXPANDMAX: src0/dst dtype must match"); + "TCOLEXPANDSUB: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 73, %D1\n" + "BSTART.TEPL 86, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" @@ -10731,111 +15281,94 @@ void TROWEXPANDMAX(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { static_assert(std::is_same::value, - "TROWEXPANDMAX: src0/src1 dtype must match"); + "TCOLEXPANDSUB: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TROWEXPANDMAX: src0/dst dtype must match"); + "TCOLEXPANDSUB: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 73, %D1\n" + "BSTART.TEPL 86, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TROWEXPANDMIN: row broadcast min -// src1 is a per-row scalar/byte-strip operand whose shape may differ from src0 -// (see pto/TROWEXPANDMUL.md Mode 1/2); only dtype is required to match. -template -void TROWEXPANDMIN(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { - // ASL (expansion): row expansion broadcasts a one-column source; the - // destination geometry comes from the destination B.DIM, not the source. - static_assert(tile_shape_in1::ValidCol == DYNAMIC || tile_shape_in1::ValidCol == 1, - "TROWEXPANDMIN broadcast source must be a one-column tile"); - if constexpr (tile_shape_in0::ValidCol > 0 && tile_shape_in0::ValidRow > 0) { + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { static_assert(std::is_same::value, - "TROWEXPANDMIN: src0/src1 dtype must match"); + "TCOLEXPANDSUB: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TROWEXPANDMIN: src0/dst dtype must match"); + "TCOLEXPANDSUB: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 74, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 86, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), + [dst____dimcol] "r"(dst.GetValidCol()), "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else { static_assert(std::is_same::value, - "TROWEXPANDMIN: src0/src1 dtype must match"); + "TCOLEXPANDSUB: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TROWEXPANDMIN: src0/dst dtype must match"); + "TCOLEXPANDSUB: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 74, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 86, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } } -// TROWEXPANDEXPDIF: row exp(src0-src1) -// src1 is a per-row scalar/byte-strip operand whose shape may differ from src0 -// (see pto/TROWEXPANDMUL.md Mode 1/2); only dtype is required to match. +// TCOLEXPANDMUL: col broadcast mul +// src1 is a per-column scalar/byte-strip operand whose shape may differ from src0 +// (col-broadcast analog of pto/TROWEXPANDMUL.md Mode 1/2); only dtype must match. template -void TROWEXPANDEXPDIF(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { - // ASL (expansion): row expansion broadcasts a one-column source; the +void TCOLEXPANDMUL(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { + // ASL (expansion): column expansion broadcasts a one-row source; the // destination geometry comes from the destination B.DIM, not the source. - static_assert(tile_shape_in1::ValidCol == DYNAMIC || tile_shape_in1::ValidCol == 1, - "TROWEXPANDEXPDIF broadcast source must be a one-column tile"); - if constexpr (tile_shape_in0::ValidCol > 0 && tile_shape_in0::ValidRow > 0) { + static_assert(tile_shape_in1::ValidRow == DYNAMIC || tile_shape_in1::ValidRow == 1, + "TCOLEXPANDMUL broadcast source must be a one-row tile"); + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { static_assert(std::is_same::value, - "TROWEXPANDEXPDIF: src0/src1 dtype must match"); + "TCOLEXPANDMUL: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TROWEXPANDEXPDIF: src0/dst dtype must match"); + "TCOLEXPANDMUL: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 75, %D1\n" + "BSTART.TEPL 87, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" @@ -10849,111 +15382,94 @@ void TROWEXPANDEXPDIF(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { static_assert(std::is_same::value, - "TROWEXPANDEXPDIF: src0/src1 dtype must match"); + "TCOLEXPANDMUL: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TROWEXPANDEXPDIF: src0/dst dtype must match"); + "TCOLEXPANDMUL: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 75, %D1\n" + "BSTART.TEPL 87, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TCOLEXPANDADD: col broadcast add -// src1 is a per-column scalar/byte-strip operand whose shape may differ from src0 -// (col-broadcast analog of pto/TROWEXPANDMUL.md Mode 1/2); only dtype must match. -template -void TCOLEXPANDADD(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { - // ASL (expansion): column expansion broadcasts a one-row source; the - // destination geometry comes from the destination B.DIM, not the source. - static_assert(tile_shape_in1::ValidRow == DYNAMIC || tile_shape_in1::ValidRow == 1, - "TCOLEXPANDADD broadcast source must be a one-row tile"); - if constexpr (tile_shape_in0::ValidCol > 0 && tile_shape_in0::ValidRow > 0) { + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { static_assert(std::is_same::value, - "TCOLEXPANDADD: src0/src1 dtype must match"); + "TCOLEXPANDMUL: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TCOLEXPANDADD: src0/dst dtype must match"); + "TCOLEXPANDMUL: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 85, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 87, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), + [dst____dimcol] "r"(dst.GetValidCol()), "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else { static_assert(std::is_same::value, - "TCOLEXPANDADD: src0/src1 dtype must match"); + "TCOLEXPANDMUL: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TCOLEXPANDADD: src0/dst dtype must match"); + "TCOLEXPANDMUL: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 85, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 87, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } } -// TCOLEXPANDSUB: col broadcast sub +// TCOLEXPANDDIV: col broadcast div // src1 is a per-column scalar/byte-strip operand whose shape may differ from src0 // (col-broadcast analog of pto/TROWEXPANDMUL.md Mode 1/2); only dtype must match. template -void TCOLEXPANDSUB(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { +void TCOLEXPANDDIV(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { // ASL (expansion): column expansion broadcasts a one-row source; the // destination geometry comes from the destination B.DIM, not the source. static_assert(tile_shape_in1::ValidRow == DYNAMIC || tile_shape_in1::ValidRow == 1, - "TCOLEXPANDSUB broadcast source must be a one-row tile"); - if constexpr (tile_shape_in0::ValidCol > 0 && tile_shape_in0::ValidRow > 0) { + "TCOLEXPANDDIV broadcast source must be a one-row tile"); + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { static_assert(std::is_same::value, - "TCOLEXPANDSUB: src0/src1 dtype must match"); + "TCOLEXPANDDIV: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TCOLEXPANDSUB: src0/dst dtype must match"); + "TCOLEXPANDDIV: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 86, %D1\n" + "BSTART.TEPL 88, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" @@ -10967,111 +15483,94 @@ void TCOLEXPANDSUB(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { static_assert(std::is_same::value, - "TCOLEXPANDSUB: src0/src1 dtype must match"); + "TCOLEXPANDDIV: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TCOLEXPANDSUB: src0/dst dtype must match"); + "TCOLEXPANDDIV: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 86, %D1\n" + "BSTART.TEPL 88, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TCOLEXPANDMUL: col broadcast mul -// src1 is a per-column scalar/byte-strip operand whose shape may differ from src0 -// (col-broadcast analog of pto/TROWEXPANDMUL.md Mode 1/2); only dtype must match. -template -void TCOLEXPANDMUL(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { - // ASL (expansion): column expansion broadcasts a one-row source; the - // destination geometry comes from the destination B.DIM, not the source. - static_assert(tile_shape_in1::ValidRow == DYNAMIC || tile_shape_in1::ValidRow == 1, - "TCOLEXPANDMUL broadcast source must be a one-row tile"); - if constexpr (tile_shape_in0::ValidCol > 0 && tile_shape_in0::ValidRow > 0) { + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { static_assert(std::is_same::value, - "TCOLEXPANDMUL: src0/src1 dtype must match"); + "TCOLEXPANDDIV: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TCOLEXPANDMUL: src0/dst dtype must match"); + "TCOLEXPANDDIV: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 87, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "BSTART.TEPL 88, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), + [dst____dimcol] "r"(dst.GetValidCol()), "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else { static_assert(std::is_same::value, - "TCOLEXPANDMUL: src0/src1 dtype must match"); + "TCOLEXPANDDIV: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TCOLEXPANDMUL: src0/dst dtype must match"); + "TCOLEXPANDDIV: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 87, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "BSTART.TEPL 88, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } } -// TCOLEXPANDDIV: col broadcast div +// TCOLEXPANDMAX: col broadcast max // src1 is a per-column scalar/byte-strip operand whose shape may differ from src0 // (col-broadcast analog of pto/TROWEXPANDMUL.md Mode 1/2); only dtype must match. template -void TCOLEXPANDDIV(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { +void TCOLEXPANDMAX(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { // ASL (expansion): column expansion broadcasts a one-row source; the // destination geometry comes from the destination B.DIM, not the source. static_assert(tile_shape_in1::ValidRow == DYNAMIC || tile_shape_in1::ValidRow == 1, - "TCOLEXPANDDIV broadcast source must be a one-row tile"); - if constexpr (tile_shape_in0::ValidCol > 0 && tile_shape_in0::ValidRow > 0) { + "TCOLEXPANDMAX broadcast source must be a one-row tile"); + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { static_assert(std::is_same::value, - "TCOLEXPANDDIV: src0/src1 dtype must match"); + "TCOLEXPANDMAX: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TCOLEXPANDDIV: src0/dst dtype must match"); + "TCOLEXPANDMAX: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 88, %D1\n" + "BSTART.TEPL 89, %D1\n" "B.DIM zero, %c2, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" @@ -11085,44 +15584,29 @@ void TCOLEXPANDDIV(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { static_assert(std::is_same::value, - "TCOLEXPANDDIV: src0/src1 dtype must match"); + "TCOLEXPANDMAX: src0/src1 dtype must match"); static_assert(std::is_same::value, - "TCOLEXPANDDIV: src0/dst dtype must match"); + "TCOLEXPANDMAX: src0/dst dtype must match"); asm volatile( - "BSTART.TEPL 88, %D1\n" + "BSTART.TEPL 89, %D1\n" "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } -} - -// TCOLEXPANDMAX: col broadcast max -// src1 is a per-column scalar/byte-strip operand whose shape may differ from src0 -// (col-broadcast analog of pto/TROWEXPANDMUL.md Mode 1/2); only dtype must match. -template -void TCOLEXPANDMAX(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { - // ASL (expansion): column expansion broadcasts a one-row source; the - // destination geometry comes from the destination B.DIM, not the source. - static_assert(tile_shape_in1::ValidRow == DYNAMIC || tile_shape_in1::ValidRow == 1, - "TCOLEXPANDMAX broadcast source must be a one-row tile"); - if constexpr (tile_shape_in0::ValidCol > 0 && tile_shape_in0::ValidRow > 0) { + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { static_assert(std::is_same::value, "TCOLEXPANDMAX: src0/src1 dtype must match"); @@ -11131,21 +15615,20 @@ void TCOLEXPANDMAX(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr "TCOLEXPANDMAX: src0/dst dtype must match"); asm volatile( "BSTART.TEPL 89, %D1\n" - "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "i"(tile_shape_out::ValidCol), + [dst____dimcol] "r"(dst.GetValidCol()), "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else { static_assert(std::is_same::value, "TCOLEXPANDMAX: src0/src1 dtype must match"); @@ -11154,21 +15637,20 @@ void TCOLEXPANDMAX(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr "TCOLEXPANDMAX: src0/dst dtype must match"); asm volatile( "BSTART.TEPL 89, %D1\n" - "B.DIM zero, %c2, ->lb0\n" - "B.DIM zero, %c3, ->lb1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } } // TCOLEXPANDMIN: col broadcast min @@ -11181,7 +15663,7 @@ void TCOLEXPANDMIN(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr // destination geometry comes from the destination B.DIM, not the source. static_assert(tile_shape_in1::ValidRow == DYNAMIC || tile_shape_in1::ValidRow == 1, "TCOLEXPANDMIN broadcast source must be a one-row tile"); - if constexpr (tile_shape_in0::ValidCol > 0 && tile_shape_in0::ValidRow > 0) { + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { static_assert(std::is_same::value, "TCOLEXPANDMIN: src0/src1 dtype must match"); @@ -11203,8 +15685,7 @@ void TCOLEXPANDMIN(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { static_assert(std::is_same::value, "TCOLEXPANDMIN: src0/src1 dtype must match"); @@ -11214,20 +15695,63 @@ void TCOLEXPANDMIN(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &sr asm volatile( "BSTART.TEPL 90, %D1\n" "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { + static_assert(std::is_same::value, + "TCOLEXPANDMIN: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TCOLEXPANDMIN: src0/dst dtype must match"); + asm volatile( + "BSTART.TEPL 90, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + [dst____dimcol] "r"(dst.GetValidCol()), + "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } else { + static_assert(std::is_same::value, + "TCOLEXPANDMIN: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TCOLEXPANDMIN: src0/dst dtype must match"); + asm volatile( + "BSTART.TEPL 90, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } // TCOLEXPANDEXPDIF: col exp(src0-src1) @@ -11240,7 +15764,7 @@ void TCOLEXPANDEXPDIF(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 // destination geometry comes from the destination B.DIM, not the source. static_assert(tile_shape_in1::ValidRow == DYNAMIC || tile_shape_in1::ValidRow == 1, "TCOLEXPANDEXPDIF broadcast source must be a one-row tile"); - if constexpr (tile_shape_in0::ValidCol > 0 && tile_shape_in0::ValidRow > 0) { + if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow > 0) { static_assert(std::is_same::value, "TCOLEXPANDEXPDIF: src0/src1 dtype must match"); @@ -11262,8 +15786,7 @@ void TCOLEXPANDEXPDIF(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { static_assert(std::is_same::value, "TCOLEXPANDEXPDIF: src0/src1 dtype must match"); @@ -11273,20 +15796,63 @@ void TCOLEXPANDEXPDIF(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 asm volatile( "BSTART.TEPL 91, %D1\n" "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { + static_assert(std::is_same::value, + "TCOLEXPANDEXPDIF: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TCOLEXPANDEXPDIF: src0/dst dtype must match"); + asm volatile( + "BSTART.TEPL 91, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + [dst____dimcol] "r"(dst.GetValidCol()), + "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } else { + static_assert(std::is_same::value, + "TCOLEXPANDEXPDIF: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TCOLEXPANDEXPDIF: src0/dst dtype must match"); + asm volatile( + "BSTART.TEPL 91, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } //===--- TEPL Mode 3: complex ops (BSTART.TEPL) ---===// @@ -11327,8 +15893,7 @@ void TCONCAT(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } else { + ); } else if constexpr (tile_shape_out::ValidCol > 0 && tile_shape_out::ValidRow < 0) { static_assert(std::is_same::value, "TCONCAT: src0/src1 dtype must match"); @@ -11344,20 +15909,75 @@ void TCONCAT(tile_shape_out &dst, tile_shape_in0 &src0, tile_shape_in1 &src1) { asm volatile( "BSTART.TEPL 96, %D1\n" "B.DIM zero, %c2, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + "i"(tile_shape_out::ValidCol), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } else if constexpr (tile_shape_out::ValidCol < 0 && tile_shape_out::ValidRow > 0) { + static_assert(std::is_same::value, + "TCONCAT: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TCONCAT: src0/dst dtype must match"); + // Row count is invariant across concat (src0.Rows == src1.Rows == dst.Rows); + // dst's valid col / total col / row stride come from dst itself. + static_assert(tile_shape_in0::Rows == tile_shape_in1::Rows, + "TCONCAT: src0/src1 row count must match"); + static_assert(tile_shape_in0::Rows == tile_shape_out::Rows, + "TCONCAT: src0/dst row count must match"); + asm volatile( + "BSTART.TEPL 96, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" "B.DIM zero, %c3, ->lb1\n" "B.DIM zero, %c4, ->lb2\n" "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" "" : "=Tr"(dst.data()) : "i"(type_traits::TypeCode), - "ri"(dst.GetValidCol()), - "ri"(dst.GetValidRow()), + [dst____dimcol] "r"(dst.GetValidCol()), + "i"(tile_shape_out::ValidRow), "i"(tile_shape_out::Cols), "Tr"(src0.data()), "Tr"(src1.data()), "i"(tile_type_traits::TilesizeCode) - ); - } + ); } else { + static_assert(std::is_same::value, + "TCONCAT: src0/src1 dtype must match"); + static_assert(std::is_same::value, + "TCONCAT: src0/dst dtype must match"); + // Row count is invariant across concat (src0.Rows == src1.Rows == dst.Rows); + // dst's valid col / total col / row stride come from dst itself. + static_assert(tile_shape_in0::Rows == tile_shape_in1::Rows, + "TCONCAT: src0/src1 row count must match"); + static_assert(tile_shape_in0::Rows == tile_shape_out::Rows, + "TCONCAT: src0/dst row count must match"); + asm volatile( + "BSTART.TEPL 96, %D1\n" + "B.DIM %[dst____dimcol], 0, ->lb0\n" + "B.DIM %[dst____dimrow], 0, ->lb1\n" + "B.DIM zero, %c4, ->lb2\n" + "B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\n" + "" + : "=Tr"(dst.data()) + : "i"(type_traits::TypeCode), + [dst____dimcol] "r"(dst.GetValidCol()), + [dst____dimrow] "r"(dst.GetValidRow()), + "i"(tile_shape_out::Cols), + "Tr"(src0.data()), + "Tr"(src1.data()), + "i"(tile_type_traits::TilesizeCode) + ); } } // PTO ISA 0.58.5 layout-and-rearrangement operations (TEPL mode 3). From cbcf25c947c8e17b263fd0b0fc545629ae1610f5 Mon Sep 17 00:00:00 2001 From: LinxISA Automation Date: Mon, 7 Sep 2026 23:36:49 +0800 Subject: [PATCH 3/3] TileOP: add per-dimension B.DIM regression coverage - ValidShapePerDim.cpp: TADD/TMUL/TRECIP/TEXP across SS/SD/DS/DD - ValidShapePerDimSweep.cpp: compile-time sweep of ~65 interface instantiations x four combinations (elementwise, unary, row/col reduce, scalar variants, TCMP/THISTOGRAM/TLOAD_CUBE/TSTORE_CUBE); disassembly form matrix is symmetric (96 static + 96 dynamic per lb0/lb1, lb2 all immediate) with zero illegal forms - compile.all: register both fixtures - test_v058_engine_contract.py: assert no "ri" masquerade, lb2 immediate-form invariant (TPREFETCH dynamic-GM fallback exempted), TMATMUL M/N/K forms; update TCVT anchors for the removed DEFINE_TMOV_LAYOUT macro --- test/test_v058_engine_contract.py | 77 +++++++---- test/tileop_api/compile.all | 2 +- test/tileop_api/src/ValidShapePerDim.cpp | 45 +++++++ test/tileop_api/src/ValidShapePerDimSweep.cpp | 126 ++++++++++++++++++ 4 files changed, 221 insertions(+), 29 deletions(-) create mode 100644 test/tileop_api/src/ValidShapePerDim.cpp create mode 100644 test/tileop_api/src/ValidShapePerDimSweep.cpp diff --git a/test/test_v058_engine_contract.py b/test/test_v058_engine_contract.py index 84699ad..a2426de 100644 --- a/test/test_v058_engine_contract.py +++ b/test/test_v058_engine_contract.py @@ -138,7 +138,7 @@ def test_tcvt_emits_dimensions_before_terminating_iot(self) -> None: tcvt = re.search( r'(?s)template \n' - r'void TCVT_T\(.*?\n}\n\n#define DEFINE_TMOV_LAYOUT', + r'void TCVT_T\(.*?\n}\n\n\n// PTO ISA 0.58 generic Local-to-Local TMOV', self.header, ) self.assertIsNotNone(tcvt) @@ -157,7 +157,7 @@ def test_tcvt_cube_layout_closure_uses_destination_tsize(self) -> None: tcvt = re.search( r'(?s)template \n' - r'void TCVT_T\(.*?\n}\n\n#define DEFINE_TMOV_LAYOUT', + r'void TCVT_T\(.*?\n}\n\n\n// PTO ISA 0.58 generic Local-to-Local TMOV', self.header, ) self.assertIsNotNone(tcvt) @@ -244,19 +244,23 @@ def test_tsel_binds_explicit_false_source_on_both_shape_paths(self) -> None: start = self.header.index("void TSEL(") end = self.header.index("// TABS:", start) tsel = self.header[start:end] + # Per-dimension dispatch: four B.DIM combinations, each a full asm + # statement with the same binder shape. self.assertEqual( - tsel.count('"B.IOT %6, %7, mask=1111\\n"'), 2 + tsel.count('"B.IOT %6, %7, mask=1111\\n"'), 4 ) self.assertEqual( tsel.count( '"B.IOT %1, mask=1111, last, ->%0<%Z8>\\n"' ), - 2, + 4, ) - self.assertEqual(tsel.count(': [Dst] "=Tr"(dst.data())'), 2) - self.assertEqual(tsel.count('[Prior] "0"(dst.data())'), 2) + self.assertEqual(tsel.count(': [Dst] "=Tr"(dst.data())'), 4) + self.assertEqual(tsel.count('[Prior] "0"(dst.data())'), 4) self.assertIn('"B.DIM zero, %c3, ->lb0\\n"', tsel) - self.assertIn('"B.DIM %3, 0, ->lb0\\n"', tsel) + self.assertIn('"B.DIM %[mask____dimcol], 0, ->lb0\\n"', tsel) + self.assertIn('"B.DIM %[mask____dimrow], 0, ->lb1\\n"', tsel) + self.assertIn("::ValidCol > 0 && ", tsel) self.assertNotIn( '"B.IOT %5, %6, mask=1111, last, ->%0<%Z7>\\n"', tsel ) @@ -272,18 +276,10 @@ def test_valid_shape_immediate_fixture_separates_static_and_dynamic_paths(self) def test_static_valid_shape_bindings_remain_immediate_eligible(self) -> None: header = self.header - timg2col = header[header.index("void TIMG2COL"):header.index("// TFILLPAD")] - for spelling in ( - '"ri"(dst.GetValidCol())', - '"ri"(dst.GetValidRow())', - '"ri"(src.GetValidCol())', - '"ri"(src.GetValidRow())', - '"ri"(offset.GetValidCol())', - '"ri"(offset.GetValidRow())', - '[VCOL] "ri"(validCol)', - '[VROW] "ri"(validRow)', - ): - self.assertIn(spelling, header) + # Per-dimension dispatch: every B.DIM site splits into the four + # ValidCol/ValidRow combinations, so no "ri" masquerade may remain + # and fully-dynamic operands must bind as plain "r" registers. + self.assertNotIn('"ri"(', header) for spelling in ( '"r"(dst.GetValidCol())', '"r"(dst.GetValidRow())', @@ -292,9 +288,28 @@ def test_static_valid_shape_bindings_remain_immediate_eligible(self) -> None: '"r"(offset.GetValidCol())', '"r"(offset.GetValidRow())', ): - self.assertNotIn(spelling, header[: header.index("void TIMG2COL")]) - self.assertIn('"r"(dst.GetValidCol())', timg2col) - self.assertIn('"r"(dst.GetValidRow())', timg2col) + self.assertIn(spelling, header) + # A register must never land in the immediate slot of B.DIM. + self.assertNotIn("B.DIM zero, %[", header) + # lb2 must always stay the static immediate form; the only allowed + # dynamic lb2 is TPREFETCH's dynamic-GM fallback branch. + dyn_lb2 = [ + line for line in header.split("\n") + if "->lb2" in line and "B.DIM" in line and "%c" not in line + ] + tprefetch = header[header.index("void TPREFETCH"):] + tprefetch = tprefetch[: tprefetch.index("\n}")] + self.assertEqual( + len(dyn_lb2), + tprefetch.count('"B.DIM %[Col], 0, ->lb2\\n"'), + "lb2 must be immediate-form everywhere except TPREFETCH's " + "dynamic-GM fallback", + ) + # TMATMUL: M stays a runtime register (group_M), N/K are compile-time + # immediates per resolve_matmul_shape. + self.assertIn('"B.DIM %[M], 0, ->lb0\\n"', header) + self.assertIn('"B.DIM zero, %c[N], ->lb1\\n"', header) + self.assertIn('"B.DIM zero, %c[K], ->lb2\\n"', header) def test_fpatr_carries_shared_transpose_controls(self) -> None: tile = PTO_TILE.read_text(encoding="utf-8") @@ -533,12 +548,18 @@ def test_timg2col_bundle_has_gm_and_parameter_iors(self) -> None: self.assertRegex(self.header, r'B\.IOT mask=1111, last, ->%\[Dst\]') def test_timg2col_uses_destination_geometry_and_cube_output(self) -> None: - body = self.header[self.header.index("// TIMG2COL") : self.header.index("// TFILLPAD")] - self.assertIn("tile_shape_out::Loc == Location::Left", body) - self.assertIn('"r"(dst.GetValidCol())', body) - self.assertIn('"r"(dst.GetValidRow())', body) - self.assertIn("BLayout::CubeM16", body) - self.assertIn("BLayout::CubeM32", body) + body = self.header[self.header.index("void TIMG2COL"):self.header.index("// TFILLPAD")] + self.assertIn( + "requires(tile_shape_out::Loc == Location::Left", + self.header[self.header.rindex("template", 0, self.header.index("void TIMG2COL")):self.header.index("// TFILLPAD")]) + self.assertIn('[ValidCol] "r"(dst.GetValidCol())', body) + self.assertIn('[ValidRow] "r"(dst.GetValidRow())', body) + self.assertIn('[ValidCol] "i"(tile_shape_out::ValidCol)', body) + self.assertIn('[ValidRow] "i"(tile_shape_out::ValidRow)', body) + self.assertIn('"B.DIM zero, %c[ValidCol], ->lb0\\n"', body) + self.assertIn('"B.DIM zero, %c[ValidRow], ->lb1\\n"', body) + self.assertIn("LayoutCvtEnum::ND2M16", body) + self.assertIn("LayoutCvtEnum::ND2M32", body) def test_tquant_tdequant_use_datr_and_ior(self) -> None: # TQUANT/TDEQUANT: B.DATR carries named dtype/RMode and optional sat, diff --git a/test/tileop_api/compile.all b/test/tileop_api/compile.all index 796bb7f..be64cfc 100755 --- a/test/tileop_api/compile.all +++ b/test/tileop_api/compile.all @@ -25,7 +25,7 @@ ACTIVE_FIXTURES=( TAdd_mask TAdd TAdds TAnd TLoad TStore TCvt TDiv TDivs TExp TMax TMaxs TMov TMul TMuls TOr TRecip TRem TRowMax TRowSum TSqrt TSub TSubs TTrans RangeSubview RangeAssemble SharedRange TileRegion TileArray TileArrayCube TileArrayAssemblyOffsets TileArrayRegionAsm TileRegionUnaryAssembly TileRegionUnary TileRegionBinary TileRegionBinaryAssembly TileRegionUnarySubviewAssembly TileRegionScalarAssembly TileRegionTCVTSubviewAssembly TileRegionSharedSubview TileRegionCubeSubview - ValidShapeImmediate TSELCanonical + ValidShapeImmediate ValidShapePerDim ValidShapePerDimSweep TSELCanonical ) # Pre-v0.58 wrapper designs remain available to cpu_sim only and are not part diff --git a/test/tileop_api/src/ValidShapePerDim.cpp b/test/tileop_api/src/ValidShapePerDim.cpp new file mode 100644 index 0000000..9d923df --- /dev/null +++ b/test/tileop_api/src/ValidShapePerDim.cpp @@ -0,0 +1,45 @@ +// Per-dimension static/dynamic B.DIM regression: all four ValidCol/ValidRow +// combinations must lower to the correct B.DIM form. +// static dim -> B.DIM zero, imm (assembler compresses to C.B.DIMI) +// dynamic dim -> B.DIM reg, 0 +#include + +using namespace pto; + +// SS: both dims static +using TSS = Tile; +// SD: static col, dynamic row +using TSD = Tile; +// DS: dynamic col, static row +using TDS = Tile; +// DD: both dynamic +using TDD = Tile; + +__attribute__((noinline)) void case_ss_static_both(TSS &a, TSS &b, TSS &c) { TADD(a, b, c); } +__attribute__((noinline)) void case_sd_static_col(TSD &a, TSD &b) { TADD(a, b, b); } +__attribute__((noinline)) void case_ds_static_row(TDS &a, TDS &b) { TADD(a, b, b); } +__attribute__((noinline)) void case_dd_dynamic_both(TDD &a, TDD &b) { TADD(a, b, b); } + +__attribute__((noinline)) void case_tmul_sd_ds(TSD &a, TSD &b, TDS &c, TDS &d) { + TMUL(a, b, b); + TMUL(c, d, d); +} + +__attribute__((noinline)) void case_trecip_dynamic(TDD &a, TDD &b) { TRECIP(a, b); } + +__attribute__((noinline)) void case_texp_mixed(TSD &a, TSD &b) { TEXP(a, b); } + +int main() { + static TSS ss_a, ss_b, ss_c; + static TSD sd_a, sd_b; + static TDS ds_a, ds_b; + static TDD dd_a, dd_b; + case_ss_static_both(ss_a, ss_b, ss_c); + case_sd_static_col(sd_a, sd_b); + case_ds_static_row(ds_a, ds_b); + case_dd_dynamic_both(dd_a, dd_b); + case_tmul_sd_ds(sd_a, sd_b, ds_a, ds_b); + case_trecip_dynamic(dd_a, dd_b); + case_texp_mixed(sd_a, sd_b); + return 0; +} diff --git a/test/tileop_api/src/ValidShapePerDimSweep.cpp b/test/tileop_api/src/ValidShapePerDimSweep.cpp new file mode 100644 index 0000000..d62773e --- /dev/null +++ b/test/tileop_api/src/ValidShapePerDimSweep.cpp @@ -0,0 +1,126 @@ +// Per-dimension four-combination sweep: every four-branch B.DIM interface +// instantiated with all ValidCol/ValidRow combinations (SS/SD/DS/DD). +// SS -> C.B.DIMI lb0 + C.B.DIMI lb1 +// SD -> C.B.DIMI lb0 + B.DIM reg,0 lb1 +// DS -> B.DIM reg,0 lb0 + C.B.DIMI lb1 +// DD -> B.DIM reg,0 lb0 + B.DIM reg,0 lb1 +// lb2 always stays immediate-form (C.B.DIMI). +#include + +using namespace pto; + +template using T = + Tile; + +#define SWEEP3(FN, TY) \ + __attribute__((noinline)) void FN##_##TY(T &a, T &b, T &c){FN(a,b,c);} +// 简化: 每组合独立函数 +#define SWEEP2(FN) \ + __attribute__((noinline)) void FN##_ss(T<16,32> &a,T<16,32> &b,T<16,32> &c){FN(a,b,c);} \ + __attribute__((noinline)) void FN##_sd(T<16,-1> &a,T<16,-1> &b,T<16,-1> &c){FN(a,b,c);} \ + __attribute__((noinline)) void FN##_ds(T<-1,32> &a,T<-1,32> &b,T<-1,32> &c){FN(a,b,c);} \ + __attribute__((noinline)) void FN##_dd(T<-1,-1> &a,T<-1,-1> &b,T<-1,-1> &c){FN(a,b,c);} + +SWEEP2(TADD) SWEEP2(TSUB) SWEEP2(TMUL) SWEEP2(TDIV) SWEEP2(TMAX) SWEEP2(TMIN) +SWEEP2(TAND) SWEEP2(TOR) SWEEP2(TXOR) SWEEP2(TSHL) SWEEP2(TSHR) SWEEP2(TREM) + + + +template using T = + Tile; + +#define SWEEP1(FN) \ + __attribute__((noinline)) void FN##_ss(T<16,32> &a,T<16,32> &b){FN(a,b);} \ + __attribute__((noinline)) void FN##_sd(T<16,-1> &a,T<16,-1> &b){FN(a,b);} \ + __attribute__((noinline)) void FN##_ds(T<-1,32> &a,T<-1,32> &b){FN(a,b);} \ + __attribute__((noinline)) void FN##_dd(T<-1,-1> &a,T<-1,-1> &b){FN(a,b);} + +SWEEP1(TABS) SWEEP1(TEXP) SWEEP1(TLOG) SWEEP1(TNEG) SWEEP1(TNOT) +SWEEP1(TRECIP) SWEEP1(TRELU) SWEEP1(TRSQRT) SWEEP1(TSQRT) + +SWEEP1(TCVT_T) SWEEP1(TMOV) SWEEP1(TTRANS) + + +// TTRI(dst) single-arg +__attribute__((noinline)) void ttri_ss(T<16,32> &a){TTRI(a);} +__attribute__((noinline)) void ttri_sd(T<16,-1> &a){TTRI(a);} +__attribute__((noinline)) void ttri_ds(T<-1,32> &a){TTRI(a);} +__attribute__((noinline)) void ttri_dd(T<-1,-1> &a){TTRI(a);} + +// row-reduce: dst must be N x 1 (Cols==1, so only ValidRow varies) +template using R1 = + Tile; +#define SWEEPR(FN) \ + __attribute__((noinline)) void FN##_ss(R1<16,1> &a,R1<16,1> &b){FN(a,b);} \ + __attribute__((noinline)) void FN##_sd(R1<16,-1> &a,R1<16,-1> &b){FN(a,b);} \ + __attribute__((noinline)) void FN##_ds(R1<-1,1> &a,R1<-1,1> &b){FN(a,b);} \ + __attribute__((noinline)) void FN##_dd(R1<-1,-1> &a,R1<-1,-1> &b){FN(a,b);} +SWEEPR(TROWMAX) SWEEPR(TROWMIN) SWEEPR(TROWPROD) SWEEPR(TROWSUM) + +template using T = + Tile; +template using TQ = + Tile; + +#define SWEEPS(FN) \ + __attribute__((noinline)) void FN##_ss(T<16,32> &a,T<16,32> &b){FN(a,b,1.0f);} \ + __attribute__((noinline)) void FN##_sd(T<16,-1> &a,T<16,-1> &b){FN(a,b,1.0f);} \ + __attribute__((noinline)) void FN##_ds(T<-1,32> &a,T<-1,32> &b){FN(a,b,1.0f);} \ + __attribute__((noinline)) void FN##_dd(T<-1,-1> &a,T<-1,-1> &b){FN(a,b,1.0f);} +SWEEPS(TADDS) SWEEPS(TSUBS) SWEEPS(TMULS) SWEEPS(TDIVS) SWEEPS(TREMS) +SWEEPS(TANDS) SWEEPS(TORS) SWEEPS(TXORS) SWEEPS(TSHLS) SWEEPS(TSHRS) +SWEEPS(TMAXS) SWEEPS(TMINS) SWEEPS(TCMPS) + +#define SWEEPQ(FN) \ + __attribute__((noinline)) void FN##_ss(TQ<16,32> &d,T<16,32> &s){FN(d,s,1.0f,2);} \ + __attribute__((noinline)) void FN##_sd(TQ<16,-1> &d,T<16,-1> &s){FN(d,s,1.0f,2);} \ + __attribute__((noinline)) void FN##_ds(TQ<-1,32> &d,T<-1,32> &s){FN(d,s,1.0f,2);} \ + __attribute__((noinline)) void FN##_dd(TQ<-1,-1> &d,T<-1,-1> &s){FN(d,s,1.0f,2);} +SWEEPQ(TQUANT) + + +// TSELS(dst, src0, s, src1) +__attribute__((noinline)) void tsels_ss(T<16,32>&a,T<16,32>&b,T<16,32>&c){TSELS(a,b,1.0f,c);} +__attribute__((noinline)) void tsels_sd(T<16,-1>&a,T<16,-1>&b,T<16,-1>&c){TSELS(a,b,1.0f,c);} +__attribute__((noinline)) void tsels_ds(T<-1,32>&a,T<-1,32>&b,T<-1,32>&c){TSELS(a,b,1.0f,c);} +__attribute__((noinline)) void tsels_dd(T<-1,-1>&a,T<-1,-1>&b,T<-1,-1>&c){TSELS(a,b,1.0f,c);} + +template using T = + Tile; +template using TCUBE = + Tile; + +// TCMP needs CmpMode template arg +#define SWEEPC(FN) \ + __attribute__((noinline)) void FN##_ss(T<16,32> &a,T<16,32> &b,T<16,32> &c){FN(a,b,c);} \ + __attribute__((noinline)) void FN##_sd(T<16,-1> &a,T<16,-1> &b,T<16,-1> &c){FN(a,b,c);} \ + __attribute__((noinline)) void FN##_ds(T<-1,32> &a,T<-1,32> &b,T<-1,32> &c){FN(a,b,c);} \ + __attribute__((noinline)) void FN##_dd(T<-1,-1> &a,T<-1,-1> &b,T<-1,-1> &c){FN(a,b,c);} +SWEEPC(TCMP) + +// THISTOGRAM(dst, src, Idx, ByteId) +#define SWEEPH(FN) \ + __attribute__((noinline)) void FN##_ss(T<16,32> &a,T<16,32> &b,T<16,32> &c){FN(a,b,c,0);} \ + __attribute__((noinline)) void FN##_sd(T<16,-1> &a,T<16,-1> &b,T<16,-1> &c){FN(a,b,c,0);} \ + __attribute__((noinline)) void FN##_ds(T<-1,32> &a,T<-1,32> &b,T<-1,32> &c){FN(a,b,c,0);} \ + __attribute__((noinline)) void FN##_dd(T<-1,-1> &a,T<-1,-1> &b,T<-1,-1> &c){FN(a,b,c,0);} +SWEEPH(THISTOGRAM) + +// TLOAD_CUBE / TSTORE_CUBE +using gmc = GlobalTensor, Stride<1,1,16*64,64,1>, Layout::ND>; +__attribute__((noinline)) void tlcb_ss(gmc &g, TCUBE<16,64> &d){TLOAD_CUBE(d,g);} +__attribute__((noinline)) void tlcb_sd(gmc &g, TCUBE<16,-1> &d){TLOAD_CUBE(d,g);} +__attribute__((noinline)) void tlcb_ds(gmc &g, TCUBE<-1,64> &d){TLOAD_CUBE(d,g);} +__attribute__((noinline)) void tlcb_dd(gmc &g, TCUBE<-1,-1> &d){TLOAD_CUBE(d,g);} +__attribute__((noinline)) void tscb_ss(gmc &g, TCUBE<16,64> &d){TSTORE_CUBE(g,d);} +__attribute__((noinline)) void tscb_sd(gmc &g, TCUBE<16,-1> &d){TSTORE_CUBE(g,d);} +__attribute__((noinline)) void tscb_ds(gmc &g, TCUBE<-1,64> &d){TSTORE_CUBE(g,d);} +__attribute__((noinline)) void tscb_dd(gmc &g, TCUBE<-1,-1> &d){TSTORE_CUBE(g,d);} + + + +template using T = Tile; + + + +int main() { return 0; }