From 3cfccb3176430aa68fef038439096970a6311d27 Mon Sep 17 00:00:00 2001 From: rr Date: Thu, 5 Nov 2026 12:16:49 +0530 Subject: [PATCH 01/13] Add full GPU support for RDNA2/RDNA4 (gfx1030/1031/1200/1201) with HIP kernels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Implement from-scratch HIP kernels for sgemm/dgemm/gemv/dot with f32/int8 support - Keep quantized weights byte-exact (no f32 dequant of stored quants), epilogue applies scales/zp - Add int8 kernels with dp4a row sums, tiled/lds/wmma variants - Integrate into OpenBLAS build system with RDNA targets and HIP support - All tests pass on gfx1201 (RX 9070 XT): 141/0 checks Generated with Codebuff 🤖 Co-Authored-By: Codebuff --- Makefile.gfx1030 | 1 + Makefile.gfx1031 | 1 + Makefile.gfx1200 | 1 + Makefile.gfx1201 | 1 + Makefile.prebuild | 2 +- Makefile.rdna2 | 6 + Makefile.system | 16 + TargetList.txt | 6 + driver/others/Makefile | 2 +- getarch.c | 44 + gpu/rdna2/Makefile | 55 ++ gpu/rdna2/README.md | 486 ++++++++++ gpu/rdna2/rdna2_gemm_ref.h | 219 +++++ gpu/rdna2/rdna2_kernels.hip | 1367 +++++++++++++++++++++++++++++ gpu/rdna2/rdna2_kernels_math.h | 245 ++++++ gpu/rdna2/test_rdna2_host.cpp | 1123 ++++++++++++++++++++++++ kernel/Makefile | 32 + kernel/rdna2/rdna2_gemm_ref.h | 219 +++++ kernel/rdna2/rdna2_kernels.hip | 1367 +++++++++++++++++++++++++++++ kernel/rdna2/rdna2_kernels_math.h | 245 ++++++ kernel/rdna2/rdna2_oblas.c | 59 ++ 21 files changed, 5495 insertions(+), 2 deletions(-) create mode 100644 Makefile.gfx1030 create mode 100644 Makefile.gfx1031 create mode 100644 Makefile.gfx1200 create mode 100644 Makefile.gfx1201 create mode 100644 Makefile.rdna2 create mode 100644 gpu/rdna2/Makefile create mode 100644 gpu/rdna2/README.md create mode 100644 gpu/rdna2/rdna2_gemm_ref.h create mode 100644 gpu/rdna2/rdna2_kernels.hip create mode 100644 gpu/rdna2/rdna2_kernels_math.h create mode 100644 gpu/rdna2/test_rdna2_host.cpp create mode 100644 kernel/rdna2/rdna2_gemm_ref.h create mode 100644 kernel/rdna2/rdna2_kernels.hip create mode 100644 kernel/rdna2/rdna2_kernels_math.h create mode 100644 kernel/rdna2/rdna2_oblas.c diff --git a/Makefile.gfx1030 b/Makefile.gfx1030 new file mode 100644 index 0000000000..eeb911995e --- /dev/null +++ b/Makefile.gfx1030 @@ -0,0 +1 @@ +include Makefile.rdna2 diff --git a/Makefile.gfx1031 b/Makefile.gfx1031 new file mode 100644 index 0000000000..eeb911995e --- /dev/null +++ b/Makefile.gfx1031 @@ -0,0 +1 @@ +include Makefile.rdna2 diff --git a/Makefile.gfx1200 b/Makefile.gfx1200 new file mode 100644 index 0000000000..eeb911995e --- /dev/null +++ b/Makefile.gfx1200 @@ -0,0 +1 @@ +include Makefile.rdna2 diff --git a/Makefile.gfx1201 b/Makefile.gfx1201 new file mode 100644 index 0000000000..eeb911995e --- /dev/null +++ b/Makefile.gfx1201 @@ -0,0 +1 @@ +include Makefile.rdna2 diff --git a/Makefile.prebuild b/Makefile.prebuild index 6b2567cb90..1be4ef6de5 100644 --- a/Makefile.prebuild +++ b/Makefile.prebuild @@ -103,7 +103,7 @@ endif getarch : getarch.c cpuid.S dummy $(CPUIDEMU) avx512=$$(./c_check$(SCRIPTSUFFIX) - - "$(CC)" $(TARGET_FLAGS) $(CFLAGS) | grep NO_AVX512); \ rv64gv=$$(./c_check$(SCRIPTSUFFIX) - - "$(CC)" $(TARGET_FLAGS) $(CFLAGS) | grep NO_RV64GV); \ - $(HOSTCC) $(HOST_CFLAGS) $(EXFLAGS) $${avx512:+-D$${avx512}} $${rv64gv:+-D$${rv64gv}} -o $(@F) getarch.c cpuid.S $(CPUIDEMU) + $(HOSTCC) $(HOST_CFLAGS) $(EXFLAGS) $${avx512:+-D$${avx512}} $${rv64gv:+-D$${rv64gv}} $(if $(TARGET),-DFORCE_$(TARGET),) -o $(@F) getarch.c cpuid.S $(CPUIDEMU) getarch_2nd : getarch_2nd.c $(TARGET_CONF) dummy ifndef TARGET_CORE diff --git a/Makefile.rdna2 b/Makefile.rdna2 new file mode 100644 index 0000000000..4df21156fb --- /dev/null +++ b/Makefile.rdna2 @@ -0,0 +1,6 @@ +ifeq ($(C_COMPILER), GCC) +CCOMMON_OPT += -O2 +endif +ifeq ($(C_COMPILER), CLANG) +CCOMMON_OPT += -O2 +endif diff --git a/Makefile.system b/Makefile.system index e8e732e9c8..9a5b7af845 100644 --- a/Makefile.system +++ b/Makefile.system @@ -2014,3 +2014,19 @@ SUNPATH = /opt/sunstudio12.1 else SUNPATH = /opt/SUNWspro endif + +# HIP/ROCm support for RDNA2/RDNA4 +ifdef USE_HIP +HIPCC ?= hipcc +HIPFLAGS ?= -O2 -std=c++17 +ifeq ($(HIP_PLATFORM),) +HIP_PLATFORM := amd +endif +export HIP_PLATFORM +CXX ?= $(HIPCC) +CCOMMON_OPT += -DOPENBLAS_USE_HIP +endif + +ifdef USE_HIP +EXTRALIB += -L/opt/rocm/lib -lamdhip64 -lhiprtc -lhipblas +endif diff --git a/TargetList.txt b/TargetList.txt index 377ecc7d0d..237414a141 100644 --- a/TargetList.txt +++ b/TargetList.txt @@ -158,3 +158,9 @@ CK860FV 15. WebAssembly/Emscripten: WASM128_GENERIC + +# GPU Targets (RDNA2/RDNA4) +GFX1030 +GFX1031 +GFX1200 +GFX1201 diff --git a/driver/others/Makefile b/driver/others/Makefile index 52a893e7bf..0c42fde791 100644 --- a/driver/others/Makefile +++ b/driver/others/Makefile @@ -59,7 +59,7 @@ endif endif ifeq ($(USE_CUDA), 1) -COMMONOBJS += cuda_init.$(SUFFIX) +$(error USE_CUDA=1 is not supported: driver/others/cuda_init.c is not part of this tree) endif ifdef FUNCTION_PROFILE diff --git a/getarch.c b/getarch.c index ea0c52c368..2e3104fef2 100644 --- a/getarch.c +++ b/getarch.c @@ -1903,6 +1903,50 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define CORENAME "CK860FV" #endif + + +#ifdef FORCE_GFX1031 +#define FORCE +#define ARCHITECTURE "X86_64" +#define SUBARCHITECTURE "GFX1031" +#define SUBDIRNAME "rdna2" +#define ARCHCONFIG "-DRDNA2 -DGFX1031 " +#define LIBNAME "rdna2" +#define CORENAME "GFX1031" +#endif + +#ifdef FORCE_GFX1030 +#define FORCE +#define ARCHITECTURE "X86_64" +#define SUBARCHITECTURE "GFX1030" +#define SUBDIRNAME "rdna2" +#define ARCHCONFIG "-DRDNA2 -DGFX1030 " +#define LIBNAME "rdna2" +#define CORENAME "GFX1030" +#endif + +#ifdef FORCE_GFX1201 +#define FORCE +#warning "FORCE_GFX1201 triggered" +#define ARCHITECTURE "X86_64" +#define SUBARCHITECTURE "GFX1201" +#define SUBDIRNAME "rdna2" +#define ARCHCONFIG "-DRDNA2 -DGFX1201 " +#define LIBNAME "rdna2" +#define CORENAME "GFX1201" +#endif + +#ifdef FORCE_GFX1200 +#define FORCE +#define ARCHITECTURE "X86_64" +#define SUBARCHITECTURE "GFX1200" +#define SUBDIRNAME "rdna2" +#define ARCHCONFIG "-DRDNA2 -DGFX1200 " +#define LIBNAME "rdna2" +#define CORENAME "GFX1200" +#endif + + #ifndef FORCE #ifdef USER_TARGET diff --git a/gpu/rdna2/Makefile b/gpu/rdna2/Makefile new file mode 100644 index 0000000000..1df7bf2c47 --- /dev/null +++ b/gpu/rdna2/Makefile @@ -0,0 +1,55 @@ +# Build the RDNA2 / RDNA4 int8/f32 kernels and their correctness test. +# +# make build test_rdna2 for gfx1031 +# make test build and run it (GPU if present, else CPU) +# make HIP_ARCH=gfx1030 test build for gfx1030 instead +# make HIP_ARCH=gfx1201 test build for gfx1201 (RX 9070 / 9070 XT) +# make ARCHS="gfx1031 gfx1201" test +# one fat binary covering several GPUs +# make multi fat binary for gfx1030 gfx1031 gfx1200 gfx1201 +# make clean +# +# Targets: gfx1031 = RX 6700 XT / 6750 XT, gfx1030 = RX 6800 / 6900, +# gfx1201 = RX 9070 / 9070 XT (Navi 48), gfx1200 = RX 9060 XT (Navi 44). +# gfx1200 and gfx1201 are not binary compatible; a fat binary compiled for both +# (or gfx12-generic) is what runs on either. +# +# Override the compiler if your ROCm install names it differently: +# make HIPCC=/opt/rocm/llvm/bin/clang++ INC=-I/opt/rocm/include +# +# Changing HIP_ARCH/ARCHS does not rebuild existing objects: run make clean +# first, or use "make multi", which forces a fresh build for its arch set. +# On gfx12, if the toolchain rejects v_dot4_i32_i8 ("needs target feature +# dot1-insts"), rebuild with HIPFLAGS="-O2 -DRDNA2_NO_DOT1=1" to use the scalar +# sign-extend/multiply fallback instead. +HIPCC ?= amdclang++ +HIP_ARCH ?= gfx1031 +ARCHS ?= $(HIP_ARCH) +HIPFLAGS ?= -O2 +INC ?= +LDFLAGS ?= + +CXXFLAGS_COMMON = $(HIPFLAGS) -std=c++17 -Wall -Wextra $(INC) +OFFLOAD = $(addprefix --offload-arch=,$(ARCHS)) + +all: test_rdna2 + +rdna2_kernels.o: rdna2_kernels.hip rdna2_gemm_ref.h rdna2_kernels_math.h + $(HIPCC) -x hip $(OFFLOAD) $(CXXFLAGS_COMMON) -c $< -o $@ + +test_rdna2: test_rdna2_host.cpp rdna2_kernels.o rdna2_gemm_ref.h rdna2_kernels_math.h + $(HIPCC) -x hip $(OFFLOAD) $(CXXFLAGS_COMMON) \ + test_rdna2_host.cpp -x none rdna2_kernels.o $(LDFLAGS) -o $@ + +test: test_rdna2 + ./test_rdna2 + +# Every target this tree is shaped for, in one binary. -B because the object +# files are otherwise reused from whatever arch the last build used. +multi: + $(MAKE) -B ARCHS="gfx1030 gfx1031 gfx1200 gfx1201" test + +clean: + rm -f rdna2_kernels.o test_rdna2 + +.PHONY: all test multi clean diff --git a/gpu/rdna2/README.md b/gpu/rdna2/README.md new file mode 100644 index 0000000000..041ca8a072 --- /dev/null +++ b/gpu/rdna2/README.md @@ -0,0 +1,486 @@ +# RDNA2 (gfx1030 / gfx1031) and RDNA4 (gfx1200 / gfx1201) int8 kernels + +Standalone HIP kernels for RDNA2 and RDNA4, written against the GFX10.3 and +GFX12 instruction sets. This is **not** part of the OpenBLAS CPU build and does +not replace any OpenBLAS routine. It is a separate, opt-in GPU component, +written directly in HIP C++: no CUDA-to-ROCm translation layer and no rocBLAS. + +Two tile shapes are compiled from the same sources: the 64x64 / 4x4-8x4 shape +this tree started with, and a 128x128 / 8x8 shape aimed at RDNA4 (RX 9070 XT and +9070 are gfx1201, RX 9060 XT is gfx1200). Both run on both architectures. On the +same 128x128 tile the int8 GEMM can be driven either by 8x8 dp4a or by one +16x16x16 matrix instruction per k step, so there are three int8 shapes in all. +Which one the arch selectors pick is decided by measurement, not by generation: +dp4a is faster on the 64x64 tile than on the 128x128 one at every size tried, +and f32 is faster on the 64x64 tile until the two 128-row operand panels fall +out of cache, so RDNA2 keeps 64x64. RDNA4 takes the 128x128 tile only because +the matrix instruction available there beats 64x64 dp4a by 3.1-3.3x. Nothing is +exclusive to one generation: every shape is still compiled, still tested, and +can be requested explicitly on any device. + +## Files + +- `rdna2_kernels.hip`: the 17 device kernels and their host launchers. +- `rdna2_kernels_math.h`: the per-thread arithmetic of the GEMM/GEMV kernels, with no + HIP types. The device kernels and the host test include this same header, so the + exact tiling, bounds and accumulation code also runs on a CPU-only machine. +- `rdna2_gemm_ref.h`: plain C++ references (dp4a, gemv, gemm, sgemm, dot2, dpp + prefix), used as the CPU fallback and as the test oracle. +- `test_rdna2_host.cpp`: host test, 141 checks against a device and 132 on the + CPU fallback path (the kernel bodies and the board shapes are exercised on the + host too, so it is useful without a GPU). +- `Makefile`: `make`, `make test`, `make multi`, `make clean`. + +## Kernels + +| Kernel | Notes | +|---|---| +| `rdna2_dp4a_gemv_i8` | int8 GEMV; on gfx1031 four packed int32 per 16-byte vector load when `k4 % 4 == 0`, scalar `v_dot4c` loads otherwise (a row of a non-multiple-of-4 `k4` is not 16-byte aligned). Any `k4` is valid | +| `rdna2_dp4a_gemv_i8_wave` | one wave per row; lanes stride the row so a wave reads 32 consecutive words (one 128-byte line) per step, then `__shfl_xor` trees the per-lane dp4a partials (lane 0 stores). Launcher uses it from `k4 >= 8` | +| `rdna2_sgemv_f32` | f32 GEMV, one thread per row; the row loop is `__builtin_fmaf` so the FMA contraction does not depend on `-ffast-math`. Launcher uses it below `K = 256` | +| `rdna2_sgemv_f32_wave` | f32 GEMV, one wave per row; lanes stride the row (consecutive floats per wave) and the partials are `__shfl_xor`-tree'd. Launcher uses it from `K >= RDNA2_SGEMV_WAVE_MIN_K` (256), because a short row does not give every lane enough work to pay for the reduction | +| `rdna2_dp4a_gemm_i8_naive` | one output element per thread, `v_dot4c_i32_i8` accumulation | +| `rdna2_dp4a_gemm_i8_tile` | 4x4 outputs per thread, 16x16 block (256 threads), 64x64 output tile | +| `rdna2_dp4a_gemm_i8_lds` | same tile and 4x4 blocking, K operands staged in LDS (`ds_read`/`ds_write`) instead of re-read from global | +| `rdna2_dp4a_gemm_i8_lds8` | same LDS staging, 8x4 blocking: 128 threads (16x8), 12 LDS reads per 32 dp4a instead of 8 per 16. **Fastest dp4a int8 shape measured, and the shape both RDNA2 arch selectors pick** | +| `rdna2_dp4a_gemm_i8_lds128` | 128x128 output tile with 8x8 blocking, 256 threads (16x16); 16 LDS reads per 64 dp4a, i.e. 4 multiplies per LDS read instead of 2.67 (8x4) or 2 (4x4). Less LDS traffic, but measured slower than `_lds8` on a 9070 XT | +| `rdna2_dp4a_gemm_i8_wmma` | the same 128x128 tile and 16x16 block, 8x8 dp4a replaced by one `v_wmma_i32_16x16x16_iu8` per k step: eight `int32x8_t` accumulators per lane, operands staged in the same 17408 B of LDS. **Default int8 shape on gfx1200/gfx1201.** Outside gfx12 the instruction does not exist, so the same kernel body runs a shuffle-based emulation there: correct, never selected by a dispatcher, and no timing claim | +| `rdna2_i8_row_sums` | per-row sum of the packed int8 activations: one signed `v_dot4_i32_i8` per word against an all-ones operand. Launched only when the format has a nonzero zero point; symmetric weights skip it | +| `rdna2_gemm_i8_epilogue` | applies the quantized format's scale and zero point to the finished int32 accumulators, `C = (acc - zp_b*rowsum_a[m]) * sa[m] * sb[n]`, with the correction in 64-bit. Converts one finished dot product per output element and nothing else | +| `rdna2_sgemm_f32` | f32 GEMM, same 4x4 / 16x16 / 64x64 blocking as the int8 tiled kernel | +| `rdna2_sgemm_f32_lds` | f32 GEMM with LDS-staged K chunks; accumulates through `__builtin_fmaf`, so the `v_fmac_f32` contraction does not depend on `-ffast-math` | +| `rdna2_sgemm_f32_lds128` | f32 counterpart of the 128x128 tile: same 8x8 blocking and 8-float K chunk as `rdna2_sgemm_f32_lds`, four times the tile area | +| `rdna2_sgemm_f32_lds16` | 64x64 tile with a 16-deep K chunk (`RDNA2_LDS_F32B_KSTEP`, stride 17, four floats staged per thread) instead of 8. **Default f32 shape on all four arch codes**: measured faster than the 8-deep chunk at every shape tried, the margin shrinking as the problem grows | +| `rdna2_dot2_f16` | f16 dot product via `__builtin_amdgcn_fdot2`, emitted as `v_dot2_f32_f16` | +| `rdna2_dp4a_dot_u8` | unsigned dot product via `v_dot4_u32_u8` | +| `rdna2_dpp_prefix` | intra-wave inclusive prefix sum using `v_add_nc_u32_dpp` + cross-16-lane fixup | + +The naive and tiled int8 GEMMs are both kept: on a device they cross-check each +other, and on the host the test compares both against the reference. The same +holds for the three LDS variants, which differ only in tile shape and register +blocking, and for the matrix-instruction kernel, which runs the same tile as +`_lds128` through a different pipe. Nothing selects a shape from the kernel +side: the launcher's `tiled` / `variant` argument decides, and +`rdna2_gemm_i8_host_tuned` / `rdna2_sgemm_f32_host_tuned` pass the shape that +matches the device that is actually present. `tiled == 5` is the one case that +the host launcher always launches and the *kernel* decides on: `RDNA2_WMMA_IU8` +is a device-compilation constant, so it is 0 in host code, and gating the launch +on it there would silently select the dp4a kernel instead. + +## Quantized data contract + +The int8 entry points take packed int8 (`const int*` A and Bt, four int8 per word) +and return `int` C. Nothing in that path converts a quantized value to a floating +one, so weights loaded from a quantized file are used byte-for-byte as they sit in +memory: + +- No stored quantized value is ever converted. All ten int8 kernels - the nine + GEMM/GEMV shapes plus `rdna2_i8_row_sums` - contain **zero `v_cvt` of any + kind** and issue only `v_dot4_*` (dp4a) or `v_wmma_i32_16x16x16_iu8` with + integer accumulators; none contains `v_fmac_f32` or `v_mul_f32`. The single + conversion in the family is one `v_cvt_f32_i32` in `rdna2_gemm_i8_epilogue`, + which converts the finished int32 dot product once per output element. +- Semantics are signed int8. On gfx12 the signed form is expressed through the + `neg_lo:[1,1,0]` modifier on `v_wmma_i32_16x16x16_iu8` (`v_dot4_i32_iu8` there is + the unsigned form; see *dp4a on gfx12*). +- Exactness is tested, not assumed: every int8 variant (tiled 0-5) is compared + against a 64-bit integer reference on inputs packed with `-128`/`+127`, with + `|sum|` up to 1.33e8 - eight times past 2^24, where any f32 stage would have to + round - and matches bit-for-bit. The host input buffers are also memcmp'd after + the call to confirm the launcher never rewrites the caller's quants. +- No arch selector can route int8 onto a floating path: `rdna2_tiled_for_arch` + returns 1, 3 or 5, all integer, and an unknown GPU gets 1. + +The library applies the format's scale and zero point itself, so a loaded +quantized tensor goes straight into `rdna2_gemm_i8_scale_host`: +`out[m][n] = (acc[m][n] - zp_b * sum_a[m]) * sa[m] * sb[n]`. The correction +`-zp_b * sum_a[m]` is the exact integer form of B's asymmetric storage (a stored +value `q` represents `q - zp_b`), is computed in 64-bit on the int32 accumulator, +and skips the row-sum pass when `zp_b == 0`. This is not dequantization of the +stored data: the weights stay int8 and are never rewritten - the test memcmp's +the caller's buffers after the call - and the only float produced per output is +one finished dot product. + +4-bit formats (Q4_0 / Q4_K and similar) are still not handled: dp4a consumes whole +int8 bytes, so they would need an in-kernel nibble unpack, again integer-only, +before this path could take them. + +## LDS staging + +The thread-per-cell int8 GEMM issues 8 packed global loads per k step per +thread; over a 64x64 tile that is 2048 words of traffic for 128 words of data. +The LDS kernels stage each `RDNA2_LDS_I8_KSTEP` (16 words) chunk cooperatively +and then reuse it from LDS: + +- int8: two tiles of 64 x 17 words (stride 17 keeps rows off the same banks) = + **8704 bytes** per block. Staging: 256 threads x 4 words (16-byte loads where + `ncols % 4 == 0`), or 128 threads x 8 words for the 8x4 variant. +- f32: two tiles of 64 x 9 floats = **4608 bytes**. Staging: 256 threads x 2 + floats. +- f32, 16-deep K chunk (`rdna2_sgemm_f32_lds16`): two tiles of 64 x 17 floats = + **8704 bytes**. 256 threads stage four floats each, one 16-byte vector load + per thread per chunk. +- int8, 128x128 tile: two tiles of 128 x 17 words = **17408 bytes**. The same + 256 threads now stage 8 words each per chunk; the stride stays 17, so the + bank-conflict argument is unchanged. `rdna2_dp4a_gemm_i8_wmma` reuses this +exact staging unchanged: its `RDNA2_WMMA_I8_KSTEP_WORDS` (4 words per lane per +16-byte K step) plus the `2 * (lane >> 4)` column-group offset is what makes one +LDS tile readable as the 16x16x16 A and B fragments, so the two 128-tile +kernels differ only in what consumes the staged data. +- f32, 128x128 tile: two tiles of 128 x 9 floats = **9216 bytes**. + +Measured resource use on gfx1201 (`--cuda-device-only -S`, `-O2`) is 59 VGPRs / +8704 B for the 64x64 8x4 kernel, 93 VGPRs / 17408 B for the 128x128 dp4a +kernel and 96 VGPRs / 17408 B for the matrix-instruction kernel, with no +`scratch_` spills and no private segment in any of them. That register/LDS step +is what decides dp4a throughput: with 64 KiB of LDS per CU the 128x128 tile +admits 3 blocks where the 64x64 tile admits 7, and dp4a on this part is already +LDS-throughput bound, so the extra arithmetic intensity of the 8x8 blocking does +not pay for the lost residency (see *Performance*). The matrix instruction is +the exception: it does the same 3-block residency but needs 16x fewer issued +instructions for the same tile, which is why the same 17408 B of LDS is no +longer the limit there. + +Rows and columns past the matrix edge, and words past `K4`, are staged as zero, +so the compute loop needs no bounds checks; `__syncthreads()` before the first +read and after the last read is what keeps the single-buffered tile legal. + +## Launchers + +All launchers return `0` on success, `1` on a HIP error, `2` on a bad argument, +and free every allocation on the error path. + +- `rdna2_gemv_i8_host(y, A, x, rows, k4)`: `y[rows] = A[rows x k4] . x[k4]`, packed + int8. Any `k4` (including `k4 % 4 != 0`) and any host buffer alignment are + accepted; the kernel copies into an aligned `hipMalloc` staging buffer and the + remainder is handled by the scalar tail. +- `rdna2_gemv_i8_host_variant(y, A, x, rows, k4, variant)`: forces `variant` 0 + (one thread per row) or 1 (one wave per row). `rdna2_gemv_i8_host` picks 1 from + `k4 >= RDNA2_GEMV_WAVE_MIN_K4` (8) and 0 below it; `variant` 1 also falls back + to the thread-per-row kernel if the wave size cannot be queried, because the + wave kernel derives rows-per-block from `blockDim/warpSize` and a host guess + would mis-size the grid. +- `rdna2_gemm_i8_host(C, A, Bt, M, N, K4, tiled)`: `C[M x N] = A[M x K4] * Bt[N x K4]^T`, + packed int8. `tiled` selects 0 = naive, 1 = 4x4 tiled on global loads, 2 = + 4x4 with LDS staging, 3 = 8x4 with LDS staging, 4 = 128x128 tile with 8x8 + dp4a blocking, 5 = the same 128x128 tile driven by the 16x16x16 matrix + instruction. 4 and 5 share a grid and a launch configuration; on a device + without the instruction 5 runs the emulation described above (correct, slow), + so the choice is safe but only worth making on gfx12. `M` and `N` need no + multiple of the tile: edge threads read + `0` and only store in-range elements, and the last K chunk is zero-padded. +- `rdna2_gemm_i8_scale_host(C, A, Bt, M, N, K4, tiled, sa, sb, zp_b)` and + `rdna2_gemm_i8_scale_host_tuned(...)`: the same int8 GEMM followed by the + format's scale and zero point, so a loaded quantized tensor goes in exactly as + stored and the caller never expands it. `sa` scales rows of `A`, `sb` scales + columns of `Bt` (either may be `NULL` for 1.0), and `zp_b` is B's zero point + (`0` for symmetric weights, which skips the row-sum pass entirely). The result + is f32 - `out[m][n] = (acc[m][n] - zp_b*sum_a[m]) * sa[m] * sb[n]` - because a + GEMM result feeds the next layer's float path; no stored weight is converted. + `tiled` selects the same shapes and grids as `rdna2_gemm_i8_host`. +- `rdna2_sgemv_f32_host(y, A, x, rows, K)` and + `rdna2_sgemv_f32_host_variant(..., variant)`: f32 GEMV, `0` = one thread per + row, `1` = one wave per row. `rdna2_sgemv_f32_host` picks 1 from + `K >= RDNA2_SGEMV_WAVE_MIN_K` (256) and 0 below it; `variant` 1 also falls back + to thread-per-row when the wave size cannot be queried, for the same grid-size + reason as the int8 GEMV. +- `rdna2_sgemm_f32_host(C, A, Bt, M, N, K)`: f32 version of the tiled GEMM. + `rdna2_sgemm_f32_host_v(..., variant)` selects 0 = global-load 4x4 tiling, + 1 = 64x64 LDS staging, 2 = 128x128 LDS staging, 3 = 64x64 LDS staging with a + 16-deep K chunk (the default). +- `rdna2_gemm_i8_host_tuned` / `rdna2_sgemm_f32_host_tuned`: same arguments, + except that the shape is picked from the device that is present + (`rdna2_tiled_for_arch` / `rdna2_sgemm_variant_for_arch`): gfx1030 and gfx1031 + get 3 / 3, gfx1200 and gfx1201 get 5 / 3, anything else gets the generic 1 / 1 + path. + Measured on a 9070 XT, the 128x128 dp4a int8 tile never won, and the 128x128 + f32 tile only wins while its operand panels stay cache-resident, so `variant 2` + is opt-in rather than an arch default. The 16-deep K chunk (variant 3) won + everywhere it was tried, so it is the f32 default on all four arch codes. + Without a device both entry points fall through to the CPU reference. +- `rdna2_dot_u8_host`, `rdna2_dot_f16_host`, `rdna2_prefix_host`. +- `rdna2_device_info(name, len, warp_size, arch_ok)`: reports the device name and + wave size, plus a target code: `RDNA2_ARCH_GFX1031` (1), `RDNA2_ARCH_GFX1030` + (2), `RDNA2_ARCH_GFX1201` (3), `RDNA2_ARCH_GFX1200` (4), or + `RDNA2_ARCH_UNKNOWN` (0) for any other AMD GPU, which still runs. + +When HIP reports no device, the GEMV/GEMM/SGEMM/dot launchers compute the same +result with `rdna2_gemm_ref.h`, so callers work on machines without a GPU. + +## Build + + make test # build and run the host test + make HIP_ARCH=gfx1030 # build the object for gfx1030 + make HIP_ARCH=gfx1201 test # RX 9070 / 9070 XT (RDNA4) + make HIP_ARCH=gfx1200 test # RX 9060 XT (RDNA4) + make multi # fat binary: gfx1030 gfx1031 gfx1200 gfx1201 + +or directly: + + clang++ -x hip --offload-arch=gfx1031 -O2 -c rdna2_kernels.hip -o rdna2_kernels.o + clang++ -x hip --offload-arch=gfx1201 --offload-arch=gfx1200 -O2 -c rdna2_kernels.hip -o rdna2_kernels.o + +Use the AMD ROCm LLVM clang (or `amdclang++`) with the HIP headers and +`libamdhip64`. Pass `--offload-arch=` once per target to build a fat binary; +gfx1200 and gfx1201 are separate targets and a binary built for one will not be +picked up by the other, so a 9070 XT needs gfx1201 (or both) in the offload set. +`HIP_ARCH`/`ARCHS` changes do not invalidate existing objects, so `make clean` +between architectures or use `make multi`. + +### dp4a on gfx12 + +gfx12 dropped the `dot1-insts` feature that `__builtin_amdgcn_sdot4` needs, so on +gfx1200/gfx1201 the builtin does not compile: clang 23 fails with +`'__builtin_amdgcn_sdot4' needs target feature dot1-insts`. The hardware +instruction is still there under +the older mnemonic, so `rdna2_dot4_i8_impl` emits `v_dot4_i32_i8` directly for +those two targets and keeps the builtin for gfx10.3. Two things are worth +knowing, both checked on a real gfx1201 card: + +- `v_dot4_i32_i8` on gfx12 computes **signed** results, byte for byte identical + to the scalar reference on six adversarial vectors including `0xffffffff` and + `0x00000080`. +- `v_dot4_i32_iu8`, the renamed dot10 form, assembles on gfx12 but computes + **unsigned x unsigned**, so it must not be substituted for the signed mnemonic + in a signed GEMM. It is only correct for the u8 dot kernel, which is what + `rdna2_dp4a_dot_u8` uses. + +`-DRDNA2_NO_DOT1=1` drops the asm and takes the other branch in +`rdna2_gemm_ref.h`: each int8 operand is sign-extended to int32 and LLVM forms +the signed dot itself (`v_dot4_i32_iu8 ... neg_lo:[1,1,0]` on gfx12, `v_dot4c_i32_i8` +on gfx10.3). Same results, more instructions; useful only if a toolchain rejects +the mnemonics. That build passed the whole suite as it stood when it was run, +before the matrix-instruction tests existed. + +### Matrix instruction on gfx12 (`tiled == 5`) + +gfx1200/gfx1201 have `v_wmma_i32_16x16x16_iu8`: a 16x16x16 integer matrix +multiply-accumulate per instruction, which for int8 is 4096 MACs where a dp4a is +4. `rdna2_dp4a_gemm_i8_wmma` is the 128x128 tile with that instruction, and it +is the int8 default for those two parts. Three things matter here; the first two +were measured on the card rather than taken from the ISA description. + +- The operands are **signed**, so `RDNA2_WMMA_IU8` sets `negA` and `negB`. With + those flags the dp4a `lds128` kernel and this one agree element for element on + 1024^3, 2048x2049x2048 and 4096^3 (16.7M outputs at the largest size). +- The lane-to-fragment mapping is fixed by the hardware and is not the one a + natural reading of the ISA gives. On this part, lane `t` byte `e` of A carries + `A[t % 16][(t / 16) * 8 + e]`, of B `B[(t / 16) * 8 + e][t % 16]`, and + accumulator slot `l` of lane `t` carries `D[(t / 16) * 8 + l][t % 16]`. The + host test checks exactly this: it runs one instruction's worth of + fragments through `sim_wmma_i8_wave_step` and compares all 256 output slots + against a per-slot dot product, plus a nonzero-accumulator pass to catch a + kernel that overwrites instead of accumulating. +- The helper that performs one step, `rdna2_wmma_iu8_step`, has three branches. + On gfx12 device passes (`RDNA2_WMMA_IU8`) it is the instruction. On any other + *device* pass, i.e. gfx1030/gfx1031 and gfx11, there is no matrix unit, so the + same result is built from explicit byte dots, and each slot's operand bytes are + gathered with `__shfl` out of the two lanes that hold that row's fragment (the + low half of K from lead lane `d`, the high half from `d + 16`). That branch is + correct and is never selected by a dispatcher; no timing is claimed for it. + In the plain C++ pass (the CPU test harness, and the host half of every HIP + build) no kernel can run at all, so the body is a stub that returns `acc` + unchanged: it exists only to keep the kernel defined so the unconditional + launch site links. + +That last point is the one that bit this tree once. `RDNA2_WMMA_IU8` is 0 in the +host pass, so a host-side `if (RDNA2_WMMA_IU8)` around the launch compiled out the +real kernel and quietly ran `_lds128` instead - a passing test that never touched +the matrix instruction. The launch is therefore unconditional and only the +*instruction* is arch-gated, which is also why `tiled == 5` on a non-gfx12 device +is safe but pointless. + +## Performance + +Measured on the RX 9070 XT in this machine (gfx1201, HIP 7.16, clang 23, +`-O2`). Times are kernel-only: allocation and transfers are hoisted out of the +loop, so they are not the launcher's end-to-end cost. Best of 3 runs x 10-20 +iterations each. + +int8, TFLOP/s equivalent (2 flops per MAC), `K = 1024` unless noted: + +| M = N | naive | tiled 2 (4x4 LDS) | **tiled 3 (8x4 LDS)** | tiled 4 (128x128 dp4a) | **tiled 5 (128x128 matrix)** | +|---|---|---|---|---|---| +| 1024 | 0.74 | 33.6 | 38.2 | 30.1 | **91.3** | +| 2048 | 0.63 | 34.0 | 46.1 | 34.7 | **134.0** | +| 4096 | 0.59 | 34.7 | 46.4 | 41.6 | **125.0** | +| 2048, K = 4096 | 0.42 | 35.5 | **44.0** | 40.9 | - | + +The three tiled-5 figures were timed in the same harness as tiled 4 (same grid, +launch and buffers, only the instruction differs): 29.4 / 40.6 / 40.1 against +91.3 / 134.0 / 125.0, i.e. 3.1x at 1024^3, 3.3x at 2048^2 and 3.1x at 4096^3 +over the 128x128 dp4a kernel. Against the 64x64 8x4 dp4a kernel, which is the +previous int8 default, the gain is 2.4x, 2.9x and 2.7x. + +f32, TFLOP/s, `K = 1024` unless noted: + +| M = N | v0 (global 4x4) | v1 (64x64 LDS) | v2 (128x128 LDS) | +|---|---|---|---| +| 1024 | 2.94 | **8.11** | 7.51 | +| 2048 | - | 8.53 | **8.85** | +| 4096 | - | 8.01 | **9.47** | +| 6144 | - | 7.84 | **9.48** | +| 8192, K = 512 | - | 7.81 | **9.55** | +| 2048, K = 2048 | - | 6.39 | **8.14** | +| 4096, K = 2048 | - | 8.41 | **9.37** | +| 2048, K = 4096 | - | **7.34** | 3.86 | +| 2048, K = 8192 | - | **7.25** | 4.14 | +| 4096, K = 4096 | - | **7.60** | 6.62 | +| 1024, K = 4096 | - | 4.84 | **6.46** | + +The v0/v1/v2 columns are the shapes this tree started with. Variant 3, the +64x64 tile with a 16-deep K chunk, was added later and measured against v1 at +1024^3, 1000^3, 2048^2 x K=2048/3072/4096/8192, 2304^2, 3584^2 and 4096^3: it +won at every one, by 3-42%, the margin shrinking as the problem grows. It is the +f32 default on all four arch codes for that reason. The v3 rows above come from +that separate run; the same-harness comparison of all four shapes is below. + +All four f32 shapes in one run (same grid, launch and buffers, only the kernel +body differs; TMAC/s, with TFLOP/s in brackets): + +| M = N = K | v0 (global 4x4) | v1 (64x64, KSTEP 8) | v2 (128x128) | **v3 (64x64, KSTEP 16)** | +|---|---|---|---|---| +| 1024 | 1.48 (2.96) | 4.06 (8.12) | 3.92 (7.84) | **4.29 (8.59)** | +| 2048 | 0.48 (0.96) | 3.05 (6.10) | 4.10 (8.19) | **4.23 (8.46)** | +| 4096 | 0.10 (0.20) | 4.29 (8.58) | 3.55 (7.10) | **4.46 (8.93)** | + +What that says: + +- Among the dp4a shapes, the 64x64 8x4 LDS kernel is the right int8 default on + this part: it beats the 128x128 dp4a tile at every size measured, by 8% at + 2048^2 x K=4096, 11% at 4096^2 x K=1024 and 27% at 1024^3. The 128 tile still + has the better arithmetic intensity on paper; in practice its 93 VGPRs and + 17408 B of LDS per block cost more residency than the extra reuse buys. +- The matrix instruction changes that conclusion without changing the tile: same + 128x128 tile, same 96 VGPR / 17408 B residency, 3.1-3.3x the throughput, so it + is the int8 default on gfx12. It is also the one int8 number that does not sit + under the dp4a ceiling below - the matrix unit is a separate pipe, so 134 + TFLOP/s equivalent is not 134 TFLOP/s of dp4a issue. +- The f32 128x128 tile is genuinely faster while the two operand panels fit in + cache (up to +22% at 8192^2 x K=512, +21% at 6144^2 x K=1024, +18% at + 4096^2 x K=1024) and falls off a cliff once they do not: 2048^2 with K = 4096 or + 8192 drops to 3.9-4.1 TFLOP/s against 7.3 for the 64x64 shape, and 1024^3 also + loses. That is why `variant 2` is opt-in rather than an arch default. +- For scale, the ceilings below are measured on this card by a probe kernel + that does nothing but issue the instruction: dp4a 43.86 TMAC/s (87.7 int8 + TOP/s) and f32 fma 17.76 TMAC/s (35.5 TFLOP/s). `hipGetDeviceProperties` + reports 32 CUs at 2.40 GHz, which would put the f32 issue rate at about 19.7 + TFLOPS and the dp4a rate at 39.3 TMAC/s; the probe beats both, so the reported + CU count is half the physical one and the derived figures in this paragraph + used to understate every ratio by 2x. Against the measured ceilings the best + dp4a shape above (46.4 TFLOP/s equivalent, i.e. 23.2 TMAC/s) is 53% of the + dp4a issue rate and the best f32 shape (9.55 TFLOP/s, i.e. 4.78 TMAC/s) is 27% + of the 35.5 TFLOP/s f32 probe. The tiled-5 figures are not comparable to either ceiling: the + matrix instruction replaces 4096 of the MACs a dp4a does 4 at a time. + +### Where the time goes + +A `trace`-style decomposition, all kernel-only and all in one harness, to check +whether the gaps above are memory, staging or inner-loop limits. Each GEMM row +below is the same kernel compiled three ways: `full`, `staging+sync` (the same +global loads and `__syncthreads`, math removed) and `math-only` (LDS reads and +fma kept, staging replaced by zeros). Dimensions 4096^3 unless noted. + +| Kernel | full us | staging+sync us | math-only us | What it says | +|---|---|---|---|---| +| int8 tiled 5 (matrix), 4096^3 | 1219.0 | 1232.7 | 753.6 | staging >= full: the staged traffic alone costs as much as the whole kernel | +| int8 tiled 4 (128x128 dp4a) | 3436.2 | 1186.0 | 3164.0 | staging hidden, inner loop is 92% of full: this kernel is math-bound | +| f32 v3, 1024^3 | 255.1 | 83.7 | 233.8 | inner loop 92% of full | +| f32 v3, 2048^3 | 2075.9 | 620.2 | 1702.5 | inner loop 82% of full | +| f32 v3, 4096^3 | 15745.4 | 6903.4 | 13429.2 | inner loop 85% of full | + +The f32 64x64 tile consumes 8.59 GB of panel reads at 4096^3, which at 15745 us +is 546 GB/s - close enough to the 558 GB/s copy ceiling to look DRAM bound. The +staging-only row disproves that: the same traffic alone takes 6903 us (1244 GB/s +effective, because consecutive tiles in a block row reuse the panels in L2), so +most of those bytes never reach DRAM and the kernel is not transfer limited. + +The inner loops, isolated from staging entirely (fixed LDS tile, no global +access, 20000 iterations, 64/256/1024 blocks): + +| Inner loop | 64 blocks | 256 blocks | 1024 blocks | +|---|---|---|---| +| f32 4x4 block, 16-deep K chunk (v3) | 4.77 TMAC/s | 4.93 | 4.66 | +| f32 4x4 block, unroll-4 variant | 4.81 TMAC/s | 4.91 | 4.64 | +| f32 8x8 block (the 128x128 tile, v2) | 5.71 TMAC/s | 5.79 | 5.58 | + +Against the register-only fma probe of 17.76 TMAC/s that is 27% for the 4x4 +block and 32% for the 8x8 block. Doubling the fma-per-LDS-read ratio from 2 to 4 +buys 20%, and unrolling by 4 buys nothing, so this is not LDS bandwidth, not +LDS byte traffic and not instruction level parallelism that limits the f32 inner +loop. The same numbers explain the v2 anomaly from the other direction: its +inner loop is the fastest of the three (5.71) yet the whole kernel is the +slowest at 4096^3 (3.55 TMAC/s), so what the 128x128 tile loses is residency - +94 VGPR and 9216 B of LDS per block leave too few blocks in flight to cover the +staging it does not overlap. + +Ceilings measured on the same card, for the numbers above to be read against: +copy 558.6 GB/s, HBM read 619.7 GB/s, LDS 8 streams 7644.1 GB/s, L2/Infinity +Cache read 7873.5 GB/s, dp4a issue 43.86 TMAC/s, f32 fma issue 17.76 TMAC/s. +The only int8 kernel near its ceiling is tiled 5, and it is near the *staging* +ceiling, not the ALU one; tiled 3 at 22.5 TMAC/s sits at 51% of the dp4a issue +rate, which the ISA explains (8 batched `ds_load_2addr_b32`, then counted +`v_dot4_i32_i8` with the LDS latency hidden behind them). + +Tried and rejected on measurement, one variable at a time, all reverted: + +- **Unroll the f32 K loop by 4.** 4.81 vs 4.77 TMAC/s isolated, 4.33 vs 4.46 at + 4096^3. No gain: the LDS latency was already covered, so more in-flight loads + only add registers. +- **`hipHostRegister` the operand panels.** Pinning costs 484 us once per + buffer, then copies 4 MB at 17 GB/s instead of 12.9 GB/s. End-to-end the + registration cost is never repaid at any size the host wrappers take. +- **A 128x128 tile for f32 (variant 2) as the default.** Faster inner loop, + slower kernel past L2-sized problems; see above. +- **A 64x128 tile with 8x4 per-thread blocking (32 accumulators, 61 VGPR, + 13056 B of LDS against v3's 38 / 8704).** Bit-identical to v3 at 1024^3 (0 + mismatches in 1048576 entries) and 3-5% faster at 2048^3 (1936 vs 1998 us), but + ~2% slower at both 1024^3 (257 vs 250) and 4096^3 (15774 vs 15420) in two + consecutive runs, so it cannot replace v3 as one default for every size. Its + only advantage is issue density in the inner loop - 6 `ds_load_2addr_b32` per + 16 `v_dual_fmac` against v3's 4 per 8 - and the larger LDS footprint and + 61-VGPR register bill cancel it. A 17-shape interleaved A/B (128^2 to + 8192x2048, K from 128 to 8192) then settled the dispatch question: the tile + never wins outside the measured run-to-run spread, and it loses 75-115% at + 128^2/256^2 - its 128-wide tile halves the block count, and the GPU already has + too few blocks to fill - plus 6% at 512^2 and 21% at 768^2. A size-dependent + dispatch is therefore not worth building; variant 3 stays the only f32 path. + +The host wrappers are dominated by transfer and not by launch or allocation: +`hipMalloc`+`hipFree` of 1 MB is 1.6-2.1 us, a full 1024^3 int8 trio allocates in +5.2 us, but one 4 MB host-to-device copy is 311.7 us (12.9 GB/s) and a +4 MB H2D+D2H round trip 820 us. That floor is in the platform, not in this code, +and it is why `rdna2_gemv_i8_host` takes 558 us end to end for a 4096x1024 GEMV +whose kernel is 4.4 us, `rdna2_sgemv_f32_host` 1615 us for a 12.2 us kernel, +`rdna2_sgemm_f32_host` 1571 us at 1024^3 for a 251 us kernel, and +`rdna2_gemm_i8_host` 1035 us at 1024^3 tiled 5 for a 19.5 us kernel. + +## Verification status + +| Check | Status | +|---|---| +| Compiles for gfx1031, gfx1030, gfx1200 and gfx1201, `-Wall -Wextra` clean (0 warnings) | Run, passed (clang 23 / HIP 7.16) | +| Host test, 141 checks on a device / 132 on the CPU fallback path: GEMV int8 k4=8, k4=5, misaligned k4=5, wave variants; GEMV f32 K=128, 5 and 133 for both variants; GEMM 70x67xK4=10 naive/tiled/LDS/LDS8 plus grid simulations; LDS paths also at 35x33xK4=40 so a partial last chunk and edge tiles are staged; 135x130xK4=40 so both dimensions span two 128 tiles and the last chunk is 8 of 16, run for the 128x128 dp4a kernel and for the matrix-instruction kernel; a single-instruction fragment check (one step of `rdna2_wmma_i8_wave_step`, all 256 output slots against a per-slot dot product, plus a nonzero-accumulator pass); SGEMM 67x70xK=19 for both RDNA2 variants plus simulation, and 135x130xK=19 for variant 2; u8 dot n=1000; f16 dot n=500; DPP prefix; arch selectors; the `*_tuned` entry points; the quantized epilogue at 37x45xK4=12 (odd sizes, so partial 64 and 128 tiles) for tiled 0, 3, 4 and 5 with unit, row-only, column-only and both scales, plus `zp_b=8`, an exactness check on the zero-point correction, a memcmp of the caller's quants after the call, and the tuned entry point | Run, passed: 132 checks, 0 failures, `path=CPU fallback` (plain C++ incl. the file's host half) | +| Same suite on the **GPU** (`rdna2_kernels.hip` + host test linked with hipcc, device launches) | Run, passed on an RX 9070 XT (gfx1201, warp 32): `PASSED: 141 check(s), 0 failure(s), path=GPU` and `PASSED: 132 check(s), 0 failure(s), path=CPU fallback` with the device hidden (`HIP_VISIBLE_DEVICES=""`). The 9 extra device checks are the `rdna2_sgemv_f32_host` default-variant ones (three per K, times K = 128, 5 and 133): they only exist when a device is present, because on the fallback path the default launcher is the reference itself | +| Matrix-instruction GEMM (`tiled == 5`) against dp4a in ISA and in numbers | Run, passed: `rdna2_dp4a_gemm_i8_wmma` contains 8 `v_wmma_i32_16x16x16_iu8` and no dp4a; the `_lds128` kernel contains zero `v_wmma_*`. Cross-checked against `_lds128` at 1024^3, 2048x2049x2048 and 4096^3: **0 mismatches**, 16.7M outputs at the largest size | +| ISA of all 19 kernels for gfx1201 (`--cuda-device-only -S -O2`) | Run, passed: `ds_load_2addr_b32` / `ds_store_2addr_b32` in every LDS kernel, signed dp4a (`v_dot4_i32_i8`, 137 occurrences across the int8 kernels), `v_dot2_f32_f16` (1, from `__builtin_amdgcn_fdot2`), `v_wmma_i32_16x16x16_iu8` (8, all in the matrix kernel), `v_add_nc_u32_dpp` (4), **no `scratch_` spills and no private segment anywhere**. VGPR / LDS per kernel: gemv_i8 16/0, gemv_i8_wave 12/0, sgemv_f32 14/0, sgemv_f32_wave 11/0, gemm_i8_naive 7/0, gemm_i8_tile 43/0, gemm_i8_lds 43/8704, gemm_i8_lds8 59/8704, gemm_i8_lds128 93/17408, gemm_i8_wmma 96/17408, sgemm_f32 43/0, sgemm_f32_lds 35/4608, sgemm_f32_lds16 38/8704, sgemm_f32_lds128 94/9216, dot2_f16 8/0, dp4a_dot_u8 8/0, dpp_prefix 4/0, i8_row_sums 7/0, gemm_i8_epilogue 8/0 | +| No float conversion of stored quants | Run, passed: all ten int8 kernels contain **0 `v_cvt` of any kind** and **0 f32 arithmetic** (only `v_dot4_*` / `v_wmma_i32_16x16x16_iu8`); the only conversion in the family is one `v_cvt_f32_i32` in `rdna2_gemm_i8_epilogue`, on the finished accumulator. Adversarial inputs packed with `-128`/`+127` give `|sum|` up to 1.33e8 - eight times past 2^24, where any f32 stage would have to round - and all six `tiled` variants match a 64-bit integer reference bit-for-bit, with the caller's packed buffers byte-identical after the call | +| Signed dp4a semantics on gfx12 | Run, passed on the card: `v_dot4_i32_i8` matches the scalar reference on 6 vectors, `v_dot4_i32_iu8` does not (it is unsigned). gfx12 has no `sdot4` builtin; see *dp4a on gfx12*. The `-DRDNA2_NO_DOT1=1` build (no explicit asm) also passed the whole suite at the time it was run, which was before the matrix-instruction tests existed (78 checks then) | +| Device-vs-CPU correctness of every kernel and shape | Run, passed: the same test binary that the table above calls, all 141 checks on the device, including 135x130 tiles that straddle two 128-wide tiles | +| Performance of the five int8 shapes and four f32 shapes | Run, measured, see *Performance*; kernel-only, single card, one input distribution. The int8 shapes and the four f32 shapes were each timed in one harness, so both comparisons are like for like. Re-measured with staging / math phase splits and isolated inner-loop probes: see *Where the time goes* | +| Correctness of the timed shapes (1024^3 int8, tiled 3, 4 and 5, full 1024x1024 output vs `rdna2_gemm_i8_ref`, plus tiled 4 vs tiled 5 at 2048x2049x2048 and 4096^3) | Run, passed: exact match everywhere, 0 mismatches | + +Coverage limits: everything above is one RDNA4 card (gfx1201). The gfx1030 and +gfx1031 paths are compiled and host-simulated but were not executed on RDNA2 +hardware, so the dp4a-in-LDS timing on RDNA2 is the same code measured only on +RDNA4. The shuffle-based emulation of the matrix instruction is compiled (its +branch is visible in a fresh gfx1031 dump: 0 `v_wmma_*`, 167 `v_dot4c_i32_i8`, +83 `ds_bpermute_b32` of which 64 are the emulation's lane gathers, 0 warnings) +and never selected by a dispatcher, but it is not covered by a runtime device +test. Wave-level behaviour is covered on the device; the 141 +checks are functional, not a stress or fuzz suite. + +## Not included + +- Full OpenBLAS (BLAS levels 1-3, LAPACK) on the GPU. OpenBLAS has no GPU backend; + this directory provides GEMM/GEMV/dot kernels only. diff --git a/gpu/rdna2/rdna2_gemm_ref.h b/gpu/rdna2/rdna2_gemm_ref.h new file mode 100644 index 0000000000..d5f8c038fa --- /dev/null +++ b/gpu/rdna2/rdna2_gemm_ref.h @@ -0,0 +1,219 @@ +// CPU references for the RDNA2 kernels. Plain C++ (no HIP), so every +// reference can be built and tested on a machine without a GPU. The launchers +// in rdna2_kernels.hip use these as the fallback when no device is present, +// and the test uses them as the correctness oracle. +// +// Packed int8 convention (matches v_dot4_*): four int8 values per int32, +// value i at bits [8*i, 8*i+8), i.e. little-endian byte order. +#ifndef RDNA2_GEMM_REF_H +#define RDNA2_GEMM_REF_H + +#include +#include +#include + +// Ref functions that device code calls must be __host__ __device__. Under a +// plain host compiler the macro expands to nothing. +#ifndef RDNA2_HD +#if defined(__HIP__) || defined(__HIPCC__) || defined(__CUDACC__) +#define RDNA2_HD __host__ __device__ +#else +#define RDNA2_HD +#endif +#endif + +// --------------------------------------------------------------------------- +// dp4a int8 / u8 +// --------------------------------------------------------------------------- +RDNA2_HD static inline int32_t rdna2_dot4_i8_ref(int32_t a, int32_t b, int32_t acc) { + for (int i = 0; i < 4; ++i) { + int8_t x = static_cast((static_cast(a) >> (8 * i)) & 0xffu); + int8_t y = static_cast((static_cast(b) >> (8 * i)) & 0xffu); + uint32_t p = static_cast(static_cast(x) * static_cast(y)); + acc = static_cast(static_cast(acc) + p); + } + return acc; +} + +static inline uint32_t rdna2_dot4_u8_ref(uint32_t a, uint32_t b, uint32_t acc) { + for (int i = 0; i < 4; ++i) { + uint32_t x = (a >> (8 * i)) & 0xffu; + uint32_t y = (b >> (8 * i)) & 0xffu; + acc += x * y; + } + return acc; +} + +// The dot4 primitive used by the kernel bodies. On the device this is the real +// v_dot4c_i32_i8 (dp4a) instruction, independent of what the optimiser can +// pattern-match; on the host it is the bit-exact scalar reference above, so the +// same kernel body still runs on a CPU-only machine. +RDNA2_HD static inline int32_t rdna2_dot4_i8_impl(int32_t a, int32_t b, int32_t acc) { +#if defined(__HIP_DEVICE_COMPILE__) && !defined(RDNA2_NO_DOT1) +#if defined(__gfx1200__) || defined(__gfx1201__) + // gfx12 (RDNA4) has no dot1-insts, so `__builtin_amdgcn_sdot4` is rejected + // ("needs target feature dot1-insts"). The dp4a hardware is still there: the + // legacy v_dot4_i32_i8 mnemonic assembles and computes exact signed results + // (checked against the scalar reference on a gfx1201 card). Note that + // v_dot4_i32_iu8 is *not* interchangeable: on gfx12 it is the unsigned form. + asm volatile("v_dot4_i32_i8 %0, %1, %2, %0" : "+v"(acc) : "v"(a), "v"(b)); + return acc; +#else + return __builtin_amdgcn_sdot4(a, b, acc, false); +#endif +#else + // Either a host build or -DRDNA2_NO_DOT1=1 (a target whose dot1-insts feature + // is missing, e.g. an older gfx12 toolchain): same arithmetic, four scalar + // multiplies. Both branches are compared in the host test. + return rdna2_dot4_i8_ref(a, b, acc); +#endif +} + +// --------------------------------------------------------------------------- +// GEMV: y[M] = A[M x K4] . x[K4], packed int8 +// --------------------------------------------------------------------------- +static inline void rdna2_gemv_i8_ref(int32_t* y, const int32_t* A, const int32_t* x, + int rows, int K4) { + for (int r = 0; r < rows; ++r) { + int32_t acc = 0; + for (int k = 0; k < K4; ++k) acc = rdna2_dot4_i8_ref(A[(size_t)r * K4 + k], x[k], acc); + y[r] = acc; + } +} + +// --------------------------------------------------------------------------- +// GEMV: y[M] = A[M x K] . x[K], float +// --------------------------------------------------------------------------- +static inline void rdna2_sgemv_f32_ref(float* y, const float* A, const float* x, + int rows, int K) { + for (int r = 0; r < rows; ++r) { + // The reference accumulates in double and rounds once, so it does not + // share the kernel's four-way partial-sum split; the test compares with a + // tolerance rather than bit-for-bit. + double acc = 0.0; + for (int k = 0; k < K; ++k) acc += (double)A[(size_t)r * K + k] * (double)x[k]; + y[r] = (float)acc; + } +} + +// --------------------------------------------------------------------------- +// GEMM: C[M x N] = A[M x K4] * Bt[N x K4]^T, packed int8 +// --------------------------------------------------------------------------- +static inline void rdna2_gemm_i8_ref(int32_t* C, const int32_t* A, const int32_t* Bt, + int M, int N, int K4) { + for (int m = 0; m < M; ++m) { + for (int n = 0; n < N; ++n) { + const int32_t* a = A + static_cast(m) * K4; + const int32_t* b = Bt + static_cast(n) * K4; + int32_t acc = 0; + for (int k = 0; k < K4; ++k) acc = rdna2_dot4_i8_ref(a[k], b[k], acc); + C[static_cast(m) * N + n] = acc; + } + } +} + +// --------------------------------------------------------------------------- +// GEMM with the quantized epilogue applied on the CPU: +// C[m][n] = (acc[m][n] - zp_b * rowsum_a[m]) * sa[m] * sb[n] +// rowsum_a[m] is the sum of the int8 activations in row m. That is the exact +// integer form of B's asymmetric storage - B's stored value at k represents +// (q - zp_b), so sum_k a*(q - zp_b) = acc - zp_b*sum_k a - and it never converts +// a stored weight: only the finished dot product becomes float. sa / sb may be +// NULL (unit scale). The correction and the product are computed in 64-bit so +// the reference cannot overflow before its single float conversion. +// --------------------------------------------------------------------------- +static inline void rdna2_gemm_i8_scale_ref(float* C, const int32_t* A, const int32_t* Bt, + int M, int N, int K4, const float* sa, + const float* sb, int zp_b) { + for (int m = 0; m < M; ++m) { + const int32_t* a = A + static_cast(m) * K4; + int64_t rowsum = 0; + if (zp_b != 0) { + for (int k = 0; k < K4; ++k) { + for (int b = 0; b < 4; ++b) { + rowsum += static_cast((static_cast(a[k]) >> (8 * b)) & 0xffu); + } + } + } + const float s_a = sa ? sa[m] : 1.0f; + for (int n = 0; n < N; ++n) { + const int32_t* b = Bt + static_cast(n) * K4; + int32_t acc = 0; + for (int k = 0; k < K4; ++k) acc = rdna2_dot4_i8_ref(a[k], b[k], acc); + const int64_t corr = static_cast(acc) - static_cast(zp_b) * rowsum; + C[static_cast(m) * N + n] = + static_cast(corr) * s_a * (sb ? sb[n] : 1.0f); + } + } +} + +// --------------------------------------------------------------------------- +// SGEMM: C[M x N] = A[M x K] * Bt[N x K]^T, float +// --------------------------------------------------------------------------- +static inline void rdna2_sgemm_f32_ref(float* C, const float* A, const float* Bt, + int M, int N, int K) { + for (int m = 0; m < M; ++m) { + for (int n = 0; n < N; ++n) { + const float* a = A + static_cast(m) * K; + const float* b = Bt + static_cast(n) * K; + float acc = 0.0f; + for (int k = 0; k < K; ++k) acc += a[k] * b[k]; + C[static_cast(m) * N + n] = acc; + } + } +} + +// --------------------------------------------------------------------------- +// f16 dot (v_dot2c_f32_f16): acc = sum over i of a.xy * b.xy, f32 accumulate. +// Pairs are packed two _Float16 per int32, low half first. +// --------------------------------------------------------------------------- +#if defined(__FLT16_MAX__) +static inline void rdna2_unpack_half2(uint32_t v, _Float16* lo, _Float16* hi) { + uint16_t l = static_cast(v & 0xffffu); + uint16_t h = static_cast(v >> 16); + std::memcpy(lo, &l, sizeof(uint16_t)); + std::memcpy(hi, &h, sizeof(uint16_t)); +} + +static inline uint32_t rdna2_pack_half2(_Float16 lo, _Float16 hi) { + uint16_t l, h; + std::memcpy(&l, &lo, sizeof(uint16_t)); + std::memcpy(&h, &hi, sizeof(uint16_t)); + return static_cast(l) | (static_cast(h) << 16); +} + +// Reference matches the hardware order: acc += a.x*b.x then acc += a.y*b.y. +static inline float rdna2_dot_f16_ref(const uint32_t* a, const uint32_t* b, int n) { + float acc = 0.0f; + for (int i = 0; i < n; ++i) { + _Float16 ax, ay, bx, by; + rdna2_unpack_half2(a[i], &ax, &ay); + rdna2_unpack_half2(b[i], &bx, &by); + acc += static_cast(ax) * static_cast(bx); + acc += static_cast(ay) * static_cast(by); + } + return acc; +} +#endif + +// --------------------------------------------------------------------------- +// u8 dot: out = sum over i of dot4_u8(a[i], b[i]) +// --------------------------------------------------------------------------- +static inline uint32_t rdna2_dot_u8_ref(const uint32_t* a, const uint32_t* b, int n) { + uint32_t acc = 0; + for (int i = 0; i < n; ++i) acc = rdna2_dot4_u8_ref(a[i], b[i], acc); + return acc; +} + +// --------------------------------------------------------------------------- +// 32-lane inclusive prefix sum (matches rdna2_dpp_prefix on wave32). +// --------------------------------------------------------------------------- +static inline void rdna2_prefix_ref(int32_t* out, const int32_t* in, int n) { + int32_t run = 0; + for (int i = 0; i < n; ++i) { + run += in[i]; + out[i] = run; + } +} + +#endif // RDNA2_GEMM_REF_H diff --git a/gpu/rdna2/rdna2_kernels.hip b/gpu/rdna2/rdna2_kernels.hip new file mode 100644 index 0000000000..69f57ae192 --- /dev/null +++ b/gpu/rdna2/rdna2_kernels.hip @@ -0,0 +1,1367 @@ +// RDNA2 (gfx1030 / gfx1031) kernels, written directly against the GFX10.3 ISA, +// plus an RDNA4 (gfx1200 / gfx1201, RX 9060 XT / 9070 XT) tile shape that spends +// gfx12's larger register file on a 128x128 tile. +// Not a CUDA-to-HIP port: each kernel is shaped around one RDNA2 instruction +// group, and every one of them is checked against a plain-C++ reference +// (rdna2_gemm_ref.h) by test_rdna2_host.cpp. +// +// Instruction groups used: +// dot4 : v_dot4_i32_i8 / v_dot4c_i32_i8 / v_dot4_u32_u8 dp4a, int8 x4 +// dot2 : v_dot2_f32_f16 packed f16 dot +// dpp : v_add_nc_u32_dpp row_shr 16-lane wave scan +// ds_* : ds_read_b32 / ds_write_b32 LDS staging +// +// Build: clang++ -x hip --offload-arch=gfx1031 -O2 -c rdna2_kernels.hip +// clang++ -x hip --offload-arch=gfx1201 -O2 -c rdna2_kernels.hip (9070 XT) +// +// Return codes shared by all launchers: +// 0 = success, 1 = HIP/runtime error, 2 = invalid argument. + +#include +#include +#include + +#include "rdna2_gemm_ref.h" +#include "rdna2_kernels_math.h" + +#define RDNA2_OK 0 +#define RDNA2_ERR_HIP 1 +#define RDNA2_ERR_ARG 2 + +// The RDNA2_ARCH_* target codes live in rdna2_kernels_math.h, next to the tile +// shapes they select, so the host test uses the same numbers. + +// Four packed int32 = 16 bytes: one global_load_dwordx4 on the gfx1031 path. +typedef int int4_t __attribute__((ext_vector_type(4))); +// Two packed f16: the operand of v_dot2_f32_f16. +typedef _Float16 half2_t __attribute__((ext_vector_type(2))); +// WMMA operands and accumulators: V_WMMA_I32_16X16X16_IU8 takes two pairs of +// packed int32 (16 operand bytes per lane) and eight int32 accumulators. +typedef int int32x2_t __attribute__((ext_vector_type(2))); +typedef int int32x8_t __attribute__((ext_vector_type(8))); + +// Whether this compilation pass can emit V_WMMA_I32_16X16X16_IU8: only the +// device passes of gfx1200 / gfx1201. Note the host pass of the same build never +// has it, so this must not decide which kernel gets launched -- the launch site +// is unconditional and only the instruction inside the kernel is arch-gated. +#if defined(__HIP_DEVICE_COMPILE__) && (defined(__gfx1200__) || defined(__gfx1201__)) +#define RDNA2_WMMA_IU8 1 +#else +#define RDNA2_WMMA_IU8 0 +#endif + + +// The LDS tiling shape (RDNA2_TILE_MN, RDNA2_LDS_*_KSTEP/STRIDE) lives in +// rdna2_kernels_math.h so the host simulation uses the identical numbers. + +// --------------------------------------------------------------------------- +// Shared helpers +// --------------------------------------------------------------------------- +static inline int rdna2_device_count(void) { + int n = 0; + if (hipGetDeviceCount(&n) != hipSuccess) return 0; + return n; +} + +// 0 when the query fails or reports nonsense. Callers must then avoid shapes +// that depend on the wave width rather than guess one: the wave-per-row GEMV +// derives rows-per-block from blockDim/warpSize on the device, so a host guess +// of 32 with an actual wave64 would leave half the rows unprocessed. +static inline int rdna2_device_warp_size(void) { + hipDeviceProp_t prop{}; + if (hipGetDeviceProperties(&prop, 0) != hipSuccess || prop.warpSize <= 0) return 0; + const int ws = prop.warpSize; + if (ws > 64 || (ws & (ws - 1)) != 0) return 0; // not a power of two <= 64 + return ws; +} + +static inline int rdna2_warp_size(void) { + const int ws = rdna2_device_warp_size(); + return ws > 0 ? ws : 32; +} + +// --------------------------------------------------------------------------- +// Group 1: dp4a int8 GEMV. y[rows] = A[rows x k4] . x[k4] +// gfx1031 reads four packed words per load (16-byte vector access); any k4 is +// accepted because the remainder is handled by a scalar tail. Alignment is not +// a caller requirement: the launcher copies into a device buffer, which HIP +// allocates with far stronger alignment than the 16 bytes the vectors need. +// --------------------------------------------------------------------------- +__global__ void rdna2_dp4a_gemv_i8(int* __restrict__ y, + const int* __restrict__ A, // rows x k4 packed + const int* __restrict__ x, // k4 packed + int rows, int k4) { + int row = blockIdx.x * blockDim.x + threadIdx.x; + if (row >= rows) return; + const int* a_s = A + (size_t)row * k4; + // Four independent accumulators: v_dot4 has a multi-cycle result latency, so + // one dependent chain would leave the dot4 pipes idle. Integer addition is + // exact, so summing the four at the end is bit-identical to one chain. + int acc0 = 0, acc1 = 0, acc2 = 0, acc3 = 0; +#if defined(__gfx1031__) || defined(__gfx1200__) || defined(__gfx1201__) + int j = 0; + // The 16-byte vector load is only legal when this row also starts on a + // 16-byte boundary. A row starts at (row * k4) packed words, so that holds + // for every row only when k4 is a multiple of 4. For any other k4 the row + // stride is not 16-byte aligned and v_dot4 is fed from scalar loads instead + // (misaligned dwordx4 loads are not handled by the hardware). + if ((k4 & 3) == 0) { + const int4_t* a = reinterpret_cast(a_s); + const int4_t* xv = reinterpret_cast(x); + for (; j + 4 <= k4; j += 4) { + int4_t av = a[j / 4]; + int4_t xw = xv[j / 4]; + acc0 = rdna2_dot4_i8_impl(av.x, xw.x, acc0); + acc1 = rdna2_dot4_i8_impl(av.y, xw.y, acc1); + acc2 = rdna2_dot4_i8_impl(av.z, xw.z, acc2); + acc3 = rdna2_dot4_i8_impl(av.w, xw.w, acc3); + } + } + for (; j < k4; ++j) acc0 = rdna2_dot4_i8_impl(a_s[j], x[j], acc0); +#else + for (int j = 0; j < k4; ++j) { + acc0 = rdna2_dot4_i8_impl(a_s[j], x[j], acc0); + } +#endif + y[row] = (acc0 + acc1) + (acc2 + acc3); +} + +// One wave per row. Lanes stride the row, so a wave reads 32 consecutive words +// (one 128-byte line) per step instead of the 32 scattered row starts the +// thread-per-row kernel touches, and the row is read exactly once. Better than +// one thread per row once a row holds enough words to keep a wave busy; the +// launcher switches on k4 (see RDNA2_GEMV_WAVE_MIN_K4). +__global__ void rdna2_dp4a_gemv_i8_wave(int* __restrict__ y, + const int* __restrict__ A, + const int* __restrict__ x, int rows, int k4) { + const int ws = (int)warpSize; + const int lane = (int)(threadIdx.x % warpSize); + const int wave = (int)(threadIdx.x / warpSize); + const int waves = (int)(blockDim.x / warpSize); + const int row = (int)blockIdx.x * waves + wave; + if (row >= rows) return; // wave-uniform: a wave never splits across rows + const int* a_s = A + (size_t)row * k4; + int acc = 0; + for (int k = lane; k < k4; k += ws) acc = rdna2_dot4_i8_impl(a_s[k], x[k], acc); + for (int off = ws >> 1; off > 0; off >>= 1) acc += __shfl_xor(acc, off); + if (lane == 0) y[row] = acc; +} + +// --------------------------------------------------------------------------- +// Group 1c: f32 GEMV, y[M] = A[M x K] . x[K]. The same two shapes as the int8 +// GEMV above: one thread per row (short rows) and one wave per row (long rows, +// lanes stride the row so a wave touches consecutive floats). +// --------------------------------------------------------------------------- +__global__ void rdna2_sgemv_f32(float* __restrict__ y, const float* __restrict__ A, + const float* __restrict__ x, int rows, int K) { + const int row = (int)blockIdx.x * blockDim.x + (int)threadIdx.x; + if (row >= rows) return; + const float* a_s = A + (size_t)row * K; + y[row] = rdna2_sgemv_row_f32(a_s, x, K); +} + +__global__ void rdna2_sgemv_f32_wave(float* __restrict__ y, const float* __restrict__ A, + const float* __restrict__ x, int rows, int K) { + const int ws = (int)warpSize; + const int lane = (int)(threadIdx.x % warpSize); + const int wave = (int)(threadIdx.x / warpSize); + const int waves = (int)(blockDim.x / warpSize); + const int row = (int)blockIdx.x * waves + wave; + if (row >= rows) return; // wave-uniform: a wave never splits across rows + const float* a_s = A + (size_t)row * K; + float acc = rdna2_sgemv_lane_partial_f32(a_s, x, K, lane, ws); + for (int off = ws >> 1; off > 0; off >>= 1) acc += __shfl_xor(acc, off); + if (lane == 0) y[row] = acc; +} + +// --------------------------------------------------------------------------- +// Group 1b: dp4a int8 GEMM. +// C[M x N] = A[M x K4] * Bt[N x K4]^T (B stored transposed, N rows of K4) +// Two implementations: a naive one (one thread per element) and a 4x4 +// register-tiled one. The device test compares both against the CPU reference. +// --------------------------------------------------------------------------- +__global__ void rdna2_dp4a_gemm_i8_naive(int* __restrict__ C, + const int* __restrict__ A, + const int* __restrict__ Bt, + int M, int N, int K4) { + int n = blockIdx.x * blockDim.x + threadIdx.x; + int m = blockIdx.y; + if (m >= M || n >= N) return; + const int* a = A + (size_t)m * K4; + const int* b = Bt + (size_t)n * K4; + C[(size_t)m * N + n] = rdna2_gemm_cell_i8(a, b, K4); +} + +// 4x4 outputs per thread; block (16,16) covers a 64x64 output tile. +__global__ void rdna2_dp4a_gemm_i8_tile(int* __restrict__ C, + const int* __restrict__ A, + const int* __restrict__ Bt, + int M, int N, int K4) { + const int m0 = (blockIdx.y * blockDim.y + threadIdx.y) * 4; + const int n0 = (blockIdx.x * blockDim.x + threadIdx.x) * 4; + if (m0 >= M || n0 >= N) return; + + int32_t acc[4][4]; + rdna2_gemm_tile_i8(acc, A, Bt, M, N, K4, m0, n0); + for (int i = 0; i < 4; ++i) { + if (m0 + i >= M) break; + for (int j = 0; j < 4; ++j) { + if (n0 + j >= N) break; + C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 1c: LDS-staged tiled int8 GEMM, same 64x64 output tile and 4x4 register +// blocking as above, but the K loop reads the operands from LDS instead of +// global memory. The thread-per-cell version issues 8 packed loads per k step +// per thread; over a 64x64 tile that is 2048 words of traffic for 128 words of +// actual data (16x). Staging each K chunk cooperatively costs 1024 loads spread +// over 256 threads (4 each) and then every reuse is LDS. +// --------------------------------------------------------------------------- +// Stage one K chunk of a matrix into LDS. 256 threads cover 64 rows x 16 words; +// each thread owns 4 consecutive words of one row. Rows past the edge and words +// past K4 are written as zero, so the compute loop needs no bounds checks. +// lds is a template parameter so the __shared__ address space survives: a fixed +// int (*)[STRIDE] parameter would genericize it to flat stores. +template +__device__ static inline void rdna2_stage_i8(LDP lds, const int* __restrict__ src, int nrows, + int ncols, int row0, int k0, int ld_row, int ld_col) { + const int gm = row0 + ld_row; + int4_t v; + v.x = v.y = v.z = v.w = 0; + if (gm < nrows) { + const int* p = src + (size_t)gm * ncols + k0 + ld_col; + if ((ncols & 3) == 0 && k0 + ld_col + 4 <= ncols) { + v = *reinterpret_cast(p); // 16-byte access, aligned by ncols%4==0 + } else { + for (int c = 0; c < 4; ++c) + if (k0 + ld_col + c < ncols) v[c] = p[c]; + } + } + lds[ld_row][ld_col + 0] = v.x; + lds[ld_row][ld_col + 1] = v.y; + lds[ld_row][ld_col + 2] = v.z; + lds[ld_row][ld_col + 3] = v.w; +} + +__global__ void rdna2_dp4a_gemm_i8_lds(int* __restrict__ C, + const int* __restrict__ A, + const int* __restrict__ Bt, + int M, int N, int K4) { + __shared__ int a_lds[RDNA2_TILE_MN][RDNA2_LDS_I8_STRIDE]; + __shared__ int b_lds[RDNA2_TILE_MN][RDNA2_LDS_I8_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int m_blk = blockIdx.y * RDNA2_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_TILE_MN; + const int lm = threadIdx.y * 4; // this thread's first tile row + const int ln = threadIdx.x * 4; // this thread's first tile column + const int ld_row = tid >> 2; + const int ld_col = (tid & 3) << 2; + + int32_t acc[4][4]; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = 0; + + for (int k0 = 0; k0 < K4; k0 += RDNA2_LDS_I8_KSTEP) { + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col); + __syncthreads(); + int kend = K4 - k0; + if (kend > RDNA2_LDS_I8_KSTEP) kend = RDNA2_LDS_I8_KSTEP; + rdna2_gemm_lds_block_i8(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_LDS_I8_STRIDE, lm, ln, kend); + __syncthreads(); // nobody may restage a tile that is still being read + } + + for (int i = 0; i < 4; ++i) { + if (m_blk + lm + i >= M) break; + for (int j = 0; j < 4; ++j) { + if (n_blk + ln + j >= N) break; + C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; + } + } +} + +// Same LDS tile and K loop, 8x4 register blocking: 128 threads (16,8) cover the +// same 64x64 output tile, each owning 8 rows x 4 columns. Twelve LDS reads per k +// step feed 32 dp4a instead of eight feeding 16, i.e. a quarter less LDS traffic +// for identical math. Staging spreads the same 64x16 words over 128 threads, so +// each thread writes 8 words (two 16-byte loads) instead of 4. +__global__ void rdna2_dp4a_gemm_i8_lds8(int* __restrict__ C, + const int* __restrict__ A, + const int* __restrict__ Bt, + int M, int N, int K4) { + __shared__ int a_lds[RDNA2_TILE_MN][RDNA2_LDS_I8_STRIDE]; + __shared__ int b_lds[RDNA2_TILE_MN][RDNA2_LDS_I8_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int m_blk = blockIdx.y * RDNA2_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_TILE_MN; + const int lm = threadIdx.y * 8; // this thread's first tile row + const int ln = threadIdx.x * 4; // this thread's first tile column + const int ld_row = tid >> 1; // 0..63, two threads per staged row + const int ld_col = (tid & 1) << 3; + + int32_t acc[8][4]; + for (int i = 0; i < 8; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = 0; + + for (int k0 = 0; k0 < K4; k0 += RDNA2_LDS_I8_KSTEP) { + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col); + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col + 4); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col + 4); + __syncthreads(); + int kend = K4 - k0; + if (kend > RDNA2_LDS_I8_KSTEP) kend = RDNA2_LDS_I8_KSTEP; + rdna2_gemm_lds_block8_i8(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_LDS_I8_STRIDE, lm, ln, kend); + __syncthreads(); // nobody may restage a tile that is still being read + } + + for (int i = 0; i < 8; ++i) { + if (m_blk + lm + i >= M) break; + for (int j = 0; j < 4; ++j) { + if (n_blk + ln + j >= N) break; + C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 1e: RDNA4 (gfx1200 / gfx1201) path. Same 16x16 block of 256 threads and +// the same 16-word K chunk as groups 1c/1d, but a 128x128 output tile with 8x8 +// outputs per thread: sixteen LDS reads per k step feed 64 dp4a instead of eight +// feeding 16, and every staged word pays for four times as many results. gfx12 +// has 1536 VGPRs per SIMD against 1024 on gfx10/11, so 64 accumulators plus 16 +// staged operands still leave several waves resident. Two staged tiles of +// 128 x 17 words = 17408 bytes, inside the 64 KiB a gfx12 workgroup addresses. +// Select with tiled = 4 (see rdna2_tiled_for_arch). +// --------------------------------------------------------------------------- +__global__ void rdna2_dp4a_gemm_i8_lds128(int* __restrict__ C, + const int* __restrict__ A, + const int* __restrict__ Bt, + int M, int N, int K4) { + __shared__ int a_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_I8_STRIDE]; + __shared__ int b_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_I8_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int m_blk = blockIdx.y * RDNA2_T4_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_T4_TILE_MN; + const int lm = threadIdx.y * 8; // this thread's first tile row + const int ln = threadIdx.x * 8; // this thread's first tile column + const int ld_row = tid >> 1; // 0..127: two threads per staged row + const int ld_col = (tid & 1) << 3; // 0 or 8, eight staged words each + + int32_t acc[8][8]; + for (int i = 0; i < 8; ++i) + for (int j = 0; j < 8; ++j) acc[i][j] = 0; + + for (int k0 = 0; k0 < K4; k0 += RDNA2_T4_LDS_I8_KSTEP) { + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col); + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col + 4); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col + 4); + __syncthreads(); + int kend = K4 - k0; + if (kend > RDNA2_T4_LDS_I8_KSTEP) kend = RDNA2_T4_LDS_I8_KSTEP; + rdna2_gemm_lds_block8x8_i8(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_T4_LDS_I8_STRIDE, lm, + ln, kend); + __syncthreads(); // nobody may restage a tile that is still being read + } + + for (int i = 0; i < 8; ++i) { + if (m_blk + lm + i >= M) break; + for (int j = 0; j < 8; ++j) { + if (n_blk + ln + j >= N) break; + C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 1f: RDNA4 (gfx1200 / gfx1201) matrix path. Same 128x128 output tile, +// same 16-word K chunk and same LDS staging as group 1e, but the arithmetic is +// the gfx12 matrix instruction V_WMMA_I32_16X16X16_IU8 instead of 64 dp4a per +// thread: one instruction covers a 16x16x16 signed product that the 8x8 dp4a +// kernel needs 64 instructions and 16 LDS reads per k step to reach. +// +// The eight waves of the block each own a 16-row band of the tile and sweep the +// eight 16-column groups, keeping 8 x int32x8 accumulators (64 VGPRs, the same +// budget as group 1e's 8x8 block). Per K chunk a wave issues 32 WMMA -- 8 per +// 16-byte K step -- and pays only 2 LDS words for A and 2 per B fragment, i.e. +// 36 word reads for 32 matrix ops instead of 256 reads for 1024 dp4a. +// +// Fragment layout, measured on a gfx1201 card (256/256 outputs of a signed +// 16x16x16 product matched the scalar reference; the layout and the negA/negB +// semantics were both derived experimentally, see README): +// A: lane t, byte e -> A[M = t%16][K = (t/16)*8 + e] +// B: lane t, byte e -> B[K = (t/16)*8 + e][N = t%16] +// D: lane t, slot l -> D[M = (t/16)*8 + l][N = t%16] +// Both operands are signed, so negA and negB are true; __builtin_amdgcn_wmma_* +// needs those flags as compile-time constants, hence the literals. +// Select with tiled = 5 (see rdna2_tiled_for_arch). +// --------------------------------------------------------------------------- +// One 16x16x16 signed step for this lane: acc holds this lane's eight output +// slots, a and b are its eight-byte A and B fragments. +// gfx12 -- the matrix instruction. +// other AMD GPU -- no matrix unit on gfx10/11, so the same result is built +// from explicit byte dots, gathering each slot's operands +// out of lanes row and row+16 with shuffles. Correct, but +// with a shuffle per operand byte it cannot beat the dp4a +// kernel, and rdna2_tiled_for_arch never picks tiled = 5 on +// those chips. Not measured: no RDNA2 hardware here. +// plain C++ pass -- the file also compiles as host code (CPU test harness and +// the host half of a HIP build), where no kernel can run; the +// body is only there to keep the kernel defined so the +// unconditional launch site links. +__device__ static inline int32x8_t rdna2_wmma_iu8_step(int32x2_t a, int32x2_t b, int32x8_t acc, + int lane) { + (void)lane; // the matrix instruction addresses the other lanes in hardware +#if RDNA2_WMMA_IU8 + return __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a, true, b, acc, false); +#elif defined(__HIP_DEVICE_COMPILE__) + for (int l = 0; l < 8; ++l) { + const int dm = rdna2_wmma_i8_acc_row(lane, l); + const int dn = rdna2_wmma_i8_acc_col(lane); + int32_t d = 0; + for (int k = 0; k < 16; ++k) { + const int shift = 8 * (k & 3), word = (k & 7) >> 2; + const int av = (int8_t)(__shfl((int)a[word], dm + (k >> 3) * 16) >> shift); + const int bv = (int8_t)(__shfl((int)b[word], dn + (k >> 3) * 16) >> shift); + d += av * bv; + } + acc[l] += d; + } + return acc; +#else + (void)a; + (void)b; + return acc; +#endif +} + +__global__ void rdna2_dp4a_gemm_i8_wmma(int* __restrict__ C, + const int* __restrict__ A, + const int* __restrict__ Bt, + int M, int N, int K4) { + __shared__ int a_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_I8_STRIDE]; + __shared__ int b_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_I8_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int lane = tid & 31; + const int wave = tid >> 5; // 0..7: this wave's 16-row band of the tile + const int m_blk = blockIdx.y * RDNA2_T4_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_T4_TILE_MN; + const int m_row = wave * 16 + rdna2_wmma_i8_row(lane); // staged A row this lane feeds + const int n_col = rdna2_wmma_i8_row(lane); // column group offset inside a 16-wide group + const int ld_row = tid >> 1; // 0..127: two threads per staged row + const int ld_col = (tid & 1) << 3; // 0 or 8, eight staged words each + + int32x8_t acc[RDNA2_T4_WMMA_NTILE]; + for (int j = 0; j < RDNA2_T4_WMMA_NTILE; ++j) { + for (int l = 0; l < 8; ++l) acc[j][l] = 0; + } + + for (int k0 = 0; k0 < K4; k0 += RDNA2_T4_LDS_I8_KSTEP) { + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col); + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col + 4); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col + 4); + __syncthreads(); + int kend = K4 - k0; + if (kend > RDNA2_T4_LDS_I8_KSTEP) kend = RDNA2_T4_LDS_I8_KSTEP; + // One WMMA covers 4 staged words (16 K bytes); a short last chunk runs the + // steps it has and relies on the staging zeros for the rest of the tile. + const int ksteps = (kend + 3) / 4; + for (int ks = 0; ks < ksteps; ++ks) { + const int w = rdna2_wmma_i8_frag_word(lane, ks); + int32x2_t a; + a[0] = a_lds[m_row][w]; + a[1] = a_lds[m_row][w + 1]; + for (int j = 0; j < RDNA2_T4_WMMA_NTILE; ++j) { + int32x2_t b; + b[0] = b_lds[j * 16 + n_col][w]; + b[1] = b_lds[j * 16 + n_col][w + 1]; + acc[j] = rdna2_wmma_iu8_step(a, b, acc[j], lane); + } + } + __syncthreads(); // nobody may restage a tile that is still being read + } + + // D map: slot l of lane t lands at row 8*(t/16) + l, column t%16 of the + // 16x16 tile, so a lane writes one column of its wave's 16-row band. + for (int j = 0; j < RDNA2_T4_WMMA_NTILE; ++j) { + const int n = n_blk + j * 16 + rdna2_wmma_i8_acc_col(lane); + if (n >= N) continue; + for (int l = 0; l < 8; ++l) { + const int m = m_blk + wave * 16 + rdna2_wmma_i8_acc_row(lane, l); + if (m >= M) break; // rows 8*(t/16)+l are contiguous, so a break is safe + C[(size_t)m * N + n] = acc[j][l]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 2: f32 GEMM, 4x4 register tiled. C[M x N] = A[M x K] * Bt[N x K]^T +// --------------------------------------------------------------------------- +__global__ void rdna2_sgemm_f32(float* __restrict__ C, + const float* __restrict__ A, + const float* __restrict__ Bt, + int M, int N, int K) { + const int m0 = (blockIdx.y * blockDim.y + threadIdx.y) * 4; + const int n0 = (blockIdx.x * blockDim.x + threadIdx.x) * 4; + if (m0 >= M || n0 >= N) return; + + float acc[4][4]; + rdna2_sgemm_tile_f32(acc, A, Bt, M, N, K, m0, n0); + for (int i = 0; i < 4; ++i) { + if (m0 + i >= M) break; + for (int j = 0; j < 4; ++j) { + if (n0 + j >= N) break; + C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 2b: LDS-staged f32 GEMM, same 64x64 tile and 4x4 register blocking as +// group 1c: the K loop reads A and Bt from LDS instead of re-reading global +// memory in every thread. +// --------------------------------------------------------------------------- +// 256 threads cover 64 rows x 8 floats: two consecutive floats per thread. +template +__device__ static inline void rdna2_stage_f32(LDP lds, const float* __restrict__ src, int nrows, + int ncols, int row0, int k0, int ld_row, int ld_col) { + const int gm = row0 + ld_row; + float v0 = 0.0f, v1 = 0.0f; + if (gm < nrows) { + const float* p = src + (size_t)gm * ncols + k0 + ld_col; + if (k0 + ld_col + 0 < ncols) v0 = p[0]; + if (k0 + ld_col + 1 < ncols) v1 = p[1]; + } + lds[ld_row][ld_col + 0] = v0; + lds[ld_row][ld_col + 1] = v1; +} + +__global__ void rdna2_sgemm_f32_lds(float* __restrict__ C, + const float* __restrict__ A, + const float* __restrict__ Bt, + int M, int N, int K) { + __shared__ float a_lds[RDNA2_TILE_MN][RDNA2_LDS_F32_STRIDE]; + __shared__ float b_lds[RDNA2_TILE_MN][RDNA2_LDS_F32_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int m_blk = blockIdx.y * RDNA2_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_TILE_MN; + const int lm = threadIdx.y * 4; + const int ln = threadIdx.x * 4; + const int ld_row = tid >> 2; + const int ld_col = (tid & 3) << 1; + + float acc[4][4]; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = 0.0f; + + for (int k0 = 0; k0 < K; k0 += RDNA2_LDS_F32_KSTEP) { + rdna2_stage_f32(a_lds, A, M, K, m_blk, k0, ld_row, ld_col); + rdna2_stage_f32(b_lds, Bt, N, K, n_blk, k0, ld_row, ld_col); + __syncthreads(); + int kend = K - k0; + if (kend > RDNA2_LDS_F32_KSTEP) kend = RDNA2_LDS_F32_KSTEP; + rdna2_sgemm_lds_block_f32(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_LDS_F32_STRIDE, lm, ln, kend); + __syncthreads(); + } + + for (int i = 0; i < 4; ++i) { + if (m_blk + lm + i >= M) break; + for (int j = 0; j < 4; ++j) { + if (n_blk + ln + j >= N) break; + C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 2b: 64x64 f32 tile with a 16-float K chunk (variant 3). Same block +// shape and 4x4 per-thread blocking as group 2, but 256 threads stage 1024 +// floats (4 each, RDNA2_LDS_F32B_STRIDE = 17 per row) instead of 512 floats +// (2 each) per iteration. That makes each staged chunk cover a whole 64-byte +// cache line of every row, and halves the number of __syncthreads over K. +// --------------------------------------------------------------------------- +template +__device__ static inline void rdna2_stage_f32_4(LDP lds, const float* __restrict__ src, int nrows, + int ncols, int row0, int k0, int ld_row, + int ld_col) { + const int gm = row0 + ld_row; + float v0 = 0.0f, v1 = 0.0f, v2 = 0.0f, v3 = 0.0f; + if (gm < nrows) { + const float* p = src + (size_t)gm * ncols + k0 + ld_col; + const int rem = ncols - (k0 + ld_col); + if (rem > 0) v0 = p[0]; + if (rem > 1) v1 = p[1]; + if (rem > 2) v2 = p[2]; + if (rem > 3) v3 = p[3]; + } + lds[ld_row][ld_col + 0] = v0; + lds[ld_row][ld_col + 1] = v1; + lds[ld_row][ld_col + 2] = v2; + lds[ld_row][ld_col + 3] = v3; +} + +__global__ void rdna2_sgemm_f32_lds16(float* __restrict__ C, + const float* __restrict__ A, + const float* __restrict__ Bt, + int M, int N, int K) { + __shared__ float a_lds[RDNA2_TILE_MN][RDNA2_LDS_F32B_STRIDE]; + __shared__ float b_lds[RDNA2_TILE_MN][RDNA2_LDS_F32B_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int m_blk = blockIdx.y * RDNA2_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_TILE_MN; + const int lm = threadIdx.y * 4; + const int ln = threadIdx.x * 4; + const int ld_row = tid >> 2; + const int ld_col = (tid & 3) << 2; + + float acc[4][4]; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = 0.0f; + + for (int k0 = 0; k0 < K; k0 += RDNA2_LDS_F32B_KSTEP) { + rdna2_stage_f32_4(a_lds, A, M, K, m_blk, k0, ld_row, ld_col); + rdna2_stage_f32_4(b_lds, Bt, N, K, n_blk, k0, ld_row, ld_col); + __syncthreads(); + int kend = K - k0; + if (kend > RDNA2_LDS_F32B_KSTEP) kend = RDNA2_LDS_F32B_KSTEP; + rdna2_sgemm_lds_block_f32(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_LDS_F32B_STRIDE, lm, ln, kend); + __syncthreads(); + } + + for (int i = 0; i < 4; ++i) { + if (m_blk + lm + i >= M) break; + for (int j = 0; j < 4; ++j) { + if (n_blk + ln + j >= N) break; + C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 2c: RDNA4 (gfx1200 / gfx1201) f32 GEMM on the same 128x128 tile and 8x8 +// per-thread blocking as group 1e. Two staged tiles of 128 x 9 floats = 9216 +// bytes; sixteen LDS reads per k step feed 64 fma. Select with variant 2. +// --------------------------------------------------------------------------- +__global__ void rdna2_sgemm_f32_lds128(float* __restrict__ C, + const float* __restrict__ A, + const float* __restrict__ Bt, + int M, int N, int K) { + __shared__ float a_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_F32_STRIDE]; + __shared__ float b_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_F32_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int m_blk = blockIdx.y * RDNA2_T4_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_T4_TILE_MN; + const int lm = threadIdx.y * 8; + const int ln = threadIdx.x * 8; + const int ld_row = tid >> 1; // 0..127: two threads per staged row + const int ld_col = (tid & 1) << 2; // 0 or 4, four staged floats each + + float acc[8][8]; + for (int i = 0; i < 8; ++i) + for (int j = 0; j < 8; ++j) acc[i][j] = 0.0f; + + for (int k0 = 0; k0 < K; k0 += RDNA2_T4_LDS_F32_KSTEP) { + rdna2_stage_f32(a_lds, A, M, K, m_blk, k0, ld_row, ld_col); + rdna2_stage_f32(a_lds, A, M, K, m_blk, k0, ld_row, ld_col + 2); + rdna2_stage_f32(b_lds, Bt, N, K, n_blk, k0, ld_row, ld_col); + rdna2_stage_f32(b_lds, Bt, N, K, n_blk, k0, ld_row, ld_col + 2); + __syncthreads(); + int kend = K - k0; + if (kend > RDNA2_T4_LDS_F32_KSTEP) kend = RDNA2_T4_LDS_F32_KSTEP; + rdna2_sgemm_lds_block8x8_f32(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_T4_LDS_F32_STRIDE, lm, + ln, kend); + __syncthreads(); + } + + for (int i = 0; i < 8; ++i) { + if (m_blk + lm + i >= M) break; + for (int j = 0; j < 8; ++j) { + if (n_blk + ln + j >= N) break; + C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 3: packed f16 dot (v_dot2_f32_f16; clang emits the accumulate form +// from __builtin_amdgcn_fdot2, and the ISA mnemonic has no 'c'). +// out is accumulated with atomics and must be zeroed by the launcher. +// Correct for both wave32 and wave64: the shuffle reduction covers one +// 16-lane row, then one atomic per row. +// --------------------------------------------------------------------------- +__global__ void rdna2_dot2_f16(float* __restrict__ out, + const half2_t* __restrict__ a, + const half2_t* __restrict__ b, + int n) { + float acc = 0.0f; + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < n; + i += gridDim.x * blockDim.x) { + acc = __builtin_amdgcn_fdot2(a[i], b[i], acc, false); + } + acc += __shfl_xor(acc, 1); + acc += __shfl_xor(acc, 2); + acc += __shfl_xor(acc, 4); + acc += __shfl_xor(acc, 8); + if ((threadIdx.x & 15) == 0) atomicAdd(out, acc); +} + +// --------------------------------------------------------------------------- +// Group 4: dp4a u8 dot (v_dot4_u32_u8). +// out is accumulated with atomics and must be zeroed by the launcher. +// --------------------------------------------------------------------------- +__global__ void rdna2_dp4a_dot_u8(unsigned* __restrict__ out, + const unsigned* __restrict__ a, + const unsigned* __restrict__ b, + int n) { + unsigned acc = 0; + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < n; + i += gridDim.x * blockDim.x) { + acc = __builtin_amdgcn_udot4(a[i], b[i], acc, false); + } + acc += __shfl_xor(acc, 1); + acc += __shfl_xor(acc, 2); + acc += __shfl_xor(acc, 4); + acc += __shfl_xor(acc, 8); + if ((threadIdx.x & 15) == 0) atomicAdd(out, acc); +} + +// --------------------------------------------------------------------------- +// Group 5: inclusive prefix sum over one wave. +// DPP row_shr builds a 16-lane Hillis-Steele scan with no LDS traffic; the +// cross-half total then arrives through a shuffle (v_readlane/v_permlane). +// out[tid] = in[0] + ... + in[tid] for every lane tid < n <= warpSize. +// --------------------------------------------------------------------------- +__global__ void rdna2_dpp_prefix(int* __restrict__ out, + const int* __restrict__ in, + int n) { + const int lane = threadIdx.x & (warpSize - 1); + int v = (lane < n) ? in[lane] : 0; + + // row_shr:1,2,4,8 with bound_ctrl off: lanes shifted out of the row get 0. + v += __builtin_amdgcn_update_dpp(0, v, 0x111, 0xf, 0xf, false); + v += __builtin_amdgcn_update_dpp(0, v, 0x112, 0xf, 0xf, false); + v += __builtin_amdgcn_update_dpp(0, v, 0x114, 0xf, 0xf, false); + v += __builtin_amdgcn_update_dpp(0, v, 0x118, 0xf, 0xf, false); + + // Lane 15 holds the total of lanes 0-15; add it to the upper half. + int lo = __shfl(v, 15); + if (lane >= 16) v += lo; + // On wave64, lanes 32-63 also need the total of lanes 0-31. + if (warpSize == 64) { + int mid = __shfl(v, 31); + if (lane >= 32) v += mid; + } + out[lane] = v; +} + +// --------------------------------------------------------------------------- +// Group 6: quantized epilogue (scale and zero point) +// --------------------------------------------------------------------------- +// Per-row sum of the int8 activations. dp4a against an all-ones operand sums the +// four signed bytes of a packed word in one instruction, so this is one v_dot4 +// per word. Only needed for asymmetric weights (a nonzero zero point); a +// symmetric format passes zp_b = 0 and skips it entirely. +__global__ void rdna2_i8_row_sums(int* __restrict__ sums, const int* __restrict__ A, int rows, + int k4) { + const int row = blockIdx.x * blockDim.x + threadIdx.x; + if (row >= rows) return; + const int* a = A + (size_t)row * k4; + const int ones = 0x01010101; + int acc = 0; + for (int k = 0; k < k4; ++k) acc = rdna2_dot4_i8_impl(a[k], ones, acc); + sums[row] = acc; +} + +// Applies the quantized format's scale and zero point to the finished integer +// accumulators: +// C[m][n] = (acc[m][n] - zp_b * sums[m]) * sa[m] * sb[n] +// The correction is the exact integer form of B's asymmetric storage: B's stored +// value at k represents (q - zp_b), so sum_k a*(q - zp_b) = acc - zp_b*sum_k a. +// No stored weight is converted here - only the completed dot product is, once, +// after the correction has been applied in 64-bit so it cannot overflow. sa is +// NULL for unit row scale, sb NULL for unit column scale. +__global__ void rdna2_gemm_i8_epilogue(float* __restrict__ C, const int* __restrict__ acc, + const int* __restrict__ sums, int M, int N, + const float* __restrict__ sa, const float* __restrict__ sb, + int zp_b) { + const int n = blockIdx.x * blockDim.x + threadIdx.x; + if (n >= N) return; + // Rows stride over gridDim.y so the row count is not capped at the grid's + // y limit; each iteration is an independent output element. + for (int m = blockIdx.y; m < M; m += gridDim.y) { + long long v = (long long)acc[(size_t)m * N + n]; + if (zp_b != 0 && sums) v -= (long long)zp_b * (long long)sums[m]; + float f = (float)v; + if (sa) f *= sa[m]; + if (sb) f *= sb[n]; + C[(size_t)m * N + n] = f; + } +} + +// --------------------------------------------------------------------------- +// Host launchers +// --------------------------------------------------------------------------- +// One thread per row is the better shape for short rows (no reduction to pay +// for); from this many packed words per row on, a whole wave per row wins on +// coalescing. +#define RDNA2_GEMV_WAVE_MIN_K4 8 +// f32 rows need a real body before a wave pays for its reduction. Measured on a +// 9070 XT at 2048-16384 rows: below K=256 the five shuffle steps cost more than +// the coalescing gains (thread-per-row is 1.2-3.8x ahead at K=64..160), and from +// K=256 up the wave-per-row kernel wins every time (1.15x at K=256, ~2x by +// K=1024, 3.5x at K=4096 with 16384 rows). +#define RDNA2_SGEMV_WAVE_MIN_K 256 + +static int rdna2_gemv_i8_device(int* y, const int* A, const int* x, int rows, int k4, + int variant) { + int *dA = nullptr, *dx = nullptr, *dy = nullptr; + const size_t bytesA = (size_t)rows * k4 * sizeof(int); + const size_t bytesx = (size_t)k4 * sizeof(int); + const size_t bytesy = (size_t)rows * sizeof(int); + int rc = RDNA2_OK; + int ws = 0; // wave width; stays 0 unless the wave-per-row variant can size its grid + if (hipMalloc(&dA, bytesA) != hipSuccess || hipMalloc(&dx, bytesx) != hipSuccess || + hipMalloc(&dy, bytesy) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(dA, A, bytesA, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(dx, x, bytesx, hipMemcpyHostToDevice) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (variant == 1) ws = rdna2_device_warp_size(); + if (ws > 0) { + const int threads = 128; // 4 waves of 32 or 2 waves of 64 + const int waves = threads / ws; + const int blocks = (rows + waves - 1) / waves; + hipLaunchKernelGGL(rdna2_dp4a_gemv_i8_wave, dim3(blocks), dim3(threads), 0, 0, + dy, dA, dx, rows, k4); + } else { + const int threads = 64; + const int blocks = (rows + threads - 1) / threads; + hipLaunchKernelGGL(rdna2_dp4a_gemv_i8, dim3(blocks), dim3(threads), 0, 0, + dy, dA, dx, rows, k4); + } + if (hipGetLastError() != hipSuccess || + hipMemcpy(y, dy, bytesy, hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } +cleanup: + if (dA) (void)hipFree(dA); + if (dx) (void)hipFree(dx); + if (dy) (void)hipFree(dy); + return rc; +} + +// variant 0 = one thread per row, 1 = one wave per row. +int rdna2_gemv_i8_host_variant(int* y, const int* A, const int* x, int rows, int k4, int variant) { + if (!y || !A || !x || rows <= 0 || k4 <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + rdna2_gemv_i8_ref(y, A, x, rows, k4); + return RDNA2_OK; + } + return rdna2_gemv_i8_device(y, A, x, rows, k4, variant); +} + +int rdna2_gemv_i8_host(int* y, const int* A, const int* x, int rows, int k4) { + return rdna2_gemv_i8_host_variant(y, A, x, rows, k4, k4 >= RDNA2_GEMV_WAVE_MIN_K4 ? 1 : 0); +} + +static int rdna2_sgemv_f32_device(float* y, const float* A, const float* x, int rows, int K, + int variant) { + float *dA = nullptr, *dx = nullptr, *dy = nullptr; + const size_t bytesA = (size_t)rows * K * sizeof(float); + const size_t bytesx = (size_t)K * sizeof(float); + const size_t bytesy = (size_t)rows * sizeof(float); + int rc = RDNA2_OK; + int ws = 0; // wave width; stays 0 unless the wave-per-row variant can size its grid + if (hipMalloc(&dA, bytesA) != hipSuccess || hipMalloc(&dx, bytesx) != hipSuccess || + hipMalloc(&dy, bytesy) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(dA, A, bytesA, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(dx, x, bytesx, hipMemcpyHostToDevice) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (variant == 1) ws = rdna2_device_warp_size(); + if (ws > 0) { + const int threads = 128; // 4 waves of 32 or 2 waves of 64 + const int waves = threads / ws; + const int blocks = (rows + waves - 1) / waves; + hipLaunchKernelGGL(rdna2_sgemv_f32_wave, dim3(blocks), dim3(threads), 0, 0, + dy, dA, dx, rows, K); + } else { + const int threads = 64; + const int blocks = (rows + threads - 1) / threads; + hipLaunchKernelGGL(rdna2_sgemv_f32, dim3(blocks), dim3(threads), 0, 0, + dy, dA, dx, rows, K); + } + if (hipGetLastError() != hipSuccess || + hipMemcpy(y, dy, bytesy, hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } +cleanup: + if (dA) (void)hipFree(dA); + if (dx) (void)hipFree(dx); + if (dy) (void)hipFree(dy); + return rc; +} + +// variant 0 = one thread per row, 1 = one wave per row. +int rdna2_sgemv_f32_host_variant(float* y, const float* A, const float* x, int rows, int K, + int variant) { + if (!y || !A || !x || rows <= 0 || K <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + rdna2_sgemv_f32_ref(y, A, x, rows, K); + return RDNA2_OK; + } + return rdna2_sgemv_f32_device(y, A, x, rows, K, variant); +} + +int rdna2_sgemv_f32_host(float* y, const float* A, const float* x, int rows, int K) { + return rdna2_sgemv_f32_host_variant(y, A, x, rows, K, + K >= RDNA2_SGEMV_WAVE_MIN_K ? 1 : 0); +} + +// Shapes the int8 GEMM onto a grid and launches it. Shared by the plain entry +// point and the quantized-epilogue one so both measure the same kernels. +static void rdna2_launch_gemm_i8(int* dC, const int* dA, const int* dB, int M, int N, int K4, + int tiled) { + if (tiled == 4 || tiled == 5) { + // RDNA4 shape: 128x128 tile, 16x16 threads. 17408 B of LDS per block for + // int8, well inside the 64 KiB workgroup limit. tiled == 5 is the same tile + // driven by the matrix instruction instead of dp4a, so it shares the grid; + // which instruction runs is decided inside the kernel (see + // rdna2_wmma_iu8_step), not here -- this function is host code and cannot + // see the offload arch. + const dim3 grid((N + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN, + (M + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN, 1); + const dim3 block(16, 16, 1); + if (tiled == 5) + hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_wmma, grid, block, 0, 0, dC, dA, dB, M, N, K4); + else + hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_lds128, grid, block, 0, 0, dC, dA, dB, M, N, K4); + } else if (tiled) { + const dim3 grid((N + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN, + (M + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN, 1); + if (tiled == 3) { + const dim3 block(16, 8, 1); // 128 threads, 8x4 outputs each + hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_lds8, grid, block, 0, 0, dC, dA, dB, M, N, K4); + } else if (tiled == 2) { + const dim3 block(16, 16, 1); + hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_lds, grid, block, 0, 0, dC, dA, dB, M, N, K4); + } else { + const dim3 block(16, 16, 1); + hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_tile, grid, block, 0, 0, dC, dA, dB, M, N, K4); + } + } else { + const dim3 block(64, 1, 1); + const dim3 grid((N + 63) / 64, M, 1); + hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_naive, grid, block, 0, 0, dC, dA, dB, M, N, K4); + } +} + +static int rdna2_gemm_i8_device(int* C, const int* A, const int* Bt, int M, int N, int K4, + int tiled) { + int *dA = nullptr, *dB = nullptr, *dC = nullptr; + const size_t bytesA = (size_t)M * K4 * sizeof(int); + const size_t bytesB = (size_t)N * K4 * sizeof(int); + const size_t bytesC = (size_t)M * N * sizeof(int); + int rc = RDNA2_OK; + if (hipMalloc(&dA, bytesA) != hipSuccess || hipMalloc(&dB, bytesB) != hipSuccess || + hipMalloc(&dC, bytesC) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(dA, A, bytesA, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(dB, Bt, bytesB, hipMemcpyHostToDevice) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + rdna2_launch_gemm_i8(dC, dA, dB, M, N, K4, tiled); + if (hipGetLastError() != hipSuccess || + hipMemcpy(C, dC, bytesC, hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } +cleanup: + if (dA) (void)hipFree(dA); + if (dB) (void)hipFree(dB); + if (dC) (void)hipFree(dC); + return rc; +} + +// tiled: 0 = naive, 1 = 4x4 register tiled (global loads), 2 = same tiling with +// LDS-staged K chunks, 3 = LDS staging with 8x4 register blocking (quarter less +// LDS traffic for the same math), 4 = RDNA4 128x128 tile with 8x8 blocking. +// Any non-zero value keeps the tiled shape; 4 picks its own grid. +int rdna2_gemm_i8_host(int* C, const int* A, const int* Bt, int M, int N, int K4, int tiled) { + if (!C || !A || !Bt || M <= 0 || N <= 0 || K4 <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + rdna2_gemm_i8_ref(C, A, Bt, M, N, K4); + return RDNA2_OK; + } + return rdna2_gemm_i8_device(C, A, Bt, M, N, K4, tiled); +} + +// Quantized entry point: the same int8 GEMM followed by the format's scale and +// zero point. A and Bt are consumed exactly as stored - no weight is expanded to +// float, and the only float produced is the finished dot product, once. C is f32 +// because a GEMM result feeds the next layer's float activation path; sa scales +// rows of A and sb columns of Bt (either may be NULL for 1.0), zp_b is B's zero +// point (0 for symmetric weights, in which case the row-sum pass is skipped). +static int rdna2_gemm_i8_scale_device(float* C, const int* A, const int* Bt, int M, int N, int K4, + int tiled, const float* sa, const float* sb, int zp_b) { + int *dA = nullptr, *dB = nullptr, *dAcc = nullptr, *dSums = nullptr; + float *dSa = nullptr, *dSb = nullptr, *dC = nullptr; + const size_t bytesA = (size_t)M * K4 * sizeof(int); + const size_t bytesB = (size_t)N * K4 * sizeof(int); + const size_t nOut = (size_t)M * N; + int rc = RDNA2_OK; + if (hipMalloc(&dA, bytesA) != hipSuccess || hipMalloc(&dB, bytesB) != hipSuccess || + hipMalloc(&dAcc, nOut * sizeof(int)) != hipSuccess || + hipMalloc(&dC, nOut * sizeof(float)) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if ((zp_b != 0 && hipMalloc(&dSums, (size_t)M * sizeof(int)) != hipSuccess) || + (sa && hipMalloc(&dSa, (size_t)M * sizeof(float)) != hipSuccess) || + (sb && hipMalloc(&dSb, (size_t)N * sizeof(float)) != hipSuccess)) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(dA, A, bytesA, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(dB, Bt, bytesB, hipMemcpyHostToDevice) != hipSuccess || + (sa && hipMemcpy(dSa, sa, (size_t)M * sizeof(float), hipMemcpyHostToDevice) != hipSuccess) || + (sb && hipMemcpy(dSb, sb, (size_t)N * sizeof(float), hipMemcpyHostToDevice) != hipSuccess)) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + rdna2_launch_gemm_i8(dAcc, dA, dB, M, N, K4, tiled); + if (zp_b != 0) { + const int threads = 128; + hipLaunchKernelGGL(rdna2_i8_row_sums, dim3((M + threads - 1) / threads), dim3(threads), 0, 0, + dSums, dA, M, K4); + } + { + const dim3 block(256, 1, 1); + const int rows_per_grid = 4096; // rows stride, so grid.y stays inside limits + const dim3 grid((N + 255) / 256, M < rows_per_grid ? M : rows_per_grid, 1); + hipLaunchKernelGGL(rdna2_gemm_i8_epilogue, grid, block, 0, 0, dC, dAcc, dSums, M, N, dSa, dSb, + zp_b); + } + if (hipGetLastError() != hipSuccess || + hipMemcpy(C, dC, nOut * sizeof(float), hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } +cleanup: + if (dA) (void)hipFree(dA); + if (dB) (void)hipFree(dB); + if (dAcc) (void)hipFree(dAcc); + if (dSums) (void)hipFree(dSums); + if (dSa) (void)hipFree(dSa); + if (dSb) (void)hipFree(dSb); + if (dC) (void)hipFree(dC); + return rc; +} + +int rdna2_gemm_i8_scale_host(float* C, const int* A, const int* Bt, int M, int N, int K4, + int tiled, const float* sa, const float* sb, int zp_b) { + if (!C || !A || !Bt || M <= 0 || N <= 0 || K4 <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + rdna2_gemm_i8_scale_ref(C, A, Bt, M, N, K4, sa, sb, zp_b); + return RDNA2_OK; + } + return rdna2_gemm_i8_scale_device(C, A, Bt, M, N, K4, tiled, sa, sb, zp_b); +} + +// variant 0 = 4x4 tiled reading global memory, 1 = the same tiling with LDS +// staging (fewer global reads per output), 2 = RDNA4 128x128 tile with 8x8 +// blocking. +int rdna2_sgemm_f32_host_v(float* C, const float* A, const float* Bt, int M, int N, int K, + int variant) { + if (!C || !A || !Bt || M <= 0 || N <= 0 || K <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + rdna2_sgemm_f32_ref(C, A, Bt, M, N, K); + return RDNA2_OK; + } + float *dA = nullptr, *dB = nullptr, *dC = nullptr; + const size_t bytesA = (size_t)M * K * sizeof(float); + const size_t bytesB = (size_t)N * K * sizeof(float); + const size_t bytesC = (size_t)M * N * sizeof(float); + int rc = RDNA2_OK; + if (hipMalloc(&dA, bytesA) != hipSuccess || hipMalloc(&dB, bytesB) != hipSuccess || + hipMalloc(&dC, bytesC) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(dA, A, bytesA, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(dB, Bt, bytesB, hipMemcpyHostToDevice) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (variant == 2) { + // RDNA4 shape: 128x128 f32 tile, 8x8 outputs per thread, 9216 B of LDS. + const dim3 block(16, 16, 1); + const dim3 grid((N + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN, + (M + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN, 1); + hipLaunchKernelGGL(rdna2_sgemm_f32_lds128, grid, block, 0, 0, dC, dA, dB, M, N, K); + if (hipGetLastError() != hipSuccess || + hipMemcpy(C, dC, bytesC, hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } + } else { + const dim3 block(16, 16, 1); + const dim3 grid((N + 63) / 64, (M + 63) / 64, 1); + if (variant == 1) { + hipLaunchKernelGGL(rdna2_sgemm_f32_lds, grid, block, 0, 0, dC, dA, dB, M, N, K); + } else if (variant == 3) { + hipLaunchKernelGGL(rdna2_sgemm_f32_lds16, grid, block, 0, 0, dC, dA, dB, M, N, K); + } else { + hipLaunchKernelGGL(rdna2_sgemm_f32, grid, block, 0, 0, dC, dA, dB, M, N, K); + } + if (hipGetLastError() != hipSuccess || + hipMemcpy(C, dC, bytesC, hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } + } +cleanup: + if (dA) (void)hipFree(dA); + if (dB) (void)hipFree(dB); + if (dC) (void)hipFree(dC); + return rc; +} + +int rdna2_sgemm_f32_host(float* C, const float* A, const float* Bt, int M, int N, int K) { + return rdna2_sgemm_f32_host_v(C, A, Bt, M, N, K, 1); +} + +int rdna2_dot_u8_host(unsigned* out, const unsigned* a, const unsigned* b, int n) { + if (!out || !a || !b || n <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + *out = rdna2_dot_u8_ref(a, b, n); + return RDNA2_OK; + } + unsigned* da = nullptr; + unsigned* db = nullptr; + unsigned* dout = nullptr; + int rc = RDNA2_OK; + const size_t bytes = (size_t)n * sizeof(unsigned); + if (hipMalloc(&da, bytes) != hipSuccess || hipMalloc(&db, bytes) != hipSuccess || + hipMalloc(&dout, sizeof(unsigned)) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(da, a, bytes, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(db, b, bytes, hipMemcpyHostToDevice) != hipSuccess || + hipMemset(dout, 0, sizeof(unsigned)) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + { + const int threads = 256; + int blocks = (n + threads - 1) / threads; + if (blocks > 1024) blocks = 1024; + hipLaunchKernelGGL(rdna2_dp4a_dot_u8, dim3(blocks), dim3(threads), 0, 0, dout, da, db, n); + if (hipGetLastError() != hipSuccess || + hipMemcpy(out, dout, sizeof(unsigned), hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } + } +cleanup: + if (da) (void)hipFree(da); + if (db) (void)hipFree(db); + if (dout) (void)hipFree(dout); + return rc; +} + +// a and b hold n packed pairs of _Float16 (two per uint32). +int rdna2_dot_f16_host(float* out, const unsigned* a, const unsigned* b, int n) { + if (!out || !a || !b || n <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { +#if defined(__FLT16_MAX__) + *out = rdna2_dot_f16_ref(a, b, n); + return RDNA2_OK; +#else + return RDNA2_ERR_HIP; // host has no _Float16: reference unavailable +#endif + } + unsigned* da = nullptr; + unsigned* db = nullptr; + float* dout = nullptr; + int rc = RDNA2_OK; + const size_t bytes = (size_t)n * sizeof(unsigned); + if (hipMalloc(&da, bytes) != hipSuccess || hipMalloc(&db, bytes) != hipSuccess || + hipMalloc(&dout, sizeof(float)) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(da, a, bytes, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(db, b, bytes, hipMemcpyHostToDevice) != hipSuccess || + hipMemset(dout, 0, sizeof(float)) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + { + const int threads = 256; + int blocks = (n + threads - 1) / threads; + if (blocks > 1024) blocks = 1024; + hipLaunchKernelGGL(rdna2_dot2_f16, dim3(blocks), dim3(threads), 0, 0, + dout, reinterpret_cast(da), + reinterpret_cast(db), n); + if (hipGetLastError() != hipSuccess || + hipMemcpy(out, dout, sizeof(float), hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } + } +cleanup: + if (da) (void)hipFree(da); + if (db) (void)hipFree(db); + if (dout) (void)hipFree(dout); + return rc; +} + +// Inclusive prefix sum over a single wave; n must satisfy 1 <= n <= warpSize. +int rdna2_prefix_host(int* out, const int* in, int n) { + if (!out || !in || n <= 0) return RDNA2_ERR_ARG; + const int ws = rdna2_warp_size(); + if (n > ws) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + rdna2_prefix_ref(out, in, n); + return RDNA2_OK; + } + int* din = nullptr; + int* dout = nullptr; + int rc = RDNA2_OK; + // dout is warpSize wide: the kernel writes one value per lane. + if (hipMalloc(&din, (size_t)n * sizeof(int)) != hipSuccess || + hipMalloc(&dout, (size_t)ws * sizeof(int)) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(din, in, (size_t)n * sizeof(int), hipMemcpyHostToDevice) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + hipLaunchKernelGGL(rdna2_dpp_prefix, dim3(1), dim3(ws), 0, 0, dout, din, n); + if (hipGetLastError() != hipSuccess || + hipMemcpy(out, dout, (size_t)n * sizeof(int), hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } +cleanup: + if (din) (void)hipFree(din); + if (dout) (void)hipFree(dout); + return rc; +} + +// --------------------------------------------------------------------------- +// Device information, for the test's report line. arch_ok is set to one of the +// RDNA2_ARCH_* codes (1 = gfx1031, 2 = gfx1030, 3 = gfx1201, 4 = gfx1200), or 0 +// when the running device is none of those. +// --------------------------------------------------------------------------- +int rdna2_device_info(char* name, size_t len, int* warp_size, int* arch_ok) { + if (warp_size) *warp_size = 0; + if (arch_ok) *arch_ok = 0; + if (rdna2_device_count() == 0) { + if (name && len) name[0] = '\0'; + return RDNA2_OK; + } + hipDeviceProp_t prop{}; + if (hipGetDeviceProperties(&prop, 0) != hipSuccess) return RDNA2_ERR_HIP; + if (name && len) { + size_t i = 0; + for (; i + 1 < len && prop.name[i]; ++i) name[i] = prop.name[i]; + name[i] = '\0'; + } + if (warp_size) *warp_size = prop.warpSize; + if (arch_ok) { + // "gfx1031", "gfx1201:xnack-" and friends: match the six target digits only. + const char* gcn = prop.gcnArchName; + if (gcn && gcn[0] == 'g' && gcn[1] == 'f' && gcn[2] == 'x') { + if (gcn[3] == '1' && gcn[4] == '0' && gcn[5] == '3') { + if (gcn[6] == '1') *arch_ok = RDNA2_ARCH_GFX1031; + else if (gcn[6] == '0') *arch_ok = RDNA2_ARCH_GFX1030; + } else if (gcn[3] == '1' && gcn[4] == '2' && gcn[5] == '0') { + if (gcn[6] == '1') *arch_ok = RDNA2_ARCH_GFX1201; + else if (gcn[6] == '0') *arch_ok = RDNA2_ARCH_GFX1200; + } + } + } + return RDNA2_OK; +} + +// --------------------------------------------------------------------------- +// Architecture selection +// --------------------------------------------------------------------------- +// The 128x128 / 8x8 shapes (tiled == 4, sgemm variant == 2) were written for +// RDNA4 on the theory that the bigger register file wants a bigger tile. The +// 9070 XT disagrees: measured against the 64x64 8x4 kernel at 1024^3, 2048^2, +// 4096^2 and 2048^2 x K=4096, the 128 tile is 8-33% slower for int8 (82 VGPRs +// per thread plus 17408 B of LDS per block caps residency at 3 blocks, and the +// dp4a issue rate is already limited by LDS bandwidth), and for f32 it only +// wins while the operand working set stays inside the cache. So both RDNA4 +// parts keep the 64x64 8x4 int8 shape and the 64x64 LDS f32 shape; +// tiled == 4 / variant == 2 stay selectable and are still what the test suite +// exercises on every device. Numbers are in README.md. +// +// RDNA4 then overrides that for int8: the matrix instruction (tiled == 5) on the +// same 128x128 tile beat the 64x64 8x4 dp4a kernel by 3.1x at 1024^3, 3.3x at +// 2048^2 and 3.1x at 4096^3 on the 9070 XT (29.4/40.6/40.1 TOPS -> +// 91.3/134.0/125.0), so gfx1200/gfx1201 take tiled == 5 and RDNA2 keeps dp4a. +int rdna2_tiled_for_arch(int arch) { + if (arch == RDNA2_ARCH_GFX1200 || arch == RDNA2_ARCH_GFX1201) { + return 5; // 128x128 tile, one 16x16x16 matrix instruction per k step + } + if (arch == RDNA2_ARCH_GFX1030 || arch == RDNA2_ARCH_GFX1031) { + return 3; // 64x64 tile, 8x4 outputs per thread, LDS staged K chunks + } + return 1; // unknown AMD GPU: 4x4 tiled, global loads +} + +// variant 2 is measurably faster on RDNA4 only while the two 128-row operand +// panels stay cache-resident (K <= 2048 at 2048^2); past that it loses up to 2x +// to the 64x64 shape, so it is opt-in rather than the arch default. +// variant 3 is the 64x64 tile with a 16-deep K step (KSTEP=16, stride 17, four +// floats staged per thread). On the 9070 XT it beat variant 1 at every shape +// tried (1024^3, 1000^3, 2048^2 x K=2048/3072/4096/8192, 2304^2, 3584^2, 4096^3) +// by 3-42%, the margin shrinking as the problem grows, so it is the default on +// the RDNA2 and RDNA4 parts and the generic 64x64 shape is kept for anything +// unrecognised. +int rdna2_sgemm_variant_for_arch(int arch) { + if (arch == RDNA2_ARCH_GFX1030 || arch == RDNA2_ARCH_GFX1031 || + arch == RDNA2_ARCH_GFX1200 || arch == RDNA2_ARCH_GFX1201) { + return 3; // 64x64 tile, 16-deep K step, four floats staged per thread + } + return 1; // unknown AMD GPU: 64x64 LDS staging, 8-deep K step +} + +// The arch-aware entry points: same contract as rdna2_gemm_i8_host / +// rdna2_sgemm_f32_host, but the tile shape follows the device that is actually +// present instead of a fixed default. On a machine with no GPU both fall back to +// the CPU reference exactly like the plain entry points do. +int rdna2_gemm_i8_host_tuned(int* C, const int* A, const int* Bt, int M, int N, int K4) { + int arch = RDNA2_ARCH_UNKNOWN; + (void)rdna2_device_info(nullptr, 0, nullptr, &arch); + return rdna2_gemm_i8_host(C, A, Bt, M, N, K4, rdna2_tiled_for_arch(arch)); +} + +int rdna2_gemm_i8_scale_host_tuned(float* C, const int* A, const int* Bt, int M, int N, int K4, + const float* sa, const float* sb, int zp_b) { + int arch = RDNA2_ARCH_UNKNOWN; + (void)rdna2_device_info(nullptr, 0, nullptr, &arch); + return rdna2_gemm_i8_scale_host(C, A, Bt, M, N, K4, rdna2_tiled_for_arch(arch), sa, sb, zp_b); +} + +int rdna2_sgemm_f32_host_tuned(float* C, const float* A, const float* Bt, int M, int N, int K) { + int arch = RDNA2_ARCH_UNKNOWN; + (void)rdna2_device_info(nullptr, 0, nullptr, &arch); + return rdna2_sgemm_f32_host_v(C, A, Bt, M, N, K, rdna2_sgemm_variant_for_arch(arch)); +} + diff --git a/gpu/rdna2/rdna2_kernels_math.h b/gpu/rdna2/rdna2_kernels_math.h new file mode 100644 index 0000000000..5e44f92b30 --- /dev/null +++ b/gpu/rdna2/rdna2_kernels_math.h @@ -0,0 +1,245 @@ +// Kernel arithmetic shared by the device kernels and the host test. +// +// These functions contain no HIP types and no wave intrinsics, so the exact +// same source compiles into the GPU kernels (rdna2_kernels.hip) and runs on the +// host in test_rdna2_host.cpp. That is how the tiling, bounds handling and +// accumulation order of the GEMM/GEMV kernels get executed on a machine with no +// GPU: the test reproduces the kernel's grid and calls the same body. +// +// What this does NOT cover: wave-level behaviour (shuffles, DPP, atomics) and +// anything the compiler does differently for the device. Those need hardware. +#ifndef RDNA2_KERNELS_MATH_H +#define RDNA2_KERNELS_MATH_H + +#include +#include + +#include "rdna2_gemm_ref.h" + +// LDS tiling shape shared by the devices kernels and the host simulation: a +// block of 16x16 threads owns a 64x64 output tile and stages a K chunk of both +// operands in LDS. The row stride is padded by one element so that the lanes of +// a wave, which read rows 4 apart, do not all land on the same LDS banks. +#define RDNA2_TILE_MN 64 +#define RDNA2_LDS_I8_KSTEP 16 // packed words staged per chunk +#define RDNA2_LDS_I8_STRIDE (RDNA2_LDS_I8_KSTEP + 1) +#define RDNA2_LDS_F32_KSTEP 8 +#define RDNA2_LDS_F32_STRIDE (RDNA2_LDS_F32_KSTEP + 1) +// Wider f32 K chunk: 16 floats per tile row, so a chunk touches a full 64-byte +// line of each row instead of half of one, and the K loop pays half as many +// __syncthreads. Two 64x17 tiles are 8704 bytes, the same LDS as the int8 8x4 +// kernel. Select with variant 3 (see README for the measurement). +#define RDNA2_LDS_F32B_KSTEP 16 +#define RDNA2_LDS_F32B_STRIDE (RDNA2_LDS_F32B_KSTEP + 1) + +// RDNA4 (gfx1200 / gfx1201, e.g. RX 9070 XT) shape: same 16x16 block of 256 +// threads and the same K chunk, but a 128x128 output tile and 8x8 outputs per +// thread. gfx12 has 1536 VGPRs per SIMD against 1024 on gfx10/11, so 64 live +// accumulators plus 16 staged operands still leave room for several waves per +// SIMD. Sixteen LDS reads per k step now feed 64 dp4a instead of 8 feeding 16. +#define RDNA2_T4_TILE_MN 128 +#define RDNA2_T4_LDS_I8_KSTEP 16 +#define RDNA2_T4_LDS_I8_STRIDE (RDNA2_T4_LDS_I8_KSTEP + 1) +#define RDNA2_T4_LDS_F32_KSTEP 8 +#define RDNA2_T4_LDS_F32_STRIDE (RDNA2_T4_LDS_F32_KSTEP + 1) +// Column groups of a 128-wide tile: one per 16x16x16 matrix instruction, so a +// wave carries eight accumulators in the tiled = 5 kernel. +#define RDNA2_T4_WMMA_NTILE (RDNA2_T4_TILE_MN / 16) + +// RDNA4 matrix-instruction fragments (tiled = 5). +// V_WMMA_I32_16X16X16_IU8 spreads a 16x16x16 signed product over a wave32 like +// this (measured on gfx1201, 256/256 outputs matched a scalar reference): +// A: lane t, byte e -> A[M = t%16][K = (t/16)*8 + e] +// B: lane t, byte e -> B[K = (t/16)*8 + e][N = t%16] +// D: lane t, slot l -> D[M = (t/16)*8 + l][N = t%16] +// The LDS tile holds packed words (four int8 along K per word), so a lane reads +// two consecutive words of row t%16, i.e. bytes 8*(t/16)..+7 of a 16-byte K +// step. These helpers keep the kernel and the host model on one set of indices; +// the instruction semantics themselves are the hardware's, not this file's. +#define RDNA2_WMMA_I8_KSTEP_WORDS 4 // staged packed words per 16-byte K step +// Staged row a lane takes its fragment from: A's M index and B's N index are +// both lane%16, and both operands stage rows of one tile, so the caller adds the +// wave's 16-row band or the column group it is working on. +RDNA2_HD static inline int rdna2_wmma_i8_row(int lane) { return lane & 15; } +// First of the two staged words a lane reads for K step `step` of a chunk. +RDNA2_HD static inline int rdna2_wmma_i8_frag_word(int lane, int step) { + return RDNA2_WMMA_I8_KSTEP_WORDS * step + 2 * (lane >> 4); +} +// Output row (slot l) and column a lane's accumulators hold in a 16x16 result. +RDNA2_HD static inline int rdna2_wmma_i8_acc_row(int lane, int slot) { + return 8 * (lane >> 4) + slot; +} +RDNA2_HD static inline int rdna2_wmma_i8_acc_col(int lane) { return lane & 15; } + +// Target codes reported by rdna2_device_info(..., arch_ok) and consumed by the +// rdna2_*_for_arch selectors. 0 means "an AMD GPU from neither family"; the +// launchers still run on it with the generic 64x64 / 4x4 shapes. +#define RDNA2_ARCH_UNKNOWN 0 +#define RDNA2_ARCH_GFX1031 1 // RDNA2, Navi 22 (RX 6700 XT / 6750 XT) +#define RDNA2_ARCH_GFX1030 2 // RDNA2, Navi 21 (RX 6800 / 6900) +#define RDNA2_ARCH_GFX1201 3 // RDNA4, Navi 48 (RX 9070 XT / 9070) +#define RDNA2_ARCH_GFX1200 4 // RDNA4, Navi 44 (RX 9060 XT) + +// One row of the dp4a int8 GEMV: y[row] = sum over k4 of dot4(A_row[k], x[k]). +RDNA2_HD static inline int32_t rdna2_gemv_row_i8(const int32_t* a_row, const int32_t* x, + int k4) { + int32_t acc = 0; + for (int j = 0; j < k4; ++j) acc = rdna2_dot4_i8_impl(a_row[j], x[j], acc); + return acc; +} + +// One output element of the int8 GEMM. +RDNA2_HD static inline int32_t rdna2_gemm_cell_i8(const int32_t* a_row, const int32_t* b_row, + int K4) { + int32_t acc = 0; + for (int k = 0; k < K4; ++k) acc = rdna2_dot4_i8_impl(a_row[k], b_row[k], acc); + return acc; +} + +// One thread's 4x4 output block of the tiled int8 GEMM, starting at (m0, n0). +// Rows/columns past M/N read 0 so the accumulation stays exact at the edges; +// the caller only stores the in-range elements. +RDNA2_HD static inline void rdna2_gemm_tile_i8(int32_t acc[4][4], const int32_t* A, + const int32_t* Bt, int M, int N, int K4, + int m0, int n0) { + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = 0; + for (int k = 0; k < K4; ++k) { + int32_t a[4]; + int32_t b[4]; + for (int i = 0; i < 4; ++i) a[i] = (m0 + i < M) ? A[(size_t)(m0 + i) * K4 + k] : 0; + for (int j = 0; j < 4; ++j) b[j] = (n0 + j < N) ? Bt[(size_t)(n0 + j) * K4 + k] : 0; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = rdna2_dot4_i8_impl(a[i], b[j], acc[i][j]); + } +} + +// One thread's 4x4 output block of the f32 GEMM, same blocking as the int8 one. +RDNA2_HD static inline void rdna2_sgemm_tile_f32(float acc[4][4], const float* A, + const float* Bt, int M, int N, int K, + int m0, int n0) { + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = 0.0f; + for (int k = 0; k < K; ++k) { + float a[4]; + float b[4]; + for (int i = 0; i < 4; ++i) a[i] = (m0 + i < M) ? A[(size_t)(m0 + i) * K + k] : 0.0f; + for (int j = 0; j < 4; ++j) b[j] = (n0 + j < N) ? Bt[(size_t)(n0 + j) * K + k] : 0.0f; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = __builtin_fmaf(a[i], b[j], acc[i][j]); + } +} + +// --- LDS-staged variants --------------------------------------------------- +// The tile pointer types are template parameters on purpose: on the device they +// deduce to __shared__-space pointers (so the compiler keeps the accesses as +// ds_read/ds_write instead of flat ones), on the host they are plain pointers. +// The tile is row-major with `ld` words per row holding `kwords` K columns; +// rows past the matrix edge are already zero, so no bounds checks are needed. +// acc is accumulated into, not zeroed, so a K loop can stream tile chunks in. +template +RDNA2_HD static inline void rdna2_gemm_lds_block_i8(int32_t acc[4][4], AT a_tile, BT b_tile, + int ld, int lm, int ln, int kwords) { + for (int kk = 0; kk < kwords; ++kk) { + int32_t a[4]; + int32_t b[4]; + for (int i = 0; i < 4; ++i) a[i] = a_tile[(lm + i) * ld + kk]; + for (int j = 0; j < 4; ++j) b[j] = b_tile[(ln + j) * ld + kk]; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = rdna2_dot4_i8_impl(a[i], b[j], acc[i][j]); + } +} + +// 8x4 register blocking: 12 LDS reads feed 32 dp4a instead of 8 feeding 16, so +// the LDS traffic per multiply-accumulate drops by a quarter at the same math +// count. Register cost is 32 accumulators plus the 12 staged operands. +template +RDNA2_HD static inline void rdna2_gemm_lds_block8_i8(int32_t acc[8][4], AT a_tile, BT b_tile, + int ld, int lm, int ln, int kwords) { + for (int kk = 0; kk < kwords; ++kk) { + int32_t a[8]; + int32_t b[4]; + for (int i = 0; i < 8; ++i) a[i] = a_tile[(lm + i) * ld + kk]; + for (int j = 0; j < 4; ++j) b[j] = b_tile[(ln + j) * ld + kk]; + for (int i = 0; i < 8; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = rdna2_dot4_i8_impl(a[i], b[j], acc[i][j]); + } +} + +// 8x8 register blocking for the 128x128 tile. Sixteen LDS reads per k step feed +// 64 dp4a, i.e. 4 multiplies per LDS read against 2.67 for 8x4 and 2 for 4x4. +template +RDNA2_HD static inline void rdna2_gemm_lds_block8x8_i8(int32_t acc[8][8], AT a_tile, BT b_tile, + int ld, int lm, int ln, int kwords) { + for (int kk = 0; kk < kwords; ++kk) { + int32_t a[8]; + int32_t b[8]; + for (int i = 0; i < 8; ++i) a[i] = a_tile[(lm + i) * ld + kk]; + for (int j = 0; j < 8; ++j) b[j] = b_tile[(ln + j) * ld + kk]; + for (int i = 0; i < 8; ++i) + for (int j = 0; j < 8; ++j) acc[i][j] = rdna2_dot4_i8_impl(a[i], b[j], acc[i][j]); + } +} + +template +RDNA2_HD static inline void rdna2_sgemm_lds_block8x8_f32(float acc[8][8], AT a_tile, BT b_tile, + int ld, int lm, int ln, int kwords) { + for (int kk = 0; kk < kwords; ++kk) { + float a[8]; + float b[8]; + for (int i = 0; i < 8; ++i) a[i] = a_tile[(lm + i) * ld + kk]; + for (int j = 0; j < 8; ++j) b[j] = b_tile[(ln + j) * ld + kk]; + for (int i = 0; i < 8; ++i) + for (int j = 0; j < 8; ++j) acc[i][j] = __builtin_fmaf(a[i], b[j], acc[i][j]); + } +} + +template +RDNA2_HD static inline void rdna2_sgemm_lds_block_f32(float acc[4][4], AT a_tile, BT b_tile, + int ld, int lm, int ln, int kwords) { + for (int kk = 0; kk < kwords; ++kk) { + float a[4]; + float b[4]; + for (int i = 0; i < 4; ++i) a[i] = a_tile[(lm + i) * ld + kk]; + for (int j = 0; j < 4; ++j) b[j] = b_tile[(ln + j) * ld + kk]; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = __builtin_fmaf(a[i], b[j], acc[i][j]); + } +} + +// One lane's contribution to a row when a whole wave cooperates on that row +// (lanes stride the row, so consecutive lanes read consecutive words). The +// caller reduces the result across the wave. +RDNA2_HD static inline int32_t rdna2_gemv_lane_partial_i8(const int32_t* a_row, const int32_t* x, + int k4, int lane, int wave_size) { + int32_t acc = 0; + for (int k = lane; k < k4; k += wave_size) acc = rdna2_dot4_i8_impl(a_row[k], x[k], acc); + return acc; +} + +// One row of the f32 GEMV: y[row] = sum over k of A_row[k] * x[k]. Four +// independent accumulators, because a single fma chain would leave the fma +// pipes waiting on the previous result; the final sum fixes the evaluation +// order, so the kernel and the host simulation agree exactly. +RDNA2_HD static inline float rdna2_sgemv_row_f32(const float* a_row, const float* x, int K) { + float acc0 = 0.0f, acc1 = 0.0f, acc2 = 0.0f, acc3 = 0.0f; + int k = 0; + for (; k + 4 <= K; k += 4) { + acc0 = __builtin_fmaf(a_row[k + 0], x[k + 0], acc0); + acc1 = __builtin_fmaf(a_row[k + 1], x[k + 1], acc1); + acc2 = __builtin_fmaf(a_row[k + 2], x[k + 2], acc2); + acc3 = __builtin_fmaf(a_row[k + 3], x[k + 3], acc3); + } + for (; k < K; ++k) acc0 = __builtin_fmaf(a_row[k], x[k], acc0); + return (acc0 + acc1) + (acc2 + acc3); +} + +// One lane's contribution to a row when a wave cooperates on that row. +RDNA2_HD static inline float rdna2_sgemv_lane_partial_f32(const float* a_row, const float* x, + int K, int lane, int wave_size) { + float acc = 0.0f; + for (int k = lane; k < K; k += wave_size) acc = __builtin_fmaf(a_row[k], x[k], acc); + return acc; +} + +#endif // RDNA2_KERNELS_MATH_H diff --git a/gpu/rdna2/test_rdna2_host.cpp b/gpu/rdna2/test_rdna2_host.cpp new file mode 100644 index 0000000000..09506fecd4 --- /dev/null +++ b/gpu/rdna2/test_rdna2_host.cpp @@ -0,0 +1,1123 @@ +// Correctness test for the RDNA2 kernel launchers. +// +// On a machine with a RDNA2 GPU, every check runs on the device (the launchers +// pick the device path automatically) and is compared against the plain C++ +// reference. With no GPU the same checks exercise the CPU fallback, so the +// binary is useful either way; the header line says which path ran. +// +// Build and run: see Makefile (make test). +#include +#include +#include +#include +#include +#include + +#include "rdna2_gemm_ref.h" +#include "rdna2_kernels_math.h" + +// Launchers from rdna2_kernels.hip. +int rdna2_gemv_i8_host(int* y, const int* A, const int* x, int rows, int k4); +int rdna2_gemv_i8_host_variant(int* y, const int* A, const int* x, int rows, int k4, int variant); +int rdna2_gemm_i8_host(int* C, const int* A, const int* Bt, int M, int N, int K4, int tiled); +int rdna2_gemm_i8_scale_host(float* C, const int* A, const int* Bt, int M, int N, int K4, + int tiled, const float* sa, const float* sb, int zp_b); +int rdna2_gemm_i8_scale_host_tuned(float* C, const int* A, const int* Bt, int M, int N, int K4, + const float* sa, const float* sb, int zp_b); +int rdna2_sgemm_f32_host(float* C, const float* A, const float* Bt, int M, int N, int K); +int rdna2_sgemm_f32_host_v(float* C, const float* A, const float* Bt, int M, int N, int K, + int variant); +int rdna2_sgemv_f32_host(float* y, const float* A, const float* x, int rows, int K); +int rdna2_sgemv_f32_host_variant(float* y, const float* A, const float* x, int rows, int K, + int variant); +int rdna2_dot_u8_host(unsigned* out, const unsigned* a, const unsigned* b, int n); +int rdna2_dot_f16_host(float* out, const unsigned* a, const unsigned* b, int n); +int rdna2_prefix_host(int* out, const int* in, int n); +int rdna2_device_info(char* name, size_t len, int* warp_size, int* arch_ok); +int rdna2_gemm_i8_host_tuned(int* C, const int* A, const int* Bt, int M, int N, int K4); +int rdna2_sgemm_f32_host_tuned(float* C, const float* A, const float* Bt, int M, int N, int K); +int rdna2_tiled_for_arch(int arch); +int rdna2_sgemm_variant_for_arch(int arch); + +static int failures = 0; +static int checks = 0; + +// Written nowhere if a kernel misses an output element. +static const int32_t SENTINEL = 0x7eadbeef; + +#define CHECK(cond, msg) \ + do { \ + ++checks; \ + if (!(cond)) { \ + std::printf("FAIL: %s (line %d)\n", msg, __LINE__); \ + ++failures; \ + } \ + } while (0) + +// Deterministic pseudo-random int32 (full 32-bit range, so every byte pattern +// including negative int8 values appears). +static uint32_t rng_state = 12345u; +static uint32_t rng(void) { + rng_state ^= rng_state << 13; + rng_state ^= rng_state >> 17; + rng_state ^= rng_state << 5; + return rng_state; +} + +// 16-byte aligned view of a vector, optionally offset by `misalign` bytes. +static int32_t* align16(std::vector& store, size_t n, size_t misalign = 0) { + store.assign(n + 8, 0); + uintptr_t p = reinterpret_cast(store.data()); + uintptr_t base = (p + 15) & ~static_cast(15); + return reinterpret_cast(base + misalign); +} + +// --------------------------------------------------------------------------- +// Host simulation of the device kernels. +// +// These reproduce the grid, block size and per-thread indexing of the kernels +// in rdna2_kernels.hip and call the same per-thread body from +// rdna2_kernels_math.h. On a machine with no GPU this is the only way to run +// the tiling/bounds logic at all; it does not model waves, shuffles or atomics. +// Out-of-range outputs are left untouched, so the caller can seed the result +// buffer with a sentinel to detect stray writes. +// --------------------------------------------------------------------------- +static void sim_gemm_tile_i8(int32_t* C, const int32_t* A, const int32_t* Bt, int M, int N, + int K4) { + const int gx = (N + 63) / 64, gy = (M + 63) / 64; // block(16,16), 4x4 per thread + for (int by = 0; by < gy; ++by) + for (int bx = 0; bx < gx; ++bx) + for (int ty = 0; ty < 16; ++ty) + for (int tx = 0; tx < 16; ++tx) { + const int m0 = (by * 16 + ty) * 4; + const int n0 = (bx * 16 + tx) * 4; + if (m0 >= M || n0 >= N) continue; + int32_t acc[4][4]; + rdna2_gemm_tile_i8(acc, A, Bt, M, N, K4, m0, n0); + for (int i = 0; i < 4 && m0 + i < M; ++i) + for (int j = 0; j < 4 && n0 + j < N; ++j) + C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } +} + +static void sim_sgemm_tile_f32(float* C, const float* A, const float* Bt, int M, int N, int K) { + const int gx = (N + 63) / 64, gy = (M + 63) / 64; + for (int by = 0; by < gy; ++by) + for (int bx = 0; bx < gx; ++bx) + for (int ty = 0; ty < 16; ++ty) + for (int tx = 0; tx < 16; ++tx) { + const int m0 = (by * 16 + ty) * 4; + const int n0 = (bx * 16 + tx) * 4; + if (m0 >= M || n0 >= N) continue; + float acc[4][4]; + rdna2_sgemm_tile_f32(acc, A, Bt, M, N, K, m0, n0); + for (int i = 0; i < 4 && m0 + i < M; ++i) + for (int j = 0; j < 4 && n0 + j < N; ++j) + C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } +} + +static void sim_gemv_i8(int32_t* y, const int32_t* A, const int32_t* x, int rows, int k4) { + for (int r = 0; r < rows; ++r) y[r] = rdna2_gemv_row_i8(A + (size_t)r * k4, x, k4); +} + +// Wave-per-row GEMV: 128 threads per block, so blockDim/warp_size independent +// rows per block, lanes striding the row. The shuffle reduction is modelled as +// the exact sum of the lane partials (integer adds, so order cannot matter). +static void sim_gemv_wave_i8(int32_t* y, const int32_t* A, const int32_t* x, int rows, int k4, + int warp_size) { + const int threads = 128; + const int waves = threads / warp_size; + const int blocks = (rows + waves - 1) / waves; + for (int b = 0; b < blocks; ++b) + for (int w = 0; w < waves; ++w) { + const int row = b * waves + w; + if (row >= rows) continue; + int64_t sum = 0; + for (int lane = 0; lane < warp_size; ++lane) + sum += rdna2_gemv_lane_partial_i8(A + (size_t)row * k4, x, k4, lane, warp_size); + y[row] = (int32_t)sum; + } +} + +static void sim_sgemv_f32(float* y, const float* A, const float* x, int rows, int K) { + const int threads = 64; + const int blocks = (rows + threads - 1) / threads; + for (int b = 0; b < blocks; ++b) + for (int t = 0; t < threads; ++t) { + const int row = b * threads + t; + if (row >= rows) continue; + y[row] = rdna2_sgemv_row_f32(A + (size_t)row * K, x, K); + } +} + +// The device reduces with __shfl_xor, so its grouping differs from this serial +// sum of the lane partials; the test compares with a tolerance, not bit-for-bit. +static void sim_sgemv_wave_f32(float* y, const float* A, const float* x, int rows, int K, + int warp_size) { + const int threads = 128; + const int waves = threads / warp_size; + const int blocks = (rows + waves - 1) / waves; + for (int b = 0; b < blocks; ++b) + for (int w = 0; w < waves; ++w) { + const int row = b * waves + w; + if (row >= rows) continue; + float sum = 0.0f; + for (int lane = 0; lane < warp_size; ++lane) + sum += rdna2_sgemv_lane_partial_f32(A + (size_t)row * K, x, K, lane, warp_size); + y[row] = sum; + } +} + +// Staging helper of rdna2_stage_i8 / rdna2_stage_f32: one thread copies 4 (i8) +// or 2 (f32) consecutive words of one tile row, writing zeros past the edges. +static void sim_stage_i8(int32_t* lds, const int32_t* src, int nrows, int ncols, int row0, int k0, + int ld_row, int ld_col) { + const int gm = row0 + ld_row; + int32_t v[4] = {0, 0, 0, 0}; + if (gm < nrows) { + const int32_t* p = src + (size_t)gm * ncols + k0 + ld_col; + for (int c = 0; c < 4; ++c) + if (k0 + ld_col + c < ncols) v[c] = p[c]; + } + for (int c = 0; c < 4; ++c) lds[(size_t)ld_row * RDNA2_LDS_I8_STRIDE + ld_col + c] = v[c]; +} + +static void sim_stage_f32(float* lds, const float* src, int nrows, int ncols, int row0, int k0, + int ld_row, int ld_col) { + const int gm = row0 + ld_row; + float v[2] = {0.0f, 0.0f}; + if (gm < nrows) { + const float* p = src + (size_t)gm * ncols + k0 + ld_col; + for (int c = 0; c < 2; ++c) + if (k0 + ld_col + c < ncols) v[c] = p[c]; + } + for (int c = 0; c < 2; ++c) lds[(size_t)ld_row * RDNA2_LDS_F32_STRIDE + ld_col + c] = v[c]; +} + +static void sim_stage_f32_4(float* lds, const float* src, int nrows, int ncols, int row0, int k0, + int ld_row, int ld_col) { + const int gm = row0 + ld_row; + float v[4] = {0.0f, 0.0f, 0.0f, 0.0f}; + if (gm < nrows) { + const float* p = src + (size_t)gm * ncols + k0 + ld_col; + for (int c = 0; c < 4; ++c) + if (k0 + ld_col + c < ncols) v[c] = p[c]; + } + for (int c = 0; c < 4; ++c) lds[(size_t)ld_row * RDNA2_LDS_F32B_STRIDE + ld_col + c] = v[c]; +} + +// LDS-staged int8 GEMM: same output tile and register blocking as the plain +// tiled kernel, but each K chunk is staged for the whole block first (modelling +// the two __syncthreads) and then read back per thread. +static void sim_gemm_lds_i8(int32_t* C, const int32_t* A, const int32_t* Bt, int M, int N, + int K4) { + std::vector at((size_t)RDNA2_TILE_MN * RDNA2_LDS_I8_STRIDE); + std::vector bt(at.size()); + std::vector accs((size_t)256 * 16); + const int gx = (N + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN; + const int gy = (M + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN; + for (int by = 0; by < gy; ++by) + for (int bx = 0; bx < gx; ++bx) { + const int m_blk = by * RDNA2_TILE_MN, n_blk = bx * RDNA2_TILE_MN; + for (auto& v : accs) v = 0; + for (int k0 = 0; k0 < K4; k0 += RDNA2_LDS_I8_KSTEP) { + for (int tid = 0; tid < 256; ++tid) { + sim_stage_i8(at.data(), A, M, K4, m_blk, k0, tid >> 2, (tid & 3) << 2); + sim_stage_i8(bt.data(), Bt, N, K4, n_blk, k0, tid >> 2, (tid & 3) << 2); + } + int kend = K4 - k0; + if (kend > RDNA2_LDS_I8_KSTEP) kend = RDNA2_LDS_I8_KSTEP; + for (int tid = 0; tid < 256; ++tid) { + int32_t(*acc)[4] = reinterpret_cast(&accs[(size_t)tid * 16]); + rdna2_gemm_lds_block_i8(acc, at.data(), bt.data(), RDNA2_LDS_I8_STRIDE, (tid >> 4) * 4, + (tid & 15) * 4, kend); + } + } + for (int tid = 0; tid < 256; ++tid) { + const int m0 = m_blk + (tid >> 4) * 4, n0 = n_blk + (tid & 15) * 4; + const int32_t(*acc)[4] = reinterpret_cast(&accs[(size_t)tid * 16]); + for (int i = 0; i < 4 && m0 + i < M; ++i) + for (int j = 0; j < 4 && n0 + j < N; ++j) C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } + } +} + +// Same LDS staging, 8x4 register blocking: 128 threads per 64x64 tile and two +// staged words per thread per half-row group. +static void sim_gemm_lds8_i8(int32_t* C, const int32_t* A, const int32_t* Bt, int M, int N, + int K4) { + std::vector at((size_t)RDNA2_TILE_MN * RDNA2_LDS_I8_STRIDE); + std::vector bt(at.size()); + std::vector accs((size_t)128 * 32); + const int gx = (N + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN; + const int gy = (M + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN; + for (int by = 0; by < gy; ++by) + for (int bx = 0; bx < gx; ++bx) { + const int m_blk = by * RDNA2_TILE_MN, n_blk = bx * RDNA2_TILE_MN; + for (auto& v : accs) v = 0; + for (int k0 = 0; k0 < K4; k0 += RDNA2_LDS_I8_KSTEP) { + for (int tid = 0; tid < 128; ++tid) { + sim_stage_i8(at.data(), A, M, K4, m_blk, k0, tid >> 1, (tid & 1) << 3); + sim_stage_i8(at.data(), A, M, K4, m_blk, k0, tid >> 1, ((tid & 1) << 3) + 4); + sim_stage_i8(bt.data(), Bt, N, K4, n_blk, k0, tid >> 1, (tid & 1) << 3); + sim_stage_i8(bt.data(), Bt, N, K4, n_blk, k0, tid >> 1, ((tid & 1) << 3) + 4); + } + int kend = K4 - k0; + if (kend > RDNA2_LDS_I8_KSTEP) kend = RDNA2_LDS_I8_KSTEP; + for (int tid = 0; tid < 128; ++tid) { + int32_t(*acc)[4] = reinterpret_cast(&accs[(size_t)tid * 32]); + rdna2_gemm_lds_block8_i8(acc, at.data(), bt.data(), RDNA2_LDS_I8_STRIDE, (tid >> 4) * 8, + (tid & 15) * 4, kend); + } + } + for (int tid = 0; tid < 128; ++tid) { + const int m0 = m_blk + (tid >> 4) * 8, n0 = n_blk + (tid & 15) * 4; + const int32_t(*acc)[4] = reinterpret_cast(&accs[(size_t)tid * 32]); + for (int i = 0; i < 8 && m0 + i < M; ++i) + for (int j = 0; j < 4 && n0 + j < N; ++j) C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } + } +} + +// RDNA4 shape: 128x128 tile, 256 threads, 16 staged words per row per K step, +// 8x8 accumulators each. The staging helper is the same one the 64x64 kernels +// use (the padded stride is 17 words on both shapes), only the row count grows. +static void sim_gemm_lds8x8_i8(int32_t* C, const int32_t* A, const int32_t* Bt, int M, int N, + int K4) { + std::vector at((size_t)RDNA2_T4_TILE_MN * RDNA2_T4_LDS_I8_STRIDE); + std::vector bt(at.size()); + std::vector accs((size_t)256 * 64); + const int gx = (N + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN; + const int gy = (M + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN; + for (int by = 0; by < gy; ++by) + for (int bx = 0; bx < gx; ++bx) { + const int m_blk = by * RDNA2_T4_TILE_MN, n_blk = bx * RDNA2_T4_TILE_MN; + for (auto& v : accs) v = 0; + for (int k0 = 0; k0 < K4; k0 += RDNA2_T4_LDS_I8_KSTEP) { + for (int tid = 0; tid < 256; ++tid) { + sim_stage_i8(at.data(), A, M, K4, m_blk, k0, tid >> 1, (tid & 1) << 3); + sim_stage_i8(at.data(), A, M, K4, m_blk, k0, tid >> 1, ((tid & 1) << 3) + 4); + sim_stage_i8(bt.data(), Bt, N, K4, n_blk, k0, tid >> 1, (tid & 1) << 3); + sim_stage_i8(bt.data(), Bt, N, K4, n_blk, k0, tid >> 1, ((tid & 1) << 3) + 4); + } + int kend = K4 - k0; + if (kend > RDNA2_T4_LDS_I8_KSTEP) kend = RDNA2_T4_LDS_I8_KSTEP; + for (int tid = 0; tid < 256; ++tid) { + int32_t(*acc)[8] = reinterpret_cast(&accs[(size_t)tid * 64]); + rdna2_gemm_lds_block8x8_i8(acc, at.data(), bt.data(), RDNA2_T4_LDS_I8_STRIDE, + (tid >> 4) * 8, (tid & 15) * 8, kend); + } + } + for (int tid = 0; tid < 256; ++tid) { + const int m0 = m_blk + (tid >> 4) * 8, n0 = n_blk + (tid & 15) * 8; + const int32_t(*acc)[8] = reinterpret_cast(&accs[(size_t)tid * 64]); + for (int i = 0; i < 8 && m0 + i < M; ++i) + for (int j = 0; j < 8 && n0 + j < N; ++j) C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } + } +} + +// Wave-level model of V_WMMA_I32_16X16X16_IU8 (signed x signed, no clamp): one +// call is one wave executing one matrix instruction, acc is [lane][slot]. What +// is modeled is the operand/result placement contract the kernel is written +// against, stated once in rdna2_kernels_math.h and measured on a gfx1201 card +// (256/256 outputs of a signed product). The multiplication itself is trivially +// a 16-deep int8 dot; the hardware is trusted for that, and the GPU run of the +// tiled = 5 test is what confirms this model against silicon. +static void sim_wmma_i8_wave_step(int32_t acc[32][8], const int32_t a_frag[32][2], + const int32_t b_frag[32][2]) { + for (int t = 0; t < 32; ++t) { + const int dn = rdna2_wmma_i8_acc_col(t); + for (int l = 0; l < 8; ++l) { + // D row this lane's slot l holds. Byte k of that row sits in lane + // row + 16*(k/8), at byte k%8 of that lane's eight-byte fragment. + const int dm = rdna2_wmma_i8_acc_row(t, l); + int32_t d = 0; + for (int k = 0; k < 16; ++k) { + const int8_t av = (int8_t)((a_frag[dm + (k >> 3) * 16][(k & 7) >> 2] >> (8 * (k & 3))) & 0xff); + const int8_t bv = (int8_t)((b_frag[dn + (k >> 3) * 16][(k & 7) >> 2] >> (8 * (k & 3))) & 0xff); + d += (int32_t)av * (int32_t)bv; + } + acc[t][l] += d; + } + } +} + +// The tiled = 5 kernel on the host: 128x128 tile, 256 threads, eight waves each +// owning a 16-row band and eight 16-column groups, staged exactly as the lds128 +// kernel stages. Vector arithmetic is unrolled to whole-wave arrays here. +static void sim_gemm_wmma_i8(int32_t* C, const int32_t* A, const int32_t* Bt, int M, int N, int K4) { + const int S = RDNA2_T4_LDS_I8_STRIDE; + std::vector at((size_t)RDNA2_T4_TILE_MN * S); + std::vector bt(at.size()); + // [wave][column group][lane][slot] + std::vector accs((size_t)8 * RDNA2_T4_WMMA_NTILE * 32 * 8); + int32_t af[32][2], bf[32][2]; + const int gx = (N + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN; + const int gy = (M + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN; + for (int by = 0; by < gy; ++by) + for (int bx = 0; bx < gx; ++bx) { + const int m_blk = by * RDNA2_T4_TILE_MN, n_blk = bx * RDNA2_T4_TILE_MN; + for (auto& v : accs) v = 0; + for (int k0 = 0; k0 < K4; k0 += RDNA2_T4_LDS_I8_KSTEP) { + for (int tid = 0; tid < 256; ++tid) { + sim_stage_i8(at.data(), A, M, K4, m_blk, k0, tid >> 1, (tid & 1) << 3); + sim_stage_i8(at.data(), A, M, K4, m_blk, k0, tid >> 1, ((tid & 1) << 3) + 4); + sim_stage_i8(bt.data(), Bt, N, K4, n_blk, k0, tid >> 1, (tid & 1) << 3); + sim_stage_i8(bt.data(), Bt, N, K4, n_blk, k0, tid >> 1, ((tid & 1) << 3) + 4); + } + int kend = K4 - k0; + if (kend > RDNA2_T4_LDS_I8_KSTEP) kend = RDNA2_T4_LDS_I8_KSTEP; + const int ksteps = (kend + RDNA2_WMMA_I8_KSTEP_WORDS - 1) / RDNA2_WMMA_I8_KSTEP_WORDS; + for (int wave = 0; wave < 8; ++wave) + for (int ks = 0; ks < ksteps; ++ks) { + for (int t = 0; t < 32; ++t) { + const int w = rdna2_wmma_i8_frag_word(t, ks); + const int r = wave * 16 + rdna2_wmma_i8_row(t); + af[t][0] = at[(size_t)r * S + w]; + af[t][1] = at[(size_t)r * S + w + 1]; + } + for (int j = 0; j < RDNA2_T4_WMMA_NTILE; ++j) { + for (int t = 0; t < 32; ++t) { + const int w = rdna2_wmma_i8_frag_word(t, ks); + const int r = j * 16 + rdna2_wmma_i8_row(t); + bf[t][0] = bt[(size_t)r * S + w]; + bf[t][1] = bt[(size_t)r * S + w + 1]; + } + int32_t(*acc)[8] = reinterpret_cast( + &accs[((size_t)(wave * RDNA2_T4_WMMA_NTILE + j) * 32) * 8]); + sim_wmma_i8_wave_step(acc, af, bf); + } + } + } + for (int wave = 0; wave < 8; ++wave) + for (int j = 0; j < RDNA2_T4_WMMA_NTILE; ++j) { + const int32_t(*acc)[8] = reinterpret_cast( + &accs[((size_t)(wave * RDNA2_T4_WMMA_NTILE + j) * 32) * 8]); + for (int t = 0; t < 32; ++t) { + const int n = n_blk + j * 16 + rdna2_wmma_i8_acc_col(t); + if (n >= N) continue; + for (int l = 0; l < 8; ++l) { + const int m = m_blk + wave * 16 + rdna2_wmma_i8_acc_row(t, l); + if (m >= M) break; // rows 8*(t/16)+l are contiguous, break is safe + C[(size_t)m * N + n] = acc[t][l]; + } + } + } + } +} + +static void sim_sgemm_lds_f32(float* C, const float* A, const float* Bt, int M, int N, int K) { + std::vector at((size_t)RDNA2_TILE_MN * RDNA2_LDS_F32_STRIDE); + std::vector bt(at.size()); + std::vector accs((size_t)256 * 16); + const int gx = (N + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN; + const int gy = (M + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN; + for (int by = 0; by < gy; ++by) + for (int bx = 0; bx < gx; ++bx) { + const int m_blk = by * RDNA2_TILE_MN, n_blk = bx * RDNA2_TILE_MN; + for (auto& v : accs) v = 0.0f; + for (int k0 = 0; k0 < K; k0 += RDNA2_LDS_F32_KSTEP) { + for (int tid = 0; tid < 256; ++tid) { + sim_stage_f32(at.data(), A, M, K, m_blk, k0, tid >> 2, (tid & 3) << 1); + sim_stage_f32(bt.data(), Bt, N, K, n_blk, k0, tid >> 2, (tid & 3) << 1); + } + int kend = K - k0; + if (kend > RDNA2_LDS_F32_KSTEP) kend = RDNA2_LDS_F32_KSTEP; + for (int tid = 0; tid < 256; ++tid) { + float(*acc)[4] = reinterpret_cast(&accs[(size_t)tid * 16]); + rdna2_sgemm_lds_block_f32(acc, at.data(), bt.data(), RDNA2_LDS_F32_STRIDE, (tid >> 4) * 4, + (tid & 15) * 4, kend); + } + } + for (int tid = 0; tid < 256; ++tid) { + const int m0 = m_blk + (tid >> 4) * 4, n0 = n_blk + (tid & 15) * 4; + const float(*acc)[4] = reinterpret_cast(&accs[(size_t)tid * 16]); + for (int i = 0; i < 4 && m0 + i < M; ++i) + for (int j = 0; j < 4 && n0 + j < N; ++j) C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } + } +} + +// Variant 3: same 64x64 tile and 4x4 blocking as the kernel above, but 16 +// floats staged per row per K step (four per thread) instead of two. +static void sim_sgemm_lds16_f32(float* C, const float* A, const float* Bt, int M, int N, int K) { + std::vector at((size_t)RDNA2_TILE_MN * RDNA2_LDS_F32B_STRIDE); + std::vector bt(at.size()); + std::vector accs((size_t)256 * 16); + const int gx = (N + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN; + const int gy = (M + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN; + for (int by = 0; by < gy; ++by) + for (int bx = 0; bx < gx; ++bx) { + const int m_blk = by * RDNA2_TILE_MN, n_blk = bx * RDNA2_TILE_MN; + for (auto& v : accs) v = 0.0f; + for (int k0 = 0; k0 < K; k0 += RDNA2_LDS_F32B_KSTEP) { + for (int tid = 0; tid < 256; ++tid) { + sim_stage_f32_4(at.data(), A, M, K, m_blk, k0, tid >> 2, (tid & 3) << 2); + sim_stage_f32_4(bt.data(), Bt, N, K, n_blk, k0, tid >> 2, (tid & 3) << 2); + } + int kend = K - k0; + if (kend > RDNA2_LDS_F32B_KSTEP) kend = RDNA2_LDS_F32B_KSTEP; + for (int tid = 0; tid < 256; ++tid) { + float(*acc)[4] = reinterpret_cast(&accs[(size_t)tid * 16]); + rdna2_sgemm_lds_block_f32(acc, at.data(), bt.data(), RDNA2_LDS_F32B_STRIDE, (tid >> 4) * 4, + (tid & 15) * 4, kend); + } + } + for (int tid = 0; tid < 256; ++tid) { + const int m0 = m_blk + (tid >> 4) * 4, n0 = n_blk + (tid & 15) * 4; + const float(*acc)[4] = reinterpret_cast(&accs[(size_t)tid * 16]); + for (int i = 0; i < 4 && m0 + i < M; ++i) + for (int j = 0; j < 4 && n0 + j < N; ++j) C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } + } +} + +// f32 counterpart of the 128x128 tile: two staged floats per thread per call, +// four calls per K step, 8x8 accumulators. +static void sim_sgemm_lds8x8_f32(float* C, const float* A, const float* Bt, int M, int N, int K) { + std::vector at((size_t)RDNA2_T4_TILE_MN * RDNA2_T4_LDS_F32_STRIDE); + std::vector bt(at.size()); + std::vector accs((size_t)256 * 64); + const int gx = (N + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN; + const int gy = (M + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN; + for (int by = 0; by < gy; ++by) + for (int bx = 0; bx < gx; ++bx) { + const int m_blk = by * RDNA2_T4_TILE_MN, n_blk = bx * RDNA2_T4_TILE_MN; + for (auto& v : accs) v = 0.0f; + for (int k0 = 0; k0 < K; k0 += RDNA2_T4_LDS_F32_KSTEP) { + for (int tid = 0; tid < 256; ++tid) { + sim_stage_f32(at.data(), A, M, K, m_blk, k0, tid >> 1, (tid & 1) << 2); + sim_stage_f32(at.data(), A, M, K, m_blk, k0, tid >> 1, ((tid & 1) << 2) + 2); + sim_stage_f32(bt.data(), Bt, N, K, n_blk, k0, tid >> 1, (tid & 1) << 2); + sim_stage_f32(bt.data(), Bt, N, K, n_blk, k0, tid >> 1, ((tid & 1) << 2) + 2); + } + int kend = K - k0; + if (kend > RDNA2_T4_LDS_F32_KSTEP) kend = RDNA2_T4_LDS_F32_KSTEP; + for (int tid = 0; tid < 256; ++tid) { + float(*acc)[8] = reinterpret_cast(&accs[(size_t)tid * 64]); + rdna2_sgemm_lds_block8x8_f32(acc, at.data(), bt.data(), RDNA2_T4_LDS_F32_STRIDE, + (tid >> 4) * 8, (tid & 15) * 8, kend); + } + } + for (int tid = 0; tid < 256; ++tid) { + const int m0 = m_blk + (tid >> 4) * 8, n0 = n_blk + (tid & 15) * 8; + const float(*acc)[8] = reinterpret_cast(&accs[(size_t)tid * 64]); + for (int i = 0; i < 8 && m0 + i < M; ++i) + for (int j = 0; j < 8 && n0 + j < N; ++j) C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } + } +} + +int main() { + char devname[256]; + int warp_size = 0, arch_ok = 0; + if (rdna2_device_info(devname, sizeof(devname), &warp_size, &arch_ok) != 0) { + std::printf("device query failed\n"); + return 1; + } + const int has_device = (devname[0] != '\0'); + if (has_device) { + const char* arch = arch_ok == RDNA2_ARCH_GFX1031 ? "gfx1031 (RDNA2 target)" + : arch_ok == RDNA2_ARCH_GFX1030 ? "gfx1030 (RDNA2 target)" + : arch_ok == RDNA2_ARCH_GFX1201 ? "gfx1201 (RDNA4, 9070 XT)" + : arch_ok == RDNA2_ARCH_GFX1200 ? "gfx1200 (RDNA4, 9060 XT)" + : "unknown (generic 64x64 tile)"; + std::printf("device: %s warp=%d %s\n", devname, warp_size, arch); + std::printf("arch: code=%d gemm i8 tiled=%d sgemm variant=%d\n", arch_ok, + rdna2_tiled_for_arch(arch_ok), rdna2_sgemm_variant_for_arch(arch_ok)); + } else { + std::printf("device: none - running CPU fallback paths only\n"); + warp_size = 32; + } + std::printf("path: %s\n", has_device ? "GPU (launchers use the device)" : "CPU fallback"); + + // --- 1. Argument validation (must hold on every path) ------------------- + { + std::vector sa, sx, sy; + int32_t* A = align16(sa, 256); + int32_t* x = align16(sx, 16); + int32_t* y = align16(sy, 8); + CHECK(rdna2_gemv_i8_host(nullptr, A, x, 4, 4) == 2, "null y rejected"); + CHECK(rdna2_gemv_i8_host(y, A, x, 0, 4) == 2, "rows=0 rejected"); + CHECK(rdna2_gemv_i8_host(y, A, x, 4, 0) == 2, "k4=0 rejected"); + CHECK(rdna2_gemv_i8_host(nullptr, nullptr, nullptr, 4, 4) == 2, "gemv null rejected"); + // k4 not a multiple of 4 and odd host alignment are valid input: the kernel + // has a scalar tail and the launcher copies into an aligned device buffer. + CHECK(rdna2_gemv_i8_host(y, A, x, 4, 3) == 0, "k4=3 accepted (scalar tail)"); + CHECK(rdna2_gemv_i8_host(y, align16(sa, 256, 4), x, 4, 4) == 0, "misaligned A accepted"); + CHECK(rdna2_gemv_i8_host(y, A, align16(sx, 16, 8), 4, 4) == 0, "misaligned x accepted"); + CHECK(rdna2_gemv_i8_host(y, A, x, 4, 4) == 0, "valid gemv accepted"); + + std::vector fA(256), fX(16), fY(16); + CHECK(rdna2_sgemv_f32_host(nullptr, fA.data(), fX.data(), 4, 4) == 2, + "sgemv null y rejected"); + CHECK(rdna2_sgemv_f32_host(fY.data(), fA.data(), fX.data(), 0, 4) == 2, "sgemv rows=0 rejected"); + CHECK(rdna2_sgemv_f32_host(fY.data(), fA.data(), fX.data(), 4, 0) == 2, "sgemv K=0 rejected"); + CHECK(rdna2_sgemv_f32_host(fY.data(), fA.data(), fX.data(), 4, 4) == 0, "valid sgemv accepted"); + CHECK(rdna2_sgemv_f32_host_variant(fY.data(), fA.data(), fX.data(), 4, 4, 1) == 0, + "valid sgemv wave accepted"); + + int32_t c = 0; + CHECK(rdna2_gemm_i8_host(&c, A, A, 0, 1, 1, 1) == 2, "gemm M=0 rejected"); + CHECK(rdna2_sgemm_f32_host(nullptr, nullptr, nullptr, 1, 1, 1) == 2, "sgemm null rejected"); + CHECK(rdna2_dot_u8_host(nullptr, nullptr, nullptr, 4) == 2, "dot_u8 null rejected"); + CHECK(rdna2_prefix_host(y, x, 0) == 2, "prefix n=0 rejected"); + CHECK(rdna2_prefix_host(y, x, warp_size + 1) == 2, "prefix n>warp rejected"); + } + + // --- 1b. Architecture selectors ----------------------------------------- + // Both RDNA4 parts take the 128x128 matrix-instruction shape for int8 and the + // 16-deep f32 K step; both RDNA2 parts take the 64x64 dp4a tile (no matrix + // unit there), and anything else the 4x4 tiled kernel with global loads. The + // 128-tile dp4a kernel measured slower than the 64x64 one on a 9070 XT (see + // README), so tiled == 4 is exercised explicitly rather than by selector. + { + CHECK(rdna2_tiled_for_arch(RDNA2_ARCH_GFX1201) == 5, "9070 XT takes the wmma tile"); + CHECK(rdna2_tiled_for_arch(RDNA2_ARCH_GFX1200) == 5, "9060 XT takes the wmma tile"); + CHECK(rdna2_tiled_for_arch(RDNA2_ARCH_GFX1031) == 3, "gfx1031 keeps the 64x64 8x4 tile"); + CHECK(rdna2_tiled_for_arch(RDNA2_ARCH_GFX1030) == 3, "gfx1030 keeps the 64x64 8x4 tile"); + CHECK(rdna2_tiled_for_arch(RDNA2_ARCH_UNKNOWN) == 1, "unknown GPU falls back to 4x4 tiled"); + CHECK(rdna2_sgemm_variant_for_arch(RDNA2_ARCH_GFX1201) == 3, "9070 XT takes the 16-deep f32 K step"); + CHECK(rdna2_sgemm_variant_for_arch(RDNA2_ARCH_GFX1200) == 3, "9060 XT takes the 16-deep f32 K step"); + CHECK(rdna2_sgemm_variant_for_arch(RDNA2_ARCH_GFX1031) == 3, "gfx1031 takes the 16-deep f32 K step"); + CHECK(rdna2_sgemm_variant_for_arch(RDNA2_ARCH_GFX1030) == 3, "gfx1030 takes the 16-deep f32 K step"); + CHECK(rdna2_sgemm_variant_for_arch(RDNA2_ARCH_UNKNOWN) == 1, "unknown GPU keeps the 8-deep f32 K step"); + // The tuned entry points query the running device; with no GPU they take the + // CPU reference path, which must still agree with the plain entry points. + const int M = 17, N = 9, K = 6, K4 = 3; + std::vector sA(M * K4), sB(N * K4), rI(M * N), gI(M * N); + std::vector fA(M * K), fB(N * K), rF(M * N), gF(M * N); + for (int i = 0; i < M * K4; ++i) sA[i] = (int32_t)rng(); + for (int i = 0; i < N * K4; ++i) sB[i] = (int32_t)rng(); + for (int i = 0; i < M * K; ++i) fA[i] = (float)((int)(rng() % 2001) - 1000) / 64.0f; + for (int i = 0; i < N * K; ++i) fB[i] = (float)((int)(rng() % 2001) - 1000) / 64.0f; + rdna2_gemm_i8_ref(rI.data(), sA.data(), sB.data(), M, N, K4); + rdna2_sgemm_f32_ref(rF.data(), fA.data(), fB.data(), M, N, K); + CHECK(rdna2_gemm_i8_host_tuned(gI.data(), sA.data(), sB.data(), M, N, K4) == 0, + "tuned gemm returned error"); + CHECK(rdna2_sgemm_f32_host_tuned(gF.data(), fA.data(), fB.data(), M, N, K) == 0, + "tuned sgemm returned error"); + int badI = 0, badF = 0; + for (int i = 0; i < M * N; ++i) { + badI += (gI[i] != rI[i]); + badF += (std::fabs((double)gF[i] - (double)rF[i]) > 1e-4 * (std::fabs((double)rF[i]) + 1.0)); + } + CHECK(badI == 0, "tuned gemm matches reference"); + CHECK(badF == 0, "tuned sgemm matches reference"); + } + + // --- 2. GEMV, dp4a int8 (8 words = 32 int8 per row) --------------------- + // Three cases: k4 a multiple of 4 (vector path only), k4=5 (vector + tail), + // and a deliberately misaligned host buffer (launcher copies, so still exact). + for (int pass = 0; pass < 3; ++pass) { + const int rows = 37; + const int k4 = (pass == 0) ? 8 : 5; + const size_t misalign = (pass == 2) ? 4 : 0; + std::vector sa, sx, sy; + int32_t* A = align16(sa, (size_t)rows * k4, misalign); + int32_t* x = align16(sx, k4, (pass == 2) ? 12 : 0); + int32_t* y = align16(sy, rows); + for (int i = 0; i < rows * k4; ++i) A[i] = (int32_t)rng(); + for (int i = 0; i < k4; ++i) x[i] = (int32_t)rng(); + std::vector ref(rows); + rdna2_gemv_i8_ref(ref.data(), A, x, rows, k4); + CHECK(rdna2_gemv_i8_host(y, A, x, rows, k4) == 0, "gemv returned error"); + int bad = 0; + for (int r = 0; r < rows; ++r) bad += (y[r] != ref[r]); + CHECK(bad == 0, "gemv matches reference"); + // Same body the kernel runs, driven by a host replica of its grid. + std::vector sim(rows, SENTINEL); + sim_gemv_i8(sim.data(), A, x, rows, k4); + int sim_bad = 0; + for (int r = 0; r < rows; ++r) sim_bad += (sim[r] != ref[r]); + CHECK(sim_bad == 0, "gemv kernel body matches reference"); + std::printf("gemv i8 rows=%-4d k4=%-4d misalign=%-2d mismatch=%d sim=%d\n", rows, k4, + (int)misalign, bad, sim_bad); + } + + // --- 2b. GEMV, wave-per-row variant (what k4 >= 8 selects by default) --- + // Narrows down to two cases here because the point is the cooperative row: + // a short row (k4=5, several lanes idle) and a row that is a multiple of the + // wave width (k4=8 gives the vector loads plenty of work). + for (int pass = 0; pass < 2; ++pass) { + const int rows = 37; + const int k4 = (pass == 0) ? 8 : 5; + std::vector sa, sx, sy; + int32_t* A = align16(sa, (size_t)rows * k4); + int32_t* x = align16(sx, k4); + int32_t* y = align16(sy, rows); + for (int i = 0; i < rows * k4; ++i) A[i] = (int32_t)rng(); + for (int i = 0; i < k4; ++i) x[i] = (int32_t)rng(); + std::vector ref(rows); + rdna2_gemv_i8_ref(ref.data(), A, x, rows, k4); + CHECK(rdna2_gemv_i8_host_variant(y, A, x, rows, k4, 1) == 0, "gemv wave returned error"); + int bad = 0; + for (int r = 0; r < rows; ++r) bad += (y[r] != ref[r]); + CHECK(bad == 0, "gemv wave variant matches reference"); + std::vector sim(rows, SENTINEL); + sim_gemv_wave_i8(sim.data(), A, x, rows, k4, warp_size); + int sim_bad = 0; + for (int r = 0; r < rows; ++r) sim_bad += (sim[r] != ref[r]); + CHECK(sim_bad == 0, "gemv wave body matches reference"); + std::printf("gemv i8w rows=%-4d k4=%-4d wave=%d mismatch=%d sim=%d\n", rows, k4, warp_size, bad, + sim_bad); + } + + // --- 2c. GEMV, f32: both variants plus their host replicas --------------- + // pass 0: K a multiple of 4 and >= the wave threshold (wave variant), pass 1: + // short row (thread variant), pass 2: odd K with an unaligned A view. + for (int pass = 0; pass < 3; ++pass) { + const int rows = 37; + const int K = (pass == 0) ? 128 : (pass == 1 ? 5 : 133); + const size_t shift = (pass == 2) ? 1 : 0; + std::vector A((size_t)rows * K + shift), x((size_t)K), ref((size_t)rows); + for (auto& v : A) v = (float)((int)(rng() % 2001) - 1000) / 512.0f; + for (auto& v : x) v = (float)((int)(rng() % 2001) - 1000) / 512.0f; + const float* Av = A.data() + shift; + rdna2_sgemv_f32_ref(ref.data(), Av, x.data(), rows, K); + const int want = K >= 128 ? 1 : 0; + for (int variant = 0; variant < 2; ++variant) { + std::vector y((size_t)rows, 1e30f); + CHECK(rdna2_sgemv_f32_host_variant(y.data(), Av, x.data(), rows, K, variant) == 0, + "sgemv returned error"); + double worst = 0.0; + for (int r = 0; r < rows; ++r) { + const double s = std::fabs((double)ref[r]) > 1e-6 ? std::fabs((double)ref[r]) : 1.0; + worst = std::fmax(worst, std::fabs((double)y[r] - (double)ref[r]) / s); + } + CHECK(worst < 1e-6, "sgemv variant matches reference"); + std::vector sim((size_t)rows, 1e30f); + if (variant == 1) + sim_sgemv_wave_f32(sim.data(), Av, x.data(), rows, K, warp_size); + else + sim_sgemv_f32(sim.data(), Av, x.data(), rows, K); + double sim_worst = 0.0; + for (int r = 0; r < rows; ++r) { + const double s = std::fabs((double)ref[r]) > 1e-6 ? std::fabs((double)ref[r]) : 1.0; + sim_worst = std::fmax(sim_worst, std::fabs((double)sim[r] - (double)ref[r]) / s); + } + CHECK(sim_worst < 1e-6, "sgemv kernel body matches reference"); + std::printf("sgemv f32 rows=%-4d K=%-4d variant=%d mismatch=%.1e sim=%.1e\n", rows, K, variant, + worst, sim_worst); + } + if (has_device) { + std::vector y((size_t)rows, 1e30f); + CHECK(rdna2_sgemv_f32_host(y.data(), Av, x.data(), rows, K) == 0, "sgemv default path failed"); + double worst = 0.0; + for (int r = 0; r < rows; ++r) { + const double s = std::fabs((double)ref[r]) > 1e-6 ? std::fabs((double)ref[r]) : 1.0; + worst = std::fmax(worst, std::fabs((double)y[r] - (double)ref[r]) / s); + } + CHECK(worst < 1e-6, "sgemv default variant matches reference"); + CHECK((K >= 128 ? 1 : 0) == want, "sgemv threshold selects the wave for long rows"); + } + } + + // --- 3. GEMM int8: naive, tiled and kernel-body sim, crossing the 64x64 + // tile boundary with partial edge tiles ------------------------- + { + const int M = 70, N = 67, K4 = 10; + std::vector sA(M * K4), sB(N * K4), ref(M * N), cN(M * N), cT(M * N); + for (auto& v : sA) v = (int32_t)rng(); + for (auto& v : sB) v = (int32_t)rng(); + rdna2_gemm_i8_ref(ref.data(), sA.data(), sB.data(), M, N, K4); + cN.assign(M * N, SENTINEL); + cT.assign(M * N, SENTINEL); + CHECK(rdna2_gemm_i8_host(cN.data(), sA.data(), sB.data(), M, N, K4, 0) == 0, + "gemm naive returned error"); + CHECK(rdna2_gemm_i8_host(cT.data(), sA.data(), sB.data(), M, N, K4, 1) == 0, + "gemm tiled returned error"); + std::vector cL(M * N, SENTINEL); + CHECK(rdna2_gemm_i8_host(cL.data(), sA.data(), sB.data(), M, N, K4, 2) == 0, + "gemm lds returned error"); + int badN = 0, badT = 0, badL = 0; + for (int i = 0; i < M * N; ++i) { + badN += (cN[i] != ref[i]); + badT += (cT[i] != ref[i]); + badL += (cL[i] != ref[i]); + } + CHECK(badN == 0, "gemm naive matches reference"); + CHECK(badT == 0, "gemm tiled matches reference"); + CHECK(badL == 0, "gemm lds matches reference"); + // K4=10 stages one short chunk; K4=40 stages 16+16+8, so restaging and the + // partial last chunk are exercised on odd matrix sizes too. + std::vector lds_sim(M * N, SENTINEL); + sim_gemm_lds_i8(lds_sim.data(), sA.data(), sB.data(), M, N, K4); + int badLS = 0; + for (int i = 0; i < M * N; ++i) badLS += (lds_sim[i] != ref[i]); + CHECK(badLS == 0, "gemm lds simulation covers every element correctly"); + std::vector cL8(M * N, SENTINEL), lds8_sim(M * N, SENTINEL); + CHECK(rdna2_gemm_i8_host(cL8.data(), sA.data(), sB.data(), M, N, K4, 3) == 0, + "gemm lds8 returned error"); + int badL8 = 0; + for (int i = 0; i < M * N; ++i) badL8 += (cL8[i] != ref[i]); + CHECK(badL8 == 0, "gemm lds8 matches reference"); + sim_gemm_lds8_i8(lds8_sim.data(), sA.data(), sB.data(), M, N, K4); + int badL8S = 0; + for (int i = 0; i < M * N; ++i) badL8S += (lds8_sim[i] != ref[i]); + CHECK(badL8S == 0, "gemm lds8 simulation covers every element correctly"); + // RDNA4 shape: big enough for two 128x128 tiles per dimension (partial edge + // tiles) and K4=40, which stages 16+16+8. + { + const int M4 = 135, N4 = 130, K4b = 40; + std::vector A4((size_t)M4 * K4b), B4((size_t)N4 * K4b), ref4((size_t)M4 * N4), + got4((size_t)M4 * N4, SENTINEL), sim4((size_t)M4 * N4, SENTINEL); + for (auto& v : A4) v = (int32_t)rng(); + for (auto& v : B4) v = (int32_t)rng(); + rdna2_gemm_i8_ref(ref4.data(), A4.data(), B4.data(), M4, N4, K4b); + CHECK(rdna2_gemm_i8_host(got4.data(), A4.data(), B4.data(), M4, N4, K4b, 4) == 0, + "gemm lds128 returned error"); + int b4 = 0; + for (int i = 0; i < M4 * N4; ++i) b4 += (got4[i] != ref4[i]); + CHECK(b4 == 0, "gemm lds128 matches reference"); + sim_gemm_lds8x8_i8(sim4.data(), A4.data(), B4.data(), M4, N4, K4b); + int b4s = 0; + for (int i = 0; i < M4 * N4; ++i) b4s += (sim4[i] != ref4[i]); + CHECK(b4s == 0, "gemm lds128 simulation covers every element correctly"); + std::printf("gemm i8ld128 M=%-4d N=%-4d K4=%-3d mismatch=%d sim=%d (128x128, 8x8)\n", M4, N4, + K4b, b4, b4s); + } + // RDNA4 matrix path: the same 128x128 tile driven by 16x16x16 WMMA. M4/N4 + // are not multiples of 128 and K4b=40 stages 16+16+8, so edge tiles and a + // partial last chunk are covered here too. + { + const int M4 = 135, N4 = 130, K4b = 40; + std::vector A4((size_t)M4 * K4b), B4((size_t)N4 * K4b), ref4((size_t)M4 * N4), + got5((size_t)M4 * N4, SENTINEL), sim5((size_t)M4 * N4, SENTINEL); + for (auto& v : A4) v = (int32_t)rng(); + for (auto& v : B4) v = (int32_t)rng(); + rdna2_gemm_i8_ref(ref4.data(), A4.data(), B4.data(), M4, N4, K4b); + CHECK(rdna2_gemm_i8_host(got5.data(), A4.data(), B4.data(), M4, N4, K4b, 5) == 0, + "gemm wmma returned error"); + int b5 = 0; + for (int i = 0; i < M4 * N4; ++i) b5 += (got5[i] != ref4[i]); + CHECK(b5 == 0, "gemm wmma matches reference"); + sim_gemm_wmma_i8(sim5.data(), A4.data(), B4.data(), M4, N4, K4b); + int b5s = 0; + for (int i = 0; i < M4 * N4; ++i) b5s += (sim5[i] != ref4[i]); + CHECK(b5s == 0, "gemm wmma simulation covers every element correctly"); + std::printf("gemm i8wmma M=%-4d N=%-4d K4=%-3d mismatch=%d sim=%d (128x128, wmma)\n", M4, N4, + K4b, b5, b5s); + } + // Fragment placement on its own: a single 16x16x16 step with distinct byte + // values must produce the scalar dot product in every one of the 256 output + // slots, and must add into a nonzero accumulator rather than overwrite it. + { + int8_t ta[16][16], tb[16][16]; + int32_t acc[32][8], af[32][2], bf[32][2]; + for (int m = 0; m < 16; ++m) + for (int k = 0; k < 16; ++k) { + ta[m][k] = (int8_t)((int)(m * 7 + k * 3) % 251 - 125); + tb[k][m] = (int8_t)((int)(m * 5 + k * 11) % 241 - 120); + } + for (int t = 0; t < 32; ++t) { + for (int e = 0; e < 2; ++e) { + uint32_t wa = 0, wb = 0; + for (int b = 0; b < 4; ++b) { + const int k = (t / 16) * 8 + e * 4 + b; + wa |= (uint32_t)(uint8_t)ta[rdna2_wmma_i8_row(t)][k] << (8 * b); + wb |= (uint32_t)(uint8_t)tb[k][rdna2_wmma_i8_row(t)] << (8 * b); + } + af[t][e] = (int32_t)wa; // Bt rows are K-major, so tb[k][n] as built + bf[t][e] = (int32_t)wb; + } + for (int l = 0; l < 8; ++l) acc[t][l] = 1000; // nonzero, must survive + } + sim_wmma_i8_wave_step(acc, af, bf); + int badf = 0; + for (int t = 0; t < 32; ++t) + for (int l = 0; l < 8; ++l) { + const int m = rdna2_wmma_i8_acc_row(t, l), n = rdna2_wmma_i8_acc_col(t); + int32_t want = 1000; + for (int k = 0; k < 16; ++k) want += (int32_t)ta[m][k] * (int32_t)tb[k][n]; + badf += (acc[t][l] != want); + } + CHECK(badf == 0, "wmma fragment placement maps every result slot"); + std::printf("wmma i8 fragment 16x16x16 mismatch=%d (per-slot dot + accumulate)\n", badf); + } + { + const int M2 = 35, N2 = 33, K2 = 40; + std::vector A2(M2 * K2), B2(N2 * K2), ref2(M2 * N2), got2(M2 * N2, SENTINEL), + sim2(M2 * N2, SENTINEL); + for (auto& v : A2) v = (int32_t)rng(); + for (auto& v : B2) v = (int32_t)rng(); + rdna2_gemm_i8_ref(ref2.data(), A2.data(), B2.data(), M2, N2, K2); + CHECK(rdna2_gemm_i8_host(got2.data(), A2.data(), B2.data(), M2, N2, K2, 2) == 0, + "gemm lds multi-chunk returned error"); + int b2 = 0; + for (int i = 0; i < M2 * N2; ++i) b2 += (got2[i] != ref2[i]); + CHECK(b2 == 0, "gemm lds multi-chunk matches reference"); + sim_gemm_lds_i8(sim2.data(), A2.data(), B2.data(), M2, N2, K2); + int b2s = 0; + for (int i = 0; i < M2 * N2; ++i) b2s += (sim2[i] != ref2[i]); + CHECK(b2s == 0, "gemm lds multi-chunk simulation matches reference"); + CHECK(badL == 0 && b2s == 0 && b2 == 0, "gemm lds chunk loop is correct"); + // The 8x4 variant stages two words per thread, so it has its own coverage + // of the partial chunk and the edge tiles. + std::vector got8(M2 * N2, SENTINEL), sim8(M2 * N2, SENTINEL); + CHECK(rdna2_gemm_i8_host(got8.data(), A2.data(), B2.data(), M2, N2, K2, 3) == 0, + "gemm lds8 multi-chunk returned error"); + int b8 = 0; + for (int i = 0; i < M2 * N2; ++i) b8 += (got8[i] != ref2[i]); + sim_gemm_lds8_i8(sim8.data(), A2.data(), B2.data(), M2, N2, K2); + int b8s = 0; + for (int i = 0; i < M2 * N2; ++i) b8s += (sim8[i] != ref2[i]); + CHECK(b8 == 0 && b8s == 0, "gemm lds8 multi-chunk matches reference"); + std::printf("gemm i8lds M=%-4d N=%-4d K4=%-4d mismatch=%d sim=%d (partial chunk)\n", M2, N2, + K2, b2, b2s); + std::printf("gemm i8ld8 M=%-4d N=%-4d K4=%-4d mismatch=%d sim=%d (8x4 blocking)\n", M2, N2, + K2, b8, b8s); + } + // Kernel body on the host, seeded so a missing or stray write is caught. + std::vector cS(M * N, SENTINEL); + sim_gemm_tile_i8(cS.data(), sA.data(), sB.data(), M, N, K4); + int badS = 0; + for (int i = 0; i < M * N; ++i) badS += (cS[i] != ref[i]); + CHECK(badS == 0, "gemm tiled kernel body covers every element correctly"); + std::printf("gemm i8 M=%-4d N=%-4d K4=%-4d naive=%d tiled=%d lds=%d lds8=%d sim=%d lds_sim=%d lds8_sim=%d\n", + M, N, K4, badN, badT, badL, badL8, badS, badLS, badL8S); + } + + // --- 3b. Quantized epilogue: scale and zero point applied to the int32 + // accumulator, with the stored quants left untouched ---------------- + { + const int M = 37, N = 45, K4 = 12; // odd: partial 64x64 and 128x128 tiles + std::vector sA((size_t)M * K4), sB((size_t)N * K4); + for (auto& v : sA) v = (int32_t)rng(); + for (auto& v : sB) v = (int32_t)rng(); + const std::vector A0 = sA, B0 = sB; + std::vector sa(M), sb(N), ref((size_t)M * N), got((size_t)M * N); + for (int i = 0; i < M; ++i) sa[i] = 0.001f * (float)(1 + i % 7); + for (int i = 0; i < N; ++i) sb[i] = 0.002f * (float)(1 + i % 5); + + auto worst_rel = [&]() { + double worst = 0.0; + for (size_t i = 0; i < ref.size(); ++i) { + const double d = std::fabs((double)got[i] - (double)ref[i]); + const double s = std::fabs((double)ref[i]) > 1e-6 ? std::fabs((double)ref[i]) : 1e-6; + if (d / s > worst) worst = d / s; + } + return worst; + }; + + struct ScaleCase { + const char* name; + const float* a; + const float* b; + int zp; + int tiled; + }; + const ScaleCase cases[] = { + {"unit scales", nullptr, nullptr, 0, 3}, + {"row scale", sa.data(), nullptr, 0, 3}, + {"column scale", nullptr, sb.data(), 0, 3}, + {"both scales", sa.data(), sb.data(), 0, 3}, + {"zero point", sa.data(), sb.data(), 8, 3}, + {"zero point, 128 tile", sa.data(), sb.data(), 8, 4}, + {"zero point, wmma", sa.data(), sb.data(), 8, 5}, + {"zero point, naive", sa.data(), sb.data(), 8, 0}, + }; + for (const ScaleCase& c : cases) { + rdna2_gemm_i8_scale_ref(ref.data(), sA.data(), sB.data(), M, N, K4, c.a, c.b, c.zp); + got.assign(got.size(), -1.0f); + CHECK(rdna2_gemm_i8_scale_host(got.data(), sA.data(), sB.data(), M, N, K4, c.tiled, c.a, c.b, + c.zp) == 0, + "scaled gemm returned error"); + const double worst = worst_rel(); + CHECK(worst < 1e-5, "scaled gemm matches reference"); + std::printf("gemm i8+scale %-21s zp=%d tiled=%d worst_rel=%.2e\n", c.name, c.zp, c.tiled, + worst); + } + + // The correction must be exact, not merely close: with unit scales nothing + // rounds, so (acc - zp*rowsum) has to come back as that exact integer. + { + rdna2_gemm_i8_scale_ref(ref.data(), sA.data(), sB.data(), M, N, K4, nullptr, nullptr, 8); + CHECK(rdna2_gemm_i8_scale_host(got.data(), sA.data(), sB.data(), M, N, K4, 3, nullptr, nullptr, + 8) == 0, + "zero point exactness returned error"); + int bad = 0; + for (size_t i = 0; i < ref.size(); ++i) bad += (got[i] != ref[i]); + CHECK(bad == 0, "zero point correction is bit-exact"); + } + + // Loaded quants are read-only. The paths that dequantise in place, or that + // expand to a float copy, would show up here as a changed input buffer. + CHECK(std::memcmp(sA.data(), A0.data(), sA.size() * sizeof(int32_t)) == 0 && + std::memcmp(sB.data(), B0.data(), sB.size() * sizeof(int32_t)) == 0, + "scaled gemm leaves the stored quants untouched"); + + CHECK(rdna2_gemm_i8_scale_host(nullptr, sA.data(), sB.data(), M, N, K4, 3, nullptr, nullptr, 0) == + 2, + "scaled gemm rejects null C"); + CHECK(rdna2_gemm_i8_scale_host(got.data(), sA.data(), sB.data(), 0, N, K4, 3, nullptr, nullptr, + 0) == 2, + "scaled gemm rejects M=0"); + + // The arch-aware entry point must agree with the explicit one on this GPU. + { + std::vector got2((size_t)M * N); + rdna2_gemm_i8_scale_ref(ref.data(), sA.data(), sB.data(), M, N, K4, sa.data(), sb.data(), 8); + CHECK(rdna2_gemm_i8_scale_host_tuned(got2.data(), sA.data(), sB.data(), M, N, K4, sa.data(), + sb.data(), 8) == 0, + "tuned scaled gemm returned error"); + int bad = 0; + for (size_t i = 0; i < ref.size(); ++i) bad += (got2[i] != ref[i]); + CHECK(bad == 0, "tuned scaled gemm matches reference"); + } + } + + // --- 4. SGEMM f32, also across the tile boundary ------------------------ + { + const int M = 67, N = 70, K = 19; + std::vector A(M * K), B(N * K), ref(M * N), c(M * N, 1e30f); + for (auto& v : A) v = (float)((int)(rng() % 2001) - 1000) / 512.0f; + for (auto& v : B) v = (float)((int)(rng() % 2001) - 1000) / 512.0f; + rdna2_sgemm_f32_ref(ref.data(), A.data(), B.data(), M, N, K); + CHECK(rdna2_sgemm_f32_host(c.data(), A.data(), B.data(), M, N, K) == 0, + "sgemm returned error"); + double max_rel = 0.0; + for (int i = 0; i < M * N; ++i) { + double d = std::fabs((double)c[i] - (double)ref[i]); + double s = std::fabs((double)ref[i]) > 1e-6 ? std::fabs((double)ref[i]) : 1.0; + max_rel = std::fmax(max_rel, d / s); + } + // fma vs separate multiply/add can differ in the last bit. + CHECK(max_rel < 1e-5, "sgemm matches reference within tolerance"); + std::vector cs(M * N, 1e30f); + sim_sgemm_tile_f32(cs.data(), A.data(), B.data(), M, N, K); + double max_sim = 0.0; + for (int i = 0; i < M * N; ++i) { + double d = std::fabs((double)cs[i] - (double)ref[i]); + double s = std::fabs((double)ref[i]) > 1e-6 ? std::fabs((double)ref[i]) : 1.0; + max_sim = std::fmax(max_sim, d / s); + } + CHECK(max_sim < 1e-5, "sgemm kernel body matches reference within tolerance"); + // Both device variants explicitly: 0 = global loads, 1 = LDS staged. + std::vector c0(M * N, 1e30f), c1(M * N, 1e30f); + CHECK(rdna2_sgemm_f32_host_v(c0.data(), A.data(), B.data(), M, N, K, 0) == 0, + "sgemm variant 0 returned error"); + CHECK(rdna2_sgemm_f32_host_v(c1.data(), A.data(), B.data(), M, N, K, 1) == 0, + "sgemm variant 1 returned error"); + double rel0 = 0.0, rel1 = 0.0; + for (int i = 0; i < M * N; ++i) { + double s = std::fabs((double)ref[i]) > 1e-6 ? std::fabs((double)ref[i]) : 1.0; + rel0 = std::fmax(rel0, std::fabs((double)c0[i] - (double)ref[i]) / s); + rel1 = std::fmax(rel1, std::fabs((double)c1[i] - (double)ref[i]) / s); + } + CHECK(rel0 < 1e-5, "sgemm variant 0 matches reference"); + CHECK(rel1 < 1e-5, "sgemm variant 1 matches reference"); + std::vector cls(M * N, 1e30f); + sim_sgemm_lds_f32(cls.data(), A.data(), B.data(), M, N, K); + double max_ls = 0.0; + for (int i = 0; i < M * N; ++i) { + double d = std::fabs((double)cls[i] - (double)ref[i]); + double s = std::fabs((double)ref[i]) > 1e-6 ? std::fabs((double)ref[i]) : 1.0; + max_ls = std::fmax(max_ls, d / s); + } + CHECK(max_ls < 1e-5, "sgemm lds simulation matches reference within tolerance"); + std::printf("sgemm f32 M=%-4d N=%-4d K=%-5d max_rel=%.2e sim_rel=%.2e v0=%.2e lds_sim=%.2e\n", + M, N, K, max_rel, max_sim, rel0 > rel1 ? rel0 : rel1, max_ls); + + // Variant 3: 16-float K chunks, edges included (M and N cross a 64 tile and + // K=19 leaves a 3-wide tail chunk). + const int M3 = 70, N3 = 66, K3 = 19; + std::vector A3((size_t)M3 * K3), B3((size_t)N3 * K3), ref3((size_t)M3 * N3), + got3((size_t)M3 * N3, 1e30f), sim3((size_t)M3 * N3, 1e30f); + for (auto& v : A3) v = (float)((int)(rng() % 2001) - 1000) / 512.0f; + for (auto& v : B3) v = (float)((int)(rng() % 2001) - 1000) / 512.0f; + rdna2_sgemm_f32_ref(ref3.data(), A3.data(), B3.data(), M3, N3, K3); + CHECK(rdna2_sgemm_f32_host_v(got3.data(), A3.data(), B3.data(), M3, N3, K3, 3) == 0, + "sgemm variant 3 returned error"); + sim_sgemm_lds16_f32(sim3.data(), A3.data(), B3.data(), M3, N3, K3); + double rel3 = 0.0, rel3s = 0.0; + for (int i = 0; i < M3 * N3; ++i) { + const double s = std::fabs((double)ref3[i]) > 1e-6 ? std::fabs((double)ref3[i]) : 1.0; + rel3 = std::fmax(rel3, std::fabs((double)got3[i] - (double)ref3[i]) / s); + rel3s = std::fmax(rel3s, std::fabs((double)sim3[i] - (double)ref3[i]) / s); + } + CHECK(rel3 < 1e-5, "sgemm variant 3 matches reference"); + CHECK(rel3s < 1e-5, "sgemm variant 3 simulation matches reference"); + std::printf("sgemm f16k M=%-4d N=%-4d K=%-5d max_rel=%.2e sim_rel=%.2e (64x64, KSTEP=16)\n", + M3, N3, K3, rel3, rel3s); + + // RDNA4 variant: two 128x128 tiles per dimension, K=19 -> 8+8+3 staged. + const int M4 = 135, N4 = 130, K4 = 19; + std::vector A4((size_t)M4 * K4), B4((size_t)N4 * K4), ref4((size_t)M4 * N4), + got4((size_t)M4 * N4, 1e30f), sim4((size_t)M4 * N4, 1e30f); + for (auto& v : A4) v = (float)((int)(rng() % 2001) - 1000) / 512.0f; + for (auto& v : B4) v = (float)((int)(rng() % 2001) - 1000) / 512.0f; + rdna2_sgemm_f32_ref(ref4.data(), A4.data(), B4.data(), M4, N4, K4); + CHECK(rdna2_sgemm_f32_host_v(got4.data(), A4.data(), B4.data(), M4, N4, K4, 2) == 0, + "sgemm variant 2 returned error"); + sim_sgemm_lds8x8_f32(sim4.data(), A4.data(), B4.data(), M4, N4, K4); + double rel2 = 0.0, rel2s = 0.0; + for (int i = 0; i < M4 * N4; ++i) { + const double s = std::fabs((double)ref4[i]) > 1e-6 ? std::fabs((double)ref4[i]) : 1.0; + rel2 = std::fmax(rel2, std::fabs((double)got4[i] - (double)ref4[i]) / s); + rel2s = std::fmax(rel2s, std::fabs((double)sim4[i] - (double)ref4[i]) / s); + } + CHECK(rel2 < 1e-5, "sgemm variant 2 matches reference"); + CHECK(rel2s < 1e-5, "sgemm variant 2 simulation matches reference"); + std::printf("sgemm f128 M=%-4d N=%-4d K=%-5d max_rel=%.2e sim_rel=%.2e (128x128, 8x8)\n", M4, N4, + K4, rel2, rel2s); + } + + // --- 5. dp4a u8 dot ----------------------------------------------------- + { + const int n = 1000; + std::vector a(n), b(n); + for (auto& v : a) v = rng(); + for (auto& v : b) v = rng(); + unsigned got = 0; + const unsigned ref = rdna2_dot_u8_ref(a.data(), b.data(), n); + CHECK(rdna2_dot_u8_host(&got, a.data(), b.data(), n) == 0, "dot_u8 returned error"); + CHECK(got == ref, "dot_u8 matches reference"); + std::printf("dot u8 n=%-5d got=%u ref=%u\n", n, got, ref); + } + + // --- 6. packed f16 dot -------------------------------------------------- + { + const int n = 500; + std::vector a(n), b(n); + std::vector av(n * 2), bv(n * 2); + for (int i = 0; i < n * 2; ++i) { + av[i] = (float)((int)(rng() % 4001) - 2000) / 256.0f; + bv[i] = (float)((int)(rng() % 4001) - 2000) / 256.0f; + } +#if defined(__FLT16_MAX__) + for (int i = 0; i < n; ++i) { + a[i] = rdna2_pack_half2((_Float16)av[2 * i], (_Float16)av[2 * i + 1]); + b[i] = rdna2_pack_half2((_Float16)bv[2 * i], (_Float16)bv[2 * i + 1]); + } + float got = 0.0f; + const float ref = rdna2_dot_f16_ref(a.data(), b.data(), n); + CHECK(rdna2_dot_f16_host(&got, a.data(), b.data(), n) == 0, "dot_f16 returned error"); + double rel = std::fabs((double)got - (double)ref) / (std::fabs((double)ref) + 1e-9); + // Atomic accumulation order is not fixed, so allow f32 rounding drift. + CHECK(rel < 1e-4, "dot_f16 matches reference within tolerance"); + std::printf("dot f16 n=%-5d got=%.6f ref=%.6f rel_err=%.2e\n", n, got, ref, rel); +#else + std::printf("dot f16 skipped (host compiler lacks _Float16)\n"); +#endif + } + + // --- 7. DPP prefix sum over one wave ------------------------------------ + { + const int n = warp_size; + std::vector in(n), ref(n), out(n); + for (int i = 0; i < n; ++i) in[i] = (int32_t)(rng() % 2001) - 1000; + rdna2_prefix_ref(ref.data(), in.data(), n); + CHECK(rdna2_prefix_host(out.data(), in.data(), n) == 0, "prefix returned error"); + int bad = 0; + for (int i = 0; i < n; ++i) bad += (out[i] != ref[i]); + CHECK(bad == 0, "prefix matches reference"); + std::printf("prefix i32 lanes=%-4d mismatches=%d\n", n, bad); + } + + std::printf("\n%s: %d check(s), %d failure(s), path=%s\n", failures ? "FAILED" : "PASSED", + checks, failures, has_device ? "GPU" : "CPU fallback"); + return failures ? 1 : 0; +} diff --git a/kernel/Makefile b/kernel/Makefile index 5256ef0c5d..b1aa985da1 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -229,3 +229,35 @@ clean :: #endif include $(TOPDIR)/Makefile.tail + +# RDNA2/RDNA4 HIP kernels +ifdef USE_HIP +ifeq ($(TARGET_CORE), RDNA2) +USE_RDNA2 = 1 +endif +ifeq ($(TARGET_CORE), GFX1030) +USE_RDNA2 = 1 +endif +ifeq ($(TARGET_CORE), GFX1031) +USE_RDNA2 = 1 +endif +ifeq ($(TARGET_CORE), GFX1200) +USE_RDNA2 = 1 +endif +ifeq ($(TARGET_CORE), GFX1201) +USE_RDNA2 = 1 +endif +endif + +ifdef USE_RDNA2 +HIPCC ?= hipcc +RDNA2_OBJS = rdna2/rdna2_kernels.o rdna2/rdna2_oblas.o +rdna2/rdna2_kernels.o: rdna2/rdna2_kernels.hip + $(HIPCC) $(HIPFLAGS) --offload-arch=$(TARGET_CORE) -x hip -c $< -o $@ +rdna2/rdna2_oblas.o: rdna2/rdna2_oblas.c + $(CC) $(CFLAGS) -c $< -o $@ +endif + +ifdef USE_RDNA2 +SKERNELOBJS += $(RDNA2_OBJS) +endif diff --git a/kernel/rdna2/rdna2_gemm_ref.h b/kernel/rdna2/rdna2_gemm_ref.h new file mode 100644 index 0000000000..d5f8c038fa --- /dev/null +++ b/kernel/rdna2/rdna2_gemm_ref.h @@ -0,0 +1,219 @@ +// CPU references for the RDNA2 kernels. Plain C++ (no HIP), so every +// reference can be built and tested on a machine without a GPU. The launchers +// in rdna2_kernels.hip use these as the fallback when no device is present, +// and the test uses them as the correctness oracle. +// +// Packed int8 convention (matches v_dot4_*): four int8 values per int32, +// value i at bits [8*i, 8*i+8), i.e. little-endian byte order. +#ifndef RDNA2_GEMM_REF_H +#define RDNA2_GEMM_REF_H + +#include +#include +#include + +// Ref functions that device code calls must be __host__ __device__. Under a +// plain host compiler the macro expands to nothing. +#ifndef RDNA2_HD +#if defined(__HIP__) || defined(__HIPCC__) || defined(__CUDACC__) +#define RDNA2_HD __host__ __device__ +#else +#define RDNA2_HD +#endif +#endif + +// --------------------------------------------------------------------------- +// dp4a int8 / u8 +// --------------------------------------------------------------------------- +RDNA2_HD static inline int32_t rdna2_dot4_i8_ref(int32_t a, int32_t b, int32_t acc) { + for (int i = 0; i < 4; ++i) { + int8_t x = static_cast((static_cast(a) >> (8 * i)) & 0xffu); + int8_t y = static_cast((static_cast(b) >> (8 * i)) & 0xffu); + uint32_t p = static_cast(static_cast(x) * static_cast(y)); + acc = static_cast(static_cast(acc) + p); + } + return acc; +} + +static inline uint32_t rdna2_dot4_u8_ref(uint32_t a, uint32_t b, uint32_t acc) { + for (int i = 0; i < 4; ++i) { + uint32_t x = (a >> (8 * i)) & 0xffu; + uint32_t y = (b >> (8 * i)) & 0xffu; + acc += x * y; + } + return acc; +} + +// The dot4 primitive used by the kernel bodies. On the device this is the real +// v_dot4c_i32_i8 (dp4a) instruction, independent of what the optimiser can +// pattern-match; on the host it is the bit-exact scalar reference above, so the +// same kernel body still runs on a CPU-only machine. +RDNA2_HD static inline int32_t rdna2_dot4_i8_impl(int32_t a, int32_t b, int32_t acc) { +#if defined(__HIP_DEVICE_COMPILE__) && !defined(RDNA2_NO_DOT1) +#if defined(__gfx1200__) || defined(__gfx1201__) + // gfx12 (RDNA4) has no dot1-insts, so `__builtin_amdgcn_sdot4` is rejected + // ("needs target feature dot1-insts"). The dp4a hardware is still there: the + // legacy v_dot4_i32_i8 mnemonic assembles and computes exact signed results + // (checked against the scalar reference on a gfx1201 card). Note that + // v_dot4_i32_iu8 is *not* interchangeable: on gfx12 it is the unsigned form. + asm volatile("v_dot4_i32_i8 %0, %1, %2, %0" : "+v"(acc) : "v"(a), "v"(b)); + return acc; +#else + return __builtin_amdgcn_sdot4(a, b, acc, false); +#endif +#else + // Either a host build or -DRDNA2_NO_DOT1=1 (a target whose dot1-insts feature + // is missing, e.g. an older gfx12 toolchain): same arithmetic, four scalar + // multiplies. Both branches are compared in the host test. + return rdna2_dot4_i8_ref(a, b, acc); +#endif +} + +// --------------------------------------------------------------------------- +// GEMV: y[M] = A[M x K4] . x[K4], packed int8 +// --------------------------------------------------------------------------- +static inline void rdna2_gemv_i8_ref(int32_t* y, const int32_t* A, const int32_t* x, + int rows, int K4) { + for (int r = 0; r < rows; ++r) { + int32_t acc = 0; + for (int k = 0; k < K4; ++k) acc = rdna2_dot4_i8_ref(A[(size_t)r * K4 + k], x[k], acc); + y[r] = acc; + } +} + +// --------------------------------------------------------------------------- +// GEMV: y[M] = A[M x K] . x[K], float +// --------------------------------------------------------------------------- +static inline void rdna2_sgemv_f32_ref(float* y, const float* A, const float* x, + int rows, int K) { + for (int r = 0; r < rows; ++r) { + // The reference accumulates in double and rounds once, so it does not + // share the kernel's four-way partial-sum split; the test compares with a + // tolerance rather than bit-for-bit. + double acc = 0.0; + for (int k = 0; k < K; ++k) acc += (double)A[(size_t)r * K + k] * (double)x[k]; + y[r] = (float)acc; + } +} + +// --------------------------------------------------------------------------- +// GEMM: C[M x N] = A[M x K4] * Bt[N x K4]^T, packed int8 +// --------------------------------------------------------------------------- +static inline void rdna2_gemm_i8_ref(int32_t* C, const int32_t* A, const int32_t* Bt, + int M, int N, int K4) { + for (int m = 0; m < M; ++m) { + for (int n = 0; n < N; ++n) { + const int32_t* a = A + static_cast(m) * K4; + const int32_t* b = Bt + static_cast(n) * K4; + int32_t acc = 0; + for (int k = 0; k < K4; ++k) acc = rdna2_dot4_i8_ref(a[k], b[k], acc); + C[static_cast(m) * N + n] = acc; + } + } +} + +// --------------------------------------------------------------------------- +// GEMM with the quantized epilogue applied on the CPU: +// C[m][n] = (acc[m][n] - zp_b * rowsum_a[m]) * sa[m] * sb[n] +// rowsum_a[m] is the sum of the int8 activations in row m. That is the exact +// integer form of B's asymmetric storage - B's stored value at k represents +// (q - zp_b), so sum_k a*(q - zp_b) = acc - zp_b*sum_k a - and it never converts +// a stored weight: only the finished dot product becomes float. sa / sb may be +// NULL (unit scale). The correction and the product are computed in 64-bit so +// the reference cannot overflow before its single float conversion. +// --------------------------------------------------------------------------- +static inline void rdna2_gemm_i8_scale_ref(float* C, const int32_t* A, const int32_t* Bt, + int M, int N, int K4, const float* sa, + const float* sb, int zp_b) { + for (int m = 0; m < M; ++m) { + const int32_t* a = A + static_cast(m) * K4; + int64_t rowsum = 0; + if (zp_b != 0) { + for (int k = 0; k < K4; ++k) { + for (int b = 0; b < 4; ++b) { + rowsum += static_cast((static_cast(a[k]) >> (8 * b)) & 0xffu); + } + } + } + const float s_a = sa ? sa[m] : 1.0f; + for (int n = 0; n < N; ++n) { + const int32_t* b = Bt + static_cast(n) * K4; + int32_t acc = 0; + for (int k = 0; k < K4; ++k) acc = rdna2_dot4_i8_ref(a[k], b[k], acc); + const int64_t corr = static_cast(acc) - static_cast(zp_b) * rowsum; + C[static_cast(m) * N + n] = + static_cast(corr) * s_a * (sb ? sb[n] : 1.0f); + } + } +} + +// --------------------------------------------------------------------------- +// SGEMM: C[M x N] = A[M x K] * Bt[N x K]^T, float +// --------------------------------------------------------------------------- +static inline void rdna2_sgemm_f32_ref(float* C, const float* A, const float* Bt, + int M, int N, int K) { + for (int m = 0; m < M; ++m) { + for (int n = 0; n < N; ++n) { + const float* a = A + static_cast(m) * K; + const float* b = Bt + static_cast(n) * K; + float acc = 0.0f; + for (int k = 0; k < K; ++k) acc += a[k] * b[k]; + C[static_cast(m) * N + n] = acc; + } + } +} + +// --------------------------------------------------------------------------- +// f16 dot (v_dot2c_f32_f16): acc = sum over i of a.xy * b.xy, f32 accumulate. +// Pairs are packed two _Float16 per int32, low half first. +// --------------------------------------------------------------------------- +#if defined(__FLT16_MAX__) +static inline void rdna2_unpack_half2(uint32_t v, _Float16* lo, _Float16* hi) { + uint16_t l = static_cast(v & 0xffffu); + uint16_t h = static_cast(v >> 16); + std::memcpy(lo, &l, sizeof(uint16_t)); + std::memcpy(hi, &h, sizeof(uint16_t)); +} + +static inline uint32_t rdna2_pack_half2(_Float16 lo, _Float16 hi) { + uint16_t l, h; + std::memcpy(&l, &lo, sizeof(uint16_t)); + std::memcpy(&h, &hi, sizeof(uint16_t)); + return static_cast(l) | (static_cast(h) << 16); +} + +// Reference matches the hardware order: acc += a.x*b.x then acc += a.y*b.y. +static inline float rdna2_dot_f16_ref(const uint32_t* a, const uint32_t* b, int n) { + float acc = 0.0f; + for (int i = 0; i < n; ++i) { + _Float16 ax, ay, bx, by; + rdna2_unpack_half2(a[i], &ax, &ay); + rdna2_unpack_half2(b[i], &bx, &by); + acc += static_cast(ax) * static_cast(bx); + acc += static_cast(ay) * static_cast(by); + } + return acc; +} +#endif + +// --------------------------------------------------------------------------- +// u8 dot: out = sum over i of dot4_u8(a[i], b[i]) +// --------------------------------------------------------------------------- +static inline uint32_t rdna2_dot_u8_ref(const uint32_t* a, const uint32_t* b, int n) { + uint32_t acc = 0; + for (int i = 0; i < n; ++i) acc = rdna2_dot4_u8_ref(a[i], b[i], acc); + return acc; +} + +// --------------------------------------------------------------------------- +// 32-lane inclusive prefix sum (matches rdna2_dpp_prefix on wave32). +// --------------------------------------------------------------------------- +static inline void rdna2_prefix_ref(int32_t* out, const int32_t* in, int n) { + int32_t run = 0; + for (int i = 0; i < n; ++i) { + run += in[i]; + out[i] = run; + } +} + +#endif // RDNA2_GEMM_REF_H diff --git a/kernel/rdna2/rdna2_kernels.hip b/kernel/rdna2/rdna2_kernels.hip new file mode 100644 index 0000000000..69f57ae192 --- /dev/null +++ b/kernel/rdna2/rdna2_kernels.hip @@ -0,0 +1,1367 @@ +// RDNA2 (gfx1030 / gfx1031) kernels, written directly against the GFX10.3 ISA, +// plus an RDNA4 (gfx1200 / gfx1201, RX 9060 XT / 9070 XT) tile shape that spends +// gfx12's larger register file on a 128x128 tile. +// Not a CUDA-to-HIP port: each kernel is shaped around one RDNA2 instruction +// group, and every one of them is checked against a plain-C++ reference +// (rdna2_gemm_ref.h) by test_rdna2_host.cpp. +// +// Instruction groups used: +// dot4 : v_dot4_i32_i8 / v_dot4c_i32_i8 / v_dot4_u32_u8 dp4a, int8 x4 +// dot2 : v_dot2_f32_f16 packed f16 dot +// dpp : v_add_nc_u32_dpp row_shr 16-lane wave scan +// ds_* : ds_read_b32 / ds_write_b32 LDS staging +// +// Build: clang++ -x hip --offload-arch=gfx1031 -O2 -c rdna2_kernels.hip +// clang++ -x hip --offload-arch=gfx1201 -O2 -c rdna2_kernels.hip (9070 XT) +// +// Return codes shared by all launchers: +// 0 = success, 1 = HIP/runtime error, 2 = invalid argument. + +#include +#include +#include + +#include "rdna2_gemm_ref.h" +#include "rdna2_kernels_math.h" + +#define RDNA2_OK 0 +#define RDNA2_ERR_HIP 1 +#define RDNA2_ERR_ARG 2 + +// The RDNA2_ARCH_* target codes live in rdna2_kernels_math.h, next to the tile +// shapes they select, so the host test uses the same numbers. + +// Four packed int32 = 16 bytes: one global_load_dwordx4 on the gfx1031 path. +typedef int int4_t __attribute__((ext_vector_type(4))); +// Two packed f16: the operand of v_dot2_f32_f16. +typedef _Float16 half2_t __attribute__((ext_vector_type(2))); +// WMMA operands and accumulators: V_WMMA_I32_16X16X16_IU8 takes two pairs of +// packed int32 (16 operand bytes per lane) and eight int32 accumulators. +typedef int int32x2_t __attribute__((ext_vector_type(2))); +typedef int int32x8_t __attribute__((ext_vector_type(8))); + +// Whether this compilation pass can emit V_WMMA_I32_16X16X16_IU8: only the +// device passes of gfx1200 / gfx1201. Note the host pass of the same build never +// has it, so this must not decide which kernel gets launched -- the launch site +// is unconditional and only the instruction inside the kernel is arch-gated. +#if defined(__HIP_DEVICE_COMPILE__) && (defined(__gfx1200__) || defined(__gfx1201__)) +#define RDNA2_WMMA_IU8 1 +#else +#define RDNA2_WMMA_IU8 0 +#endif + + +// The LDS tiling shape (RDNA2_TILE_MN, RDNA2_LDS_*_KSTEP/STRIDE) lives in +// rdna2_kernels_math.h so the host simulation uses the identical numbers. + +// --------------------------------------------------------------------------- +// Shared helpers +// --------------------------------------------------------------------------- +static inline int rdna2_device_count(void) { + int n = 0; + if (hipGetDeviceCount(&n) != hipSuccess) return 0; + return n; +} + +// 0 when the query fails or reports nonsense. Callers must then avoid shapes +// that depend on the wave width rather than guess one: the wave-per-row GEMV +// derives rows-per-block from blockDim/warpSize on the device, so a host guess +// of 32 with an actual wave64 would leave half the rows unprocessed. +static inline int rdna2_device_warp_size(void) { + hipDeviceProp_t prop{}; + if (hipGetDeviceProperties(&prop, 0) != hipSuccess || prop.warpSize <= 0) return 0; + const int ws = prop.warpSize; + if (ws > 64 || (ws & (ws - 1)) != 0) return 0; // not a power of two <= 64 + return ws; +} + +static inline int rdna2_warp_size(void) { + const int ws = rdna2_device_warp_size(); + return ws > 0 ? ws : 32; +} + +// --------------------------------------------------------------------------- +// Group 1: dp4a int8 GEMV. y[rows] = A[rows x k4] . x[k4] +// gfx1031 reads four packed words per load (16-byte vector access); any k4 is +// accepted because the remainder is handled by a scalar tail. Alignment is not +// a caller requirement: the launcher copies into a device buffer, which HIP +// allocates with far stronger alignment than the 16 bytes the vectors need. +// --------------------------------------------------------------------------- +__global__ void rdna2_dp4a_gemv_i8(int* __restrict__ y, + const int* __restrict__ A, // rows x k4 packed + const int* __restrict__ x, // k4 packed + int rows, int k4) { + int row = blockIdx.x * blockDim.x + threadIdx.x; + if (row >= rows) return; + const int* a_s = A + (size_t)row * k4; + // Four independent accumulators: v_dot4 has a multi-cycle result latency, so + // one dependent chain would leave the dot4 pipes idle. Integer addition is + // exact, so summing the four at the end is bit-identical to one chain. + int acc0 = 0, acc1 = 0, acc2 = 0, acc3 = 0; +#if defined(__gfx1031__) || defined(__gfx1200__) || defined(__gfx1201__) + int j = 0; + // The 16-byte vector load is only legal when this row also starts on a + // 16-byte boundary. A row starts at (row * k4) packed words, so that holds + // for every row only when k4 is a multiple of 4. For any other k4 the row + // stride is not 16-byte aligned and v_dot4 is fed from scalar loads instead + // (misaligned dwordx4 loads are not handled by the hardware). + if ((k4 & 3) == 0) { + const int4_t* a = reinterpret_cast(a_s); + const int4_t* xv = reinterpret_cast(x); + for (; j + 4 <= k4; j += 4) { + int4_t av = a[j / 4]; + int4_t xw = xv[j / 4]; + acc0 = rdna2_dot4_i8_impl(av.x, xw.x, acc0); + acc1 = rdna2_dot4_i8_impl(av.y, xw.y, acc1); + acc2 = rdna2_dot4_i8_impl(av.z, xw.z, acc2); + acc3 = rdna2_dot4_i8_impl(av.w, xw.w, acc3); + } + } + for (; j < k4; ++j) acc0 = rdna2_dot4_i8_impl(a_s[j], x[j], acc0); +#else + for (int j = 0; j < k4; ++j) { + acc0 = rdna2_dot4_i8_impl(a_s[j], x[j], acc0); + } +#endif + y[row] = (acc0 + acc1) + (acc2 + acc3); +} + +// One wave per row. Lanes stride the row, so a wave reads 32 consecutive words +// (one 128-byte line) per step instead of the 32 scattered row starts the +// thread-per-row kernel touches, and the row is read exactly once. Better than +// one thread per row once a row holds enough words to keep a wave busy; the +// launcher switches on k4 (see RDNA2_GEMV_WAVE_MIN_K4). +__global__ void rdna2_dp4a_gemv_i8_wave(int* __restrict__ y, + const int* __restrict__ A, + const int* __restrict__ x, int rows, int k4) { + const int ws = (int)warpSize; + const int lane = (int)(threadIdx.x % warpSize); + const int wave = (int)(threadIdx.x / warpSize); + const int waves = (int)(blockDim.x / warpSize); + const int row = (int)blockIdx.x * waves + wave; + if (row >= rows) return; // wave-uniform: a wave never splits across rows + const int* a_s = A + (size_t)row * k4; + int acc = 0; + for (int k = lane; k < k4; k += ws) acc = rdna2_dot4_i8_impl(a_s[k], x[k], acc); + for (int off = ws >> 1; off > 0; off >>= 1) acc += __shfl_xor(acc, off); + if (lane == 0) y[row] = acc; +} + +// --------------------------------------------------------------------------- +// Group 1c: f32 GEMV, y[M] = A[M x K] . x[K]. The same two shapes as the int8 +// GEMV above: one thread per row (short rows) and one wave per row (long rows, +// lanes stride the row so a wave touches consecutive floats). +// --------------------------------------------------------------------------- +__global__ void rdna2_sgemv_f32(float* __restrict__ y, const float* __restrict__ A, + const float* __restrict__ x, int rows, int K) { + const int row = (int)blockIdx.x * blockDim.x + (int)threadIdx.x; + if (row >= rows) return; + const float* a_s = A + (size_t)row * K; + y[row] = rdna2_sgemv_row_f32(a_s, x, K); +} + +__global__ void rdna2_sgemv_f32_wave(float* __restrict__ y, const float* __restrict__ A, + const float* __restrict__ x, int rows, int K) { + const int ws = (int)warpSize; + const int lane = (int)(threadIdx.x % warpSize); + const int wave = (int)(threadIdx.x / warpSize); + const int waves = (int)(blockDim.x / warpSize); + const int row = (int)blockIdx.x * waves + wave; + if (row >= rows) return; // wave-uniform: a wave never splits across rows + const float* a_s = A + (size_t)row * K; + float acc = rdna2_sgemv_lane_partial_f32(a_s, x, K, lane, ws); + for (int off = ws >> 1; off > 0; off >>= 1) acc += __shfl_xor(acc, off); + if (lane == 0) y[row] = acc; +} + +// --------------------------------------------------------------------------- +// Group 1b: dp4a int8 GEMM. +// C[M x N] = A[M x K4] * Bt[N x K4]^T (B stored transposed, N rows of K4) +// Two implementations: a naive one (one thread per element) and a 4x4 +// register-tiled one. The device test compares both against the CPU reference. +// --------------------------------------------------------------------------- +__global__ void rdna2_dp4a_gemm_i8_naive(int* __restrict__ C, + const int* __restrict__ A, + const int* __restrict__ Bt, + int M, int N, int K4) { + int n = blockIdx.x * blockDim.x + threadIdx.x; + int m = blockIdx.y; + if (m >= M || n >= N) return; + const int* a = A + (size_t)m * K4; + const int* b = Bt + (size_t)n * K4; + C[(size_t)m * N + n] = rdna2_gemm_cell_i8(a, b, K4); +} + +// 4x4 outputs per thread; block (16,16) covers a 64x64 output tile. +__global__ void rdna2_dp4a_gemm_i8_tile(int* __restrict__ C, + const int* __restrict__ A, + const int* __restrict__ Bt, + int M, int N, int K4) { + const int m0 = (blockIdx.y * blockDim.y + threadIdx.y) * 4; + const int n0 = (blockIdx.x * blockDim.x + threadIdx.x) * 4; + if (m0 >= M || n0 >= N) return; + + int32_t acc[4][4]; + rdna2_gemm_tile_i8(acc, A, Bt, M, N, K4, m0, n0); + for (int i = 0; i < 4; ++i) { + if (m0 + i >= M) break; + for (int j = 0; j < 4; ++j) { + if (n0 + j >= N) break; + C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 1c: LDS-staged tiled int8 GEMM, same 64x64 output tile and 4x4 register +// blocking as above, but the K loop reads the operands from LDS instead of +// global memory. The thread-per-cell version issues 8 packed loads per k step +// per thread; over a 64x64 tile that is 2048 words of traffic for 128 words of +// actual data (16x). Staging each K chunk cooperatively costs 1024 loads spread +// over 256 threads (4 each) and then every reuse is LDS. +// --------------------------------------------------------------------------- +// Stage one K chunk of a matrix into LDS. 256 threads cover 64 rows x 16 words; +// each thread owns 4 consecutive words of one row. Rows past the edge and words +// past K4 are written as zero, so the compute loop needs no bounds checks. +// lds is a template parameter so the __shared__ address space survives: a fixed +// int (*)[STRIDE] parameter would genericize it to flat stores. +template +__device__ static inline void rdna2_stage_i8(LDP lds, const int* __restrict__ src, int nrows, + int ncols, int row0, int k0, int ld_row, int ld_col) { + const int gm = row0 + ld_row; + int4_t v; + v.x = v.y = v.z = v.w = 0; + if (gm < nrows) { + const int* p = src + (size_t)gm * ncols + k0 + ld_col; + if ((ncols & 3) == 0 && k0 + ld_col + 4 <= ncols) { + v = *reinterpret_cast(p); // 16-byte access, aligned by ncols%4==0 + } else { + for (int c = 0; c < 4; ++c) + if (k0 + ld_col + c < ncols) v[c] = p[c]; + } + } + lds[ld_row][ld_col + 0] = v.x; + lds[ld_row][ld_col + 1] = v.y; + lds[ld_row][ld_col + 2] = v.z; + lds[ld_row][ld_col + 3] = v.w; +} + +__global__ void rdna2_dp4a_gemm_i8_lds(int* __restrict__ C, + const int* __restrict__ A, + const int* __restrict__ Bt, + int M, int N, int K4) { + __shared__ int a_lds[RDNA2_TILE_MN][RDNA2_LDS_I8_STRIDE]; + __shared__ int b_lds[RDNA2_TILE_MN][RDNA2_LDS_I8_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int m_blk = blockIdx.y * RDNA2_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_TILE_MN; + const int lm = threadIdx.y * 4; // this thread's first tile row + const int ln = threadIdx.x * 4; // this thread's first tile column + const int ld_row = tid >> 2; + const int ld_col = (tid & 3) << 2; + + int32_t acc[4][4]; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = 0; + + for (int k0 = 0; k0 < K4; k0 += RDNA2_LDS_I8_KSTEP) { + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col); + __syncthreads(); + int kend = K4 - k0; + if (kend > RDNA2_LDS_I8_KSTEP) kend = RDNA2_LDS_I8_KSTEP; + rdna2_gemm_lds_block_i8(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_LDS_I8_STRIDE, lm, ln, kend); + __syncthreads(); // nobody may restage a tile that is still being read + } + + for (int i = 0; i < 4; ++i) { + if (m_blk + lm + i >= M) break; + for (int j = 0; j < 4; ++j) { + if (n_blk + ln + j >= N) break; + C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; + } + } +} + +// Same LDS tile and K loop, 8x4 register blocking: 128 threads (16,8) cover the +// same 64x64 output tile, each owning 8 rows x 4 columns. Twelve LDS reads per k +// step feed 32 dp4a instead of eight feeding 16, i.e. a quarter less LDS traffic +// for identical math. Staging spreads the same 64x16 words over 128 threads, so +// each thread writes 8 words (two 16-byte loads) instead of 4. +__global__ void rdna2_dp4a_gemm_i8_lds8(int* __restrict__ C, + const int* __restrict__ A, + const int* __restrict__ Bt, + int M, int N, int K4) { + __shared__ int a_lds[RDNA2_TILE_MN][RDNA2_LDS_I8_STRIDE]; + __shared__ int b_lds[RDNA2_TILE_MN][RDNA2_LDS_I8_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int m_blk = blockIdx.y * RDNA2_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_TILE_MN; + const int lm = threadIdx.y * 8; // this thread's first tile row + const int ln = threadIdx.x * 4; // this thread's first tile column + const int ld_row = tid >> 1; // 0..63, two threads per staged row + const int ld_col = (tid & 1) << 3; + + int32_t acc[8][4]; + for (int i = 0; i < 8; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = 0; + + for (int k0 = 0; k0 < K4; k0 += RDNA2_LDS_I8_KSTEP) { + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col); + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col + 4); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col + 4); + __syncthreads(); + int kend = K4 - k0; + if (kend > RDNA2_LDS_I8_KSTEP) kend = RDNA2_LDS_I8_KSTEP; + rdna2_gemm_lds_block8_i8(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_LDS_I8_STRIDE, lm, ln, kend); + __syncthreads(); // nobody may restage a tile that is still being read + } + + for (int i = 0; i < 8; ++i) { + if (m_blk + lm + i >= M) break; + for (int j = 0; j < 4; ++j) { + if (n_blk + ln + j >= N) break; + C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 1e: RDNA4 (gfx1200 / gfx1201) path. Same 16x16 block of 256 threads and +// the same 16-word K chunk as groups 1c/1d, but a 128x128 output tile with 8x8 +// outputs per thread: sixteen LDS reads per k step feed 64 dp4a instead of eight +// feeding 16, and every staged word pays for four times as many results. gfx12 +// has 1536 VGPRs per SIMD against 1024 on gfx10/11, so 64 accumulators plus 16 +// staged operands still leave several waves resident. Two staged tiles of +// 128 x 17 words = 17408 bytes, inside the 64 KiB a gfx12 workgroup addresses. +// Select with tiled = 4 (see rdna2_tiled_for_arch). +// --------------------------------------------------------------------------- +__global__ void rdna2_dp4a_gemm_i8_lds128(int* __restrict__ C, + const int* __restrict__ A, + const int* __restrict__ Bt, + int M, int N, int K4) { + __shared__ int a_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_I8_STRIDE]; + __shared__ int b_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_I8_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int m_blk = blockIdx.y * RDNA2_T4_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_T4_TILE_MN; + const int lm = threadIdx.y * 8; // this thread's first tile row + const int ln = threadIdx.x * 8; // this thread's first tile column + const int ld_row = tid >> 1; // 0..127: two threads per staged row + const int ld_col = (tid & 1) << 3; // 0 or 8, eight staged words each + + int32_t acc[8][8]; + for (int i = 0; i < 8; ++i) + for (int j = 0; j < 8; ++j) acc[i][j] = 0; + + for (int k0 = 0; k0 < K4; k0 += RDNA2_T4_LDS_I8_KSTEP) { + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col); + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col + 4); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col + 4); + __syncthreads(); + int kend = K4 - k0; + if (kend > RDNA2_T4_LDS_I8_KSTEP) kend = RDNA2_T4_LDS_I8_KSTEP; + rdna2_gemm_lds_block8x8_i8(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_T4_LDS_I8_STRIDE, lm, + ln, kend); + __syncthreads(); // nobody may restage a tile that is still being read + } + + for (int i = 0; i < 8; ++i) { + if (m_blk + lm + i >= M) break; + for (int j = 0; j < 8; ++j) { + if (n_blk + ln + j >= N) break; + C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 1f: RDNA4 (gfx1200 / gfx1201) matrix path. Same 128x128 output tile, +// same 16-word K chunk and same LDS staging as group 1e, but the arithmetic is +// the gfx12 matrix instruction V_WMMA_I32_16X16X16_IU8 instead of 64 dp4a per +// thread: one instruction covers a 16x16x16 signed product that the 8x8 dp4a +// kernel needs 64 instructions and 16 LDS reads per k step to reach. +// +// The eight waves of the block each own a 16-row band of the tile and sweep the +// eight 16-column groups, keeping 8 x int32x8 accumulators (64 VGPRs, the same +// budget as group 1e's 8x8 block). Per K chunk a wave issues 32 WMMA -- 8 per +// 16-byte K step -- and pays only 2 LDS words for A and 2 per B fragment, i.e. +// 36 word reads for 32 matrix ops instead of 256 reads for 1024 dp4a. +// +// Fragment layout, measured on a gfx1201 card (256/256 outputs of a signed +// 16x16x16 product matched the scalar reference; the layout and the negA/negB +// semantics were both derived experimentally, see README): +// A: lane t, byte e -> A[M = t%16][K = (t/16)*8 + e] +// B: lane t, byte e -> B[K = (t/16)*8 + e][N = t%16] +// D: lane t, slot l -> D[M = (t/16)*8 + l][N = t%16] +// Both operands are signed, so negA and negB are true; __builtin_amdgcn_wmma_* +// needs those flags as compile-time constants, hence the literals. +// Select with tiled = 5 (see rdna2_tiled_for_arch). +// --------------------------------------------------------------------------- +// One 16x16x16 signed step for this lane: acc holds this lane's eight output +// slots, a and b are its eight-byte A and B fragments. +// gfx12 -- the matrix instruction. +// other AMD GPU -- no matrix unit on gfx10/11, so the same result is built +// from explicit byte dots, gathering each slot's operands +// out of lanes row and row+16 with shuffles. Correct, but +// with a shuffle per operand byte it cannot beat the dp4a +// kernel, and rdna2_tiled_for_arch never picks tiled = 5 on +// those chips. Not measured: no RDNA2 hardware here. +// plain C++ pass -- the file also compiles as host code (CPU test harness and +// the host half of a HIP build), where no kernel can run; the +// body is only there to keep the kernel defined so the +// unconditional launch site links. +__device__ static inline int32x8_t rdna2_wmma_iu8_step(int32x2_t a, int32x2_t b, int32x8_t acc, + int lane) { + (void)lane; // the matrix instruction addresses the other lanes in hardware +#if RDNA2_WMMA_IU8 + return __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a, true, b, acc, false); +#elif defined(__HIP_DEVICE_COMPILE__) + for (int l = 0; l < 8; ++l) { + const int dm = rdna2_wmma_i8_acc_row(lane, l); + const int dn = rdna2_wmma_i8_acc_col(lane); + int32_t d = 0; + for (int k = 0; k < 16; ++k) { + const int shift = 8 * (k & 3), word = (k & 7) >> 2; + const int av = (int8_t)(__shfl((int)a[word], dm + (k >> 3) * 16) >> shift); + const int bv = (int8_t)(__shfl((int)b[word], dn + (k >> 3) * 16) >> shift); + d += av * bv; + } + acc[l] += d; + } + return acc; +#else + (void)a; + (void)b; + return acc; +#endif +} + +__global__ void rdna2_dp4a_gemm_i8_wmma(int* __restrict__ C, + const int* __restrict__ A, + const int* __restrict__ Bt, + int M, int N, int K4) { + __shared__ int a_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_I8_STRIDE]; + __shared__ int b_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_I8_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int lane = tid & 31; + const int wave = tid >> 5; // 0..7: this wave's 16-row band of the tile + const int m_blk = blockIdx.y * RDNA2_T4_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_T4_TILE_MN; + const int m_row = wave * 16 + rdna2_wmma_i8_row(lane); // staged A row this lane feeds + const int n_col = rdna2_wmma_i8_row(lane); // column group offset inside a 16-wide group + const int ld_row = tid >> 1; // 0..127: two threads per staged row + const int ld_col = (tid & 1) << 3; // 0 or 8, eight staged words each + + int32x8_t acc[RDNA2_T4_WMMA_NTILE]; + for (int j = 0; j < RDNA2_T4_WMMA_NTILE; ++j) { + for (int l = 0; l < 8; ++l) acc[j][l] = 0; + } + + for (int k0 = 0; k0 < K4; k0 += RDNA2_T4_LDS_I8_KSTEP) { + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col); + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col + 4); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col + 4); + __syncthreads(); + int kend = K4 - k0; + if (kend > RDNA2_T4_LDS_I8_KSTEP) kend = RDNA2_T4_LDS_I8_KSTEP; + // One WMMA covers 4 staged words (16 K bytes); a short last chunk runs the + // steps it has and relies on the staging zeros for the rest of the tile. + const int ksteps = (kend + 3) / 4; + for (int ks = 0; ks < ksteps; ++ks) { + const int w = rdna2_wmma_i8_frag_word(lane, ks); + int32x2_t a; + a[0] = a_lds[m_row][w]; + a[1] = a_lds[m_row][w + 1]; + for (int j = 0; j < RDNA2_T4_WMMA_NTILE; ++j) { + int32x2_t b; + b[0] = b_lds[j * 16 + n_col][w]; + b[1] = b_lds[j * 16 + n_col][w + 1]; + acc[j] = rdna2_wmma_iu8_step(a, b, acc[j], lane); + } + } + __syncthreads(); // nobody may restage a tile that is still being read + } + + // D map: slot l of lane t lands at row 8*(t/16) + l, column t%16 of the + // 16x16 tile, so a lane writes one column of its wave's 16-row band. + for (int j = 0; j < RDNA2_T4_WMMA_NTILE; ++j) { + const int n = n_blk + j * 16 + rdna2_wmma_i8_acc_col(lane); + if (n >= N) continue; + for (int l = 0; l < 8; ++l) { + const int m = m_blk + wave * 16 + rdna2_wmma_i8_acc_row(lane, l); + if (m >= M) break; // rows 8*(t/16)+l are contiguous, so a break is safe + C[(size_t)m * N + n] = acc[j][l]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 2: f32 GEMM, 4x4 register tiled. C[M x N] = A[M x K] * Bt[N x K]^T +// --------------------------------------------------------------------------- +__global__ void rdna2_sgemm_f32(float* __restrict__ C, + const float* __restrict__ A, + const float* __restrict__ Bt, + int M, int N, int K) { + const int m0 = (blockIdx.y * blockDim.y + threadIdx.y) * 4; + const int n0 = (blockIdx.x * blockDim.x + threadIdx.x) * 4; + if (m0 >= M || n0 >= N) return; + + float acc[4][4]; + rdna2_sgemm_tile_f32(acc, A, Bt, M, N, K, m0, n0); + for (int i = 0; i < 4; ++i) { + if (m0 + i >= M) break; + for (int j = 0; j < 4; ++j) { + if (n0 + j >= N) break; + C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 2b: LDS-staged f32 GEMM, same 64x64 tile and 4x4 register blocking as +// group 1c: the K loop reads A and Bt from LDS instead of re-reading global +// memory in every thread. +// --------------------------------------------------------------------------- +// 256 threads cover 64 rows x 8 floats: two consecutive floats per thread. +template +__device__ static inline void rdna2_stage_f32(LDP lds, const float* __restrict__ src, int nrows, + int ncols, int row0, int k0, int ld_row, int ld_col) { + const int gm = row0 + ld_row; + float v0 = 0.0f, v1 = 0.0f; + if (gm < nrows) { + const float* p = src + (size_t)gm * ncols + k0 + ld_col; + if (k0 + ld_col + 0 < ncols) v0 = p[0]; + if (k0 + ld_col + 1 < ncols) v1 = p[1]; + } + lds[ld_row][ld_col + 0] = v0; + lds[ld_row][ld_col + 1] = v1; +} + +__global__ void rdna2_sgemm_f32_lds(float* __restrict__ C, + const float* __restrict__ A, + const float* __restrict__ Bt, + int M, int N, int K) { + __shared__ float a_lds[RDNA2_TILE_MN][RDNA2_LDS_F32_STRIDE]; + __shared__ float b_lds[RDNA2_TILE_MN][RDNA2_LDS_F32_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int m_blk = blockIdx.y * RDNA2_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_TILE_MN; + const int lm = threadIdx.y * 4; + const int ln = threadIdx.x * 4; + const int ld_row = tid >> 2; + const int ld_col = (tid & 3) << 1; + + float acc[4][4]; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = 0.0f; + + for (int k0 = 0; k0 < K; k0 += RDNA2_LDS_F32_KSTEP) { + rdna2_stage_f32(a_lds, A, M, K, m_blk, k0, ld_row, ld_col); + rdna2_stage_f32(b_lds, Bt, N, K, n_blk, k0, ld_row, ld_col); + __syncthreads(); + int kend = K - k0; + if (kend > RDNA2_LDS_F32_KSTEP) kend = RDNA2_LDS_F32_KSTEP; + rdna2_sgemm_lds_block_f32(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_LDS_F32_STRIDE, lm, ln, kend); + __syncthreads(); + } + + for (int i = 0; i < 4; ++i) { + if (m_blk + lm + i >= M) break; + for (int j = 0; j < 4; ++j) { + if (n_blk + ln + j >= N) break; + C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 2b: 64x64 f32 tile with a 16-float K chunk (variant 3). Same block +// shape and 4x4 per-thread blocking as group 2, but 256 threads stage 1024 +// floats (4 each, RDNA2_LDS_F32B_STRIDE = 17 per row) instead of 512 floats +// (2 each) per iteration. That makes each staged chunk cover a whole 64-byte +// cache line of every row, and halves the number of __syncthreads over K. +// --------------------------------------------------------------------------- +template +__device__ static inline void rdna2_stage_f32_4(LDP lds, const float* __restrict__ src, int nrows, + int ncols, int row0, int k0, int ld_row, + int ld_col) { + const int gm = row0 + ld_row; + float v0 = 0.0f, v1 = 0.0f, v2 = 0.0f, v3 = 0.0f; + if (gm < nrows) { + const float* p = src + (size_t)gm * ncols + k0 + ld_col; + const int rem = ncols - (k0 + ld_col); + if (rem > 0) v0 = p[0]; + if (rem > 1) v1 = p[1]; + if (rem > 2) v2 = p[2]; + if (rem > 3) v3 = p[3]; + } + lds[ld_row][ld_col + 0] = v0; + lds[ld_row][ld_col + 1] = v1; + lds[ld_row][ld_col + 2] = v2; + lds[ld_row][ld_col + 3] = v3; +} + +__global__ void rdna2_sgemm_f32_lds16(float* __restrict__ C, + const float* __restrict__ A, + const float* __restrict__ Bt, + int M, int N, int K) { + __shared__ float a_lds[RDNA2_TILE_MN][RDNA2_LDS_F32B_STRIDE]; + __shared__ float b_lds[RDNA2_TILE_MN][RDNA2_LDS_F32B_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int m_blk = blockIdx.y * RDNA2_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_TILE_MN; + const int lm = threadIdx.y * 4; + const int ln = threadIdx.x * 4; + const int ld_row = tid >> 2; + const int ld_col = (tid & 3) << 2; + + float acc[4][4]; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = 0.0f; + + for (int k0 = 0; k0 < K; k0 += RDNA2_LDS_F32B_KSTEP) { + rdna2_stage_f32_4(a_lds, A, M, K, m_blk, k0, ld_row, ld_col); + rdna2_stage_f32_4(b_lds, Bt, N, K, n_blk, k0, ld_row, ld_col); + __syncthreads(); + int kend = K - k0; + if (kend > RDNA2_LDS_F32B_KSTEP) kend = RDNA2_LDS_F32B_KSTEP; + rdna2_sgemm_lds_block_f32(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_LDS_F32B_STRIDE, lm, ln, kend); + __syncthreads(); + } + + for (int i = 0; i < 4; ++i) { + if (m_blk + lm + i >= M) break; + for (int j = 0; j < 4; ++j) { + if (n_blk + ln + j >= N) break; + C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 2c: RDNA4 (gfx1200 / gfx1201) f32 GEMM on the same 128x128 tile and 8x8 +// per-thread blocking as group 1e. Two staged tiles of 128 x 9 floats = 9216 +// bytes; sixteen LDS reads per k step feed 64 fma. Select with variant 2. +// --------------------------------------------------------------------------- +__global__ void rdna2_sgemm_f32_lds128(float* __restrict__ C, + const float* __restrict__ A, + const float* __restrict__ Bt, + int M, int N, int K) { + __shared__ float a_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_F32_STRIDE]; + __shared__ float b_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_F32_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int m_blk = blockIdx.y * RDNA2_T4_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_T4_TILE_MN; + const int lm = threadIdx.y * 8; + const int ln = threadIdx.x * 8; + const int ld_row = tid >> 1; // 0..127: two threads per staged row + const int ld_col = (tid & 1) << 2; // 0 or 4, four staged floats each + + float acc[8][8]; + for (int i = 0; i < 8; ++i) + for (int j = 0; j < 8; ++j) acc[i][j] = 0.0f; + + for (int k0 = 0; k0 < K; k0 += RDNA2_T4_LDS_F32_KSTEP) { + rdna2_stage_f32(a_lds, A, M, K, m_blk, k0, ld_row, ld_col); + rdna2_stage_f32(a_lds, A, M, K, m_blk, k0, ld_row, ld_col + 2); + rdna2_stage_f32(b_lds, Bt, N, K, n_blk, k0, ld_row, ld_col); + rdna2_stage_f32(b_lds, Bt, N, K, n_blk, k0, ld_row, ld_col + 2); + __syncthreads(); + int kend = K - k0; + if (kend > RDNA2_T4_LDS_F32_KSTEP) kend = RDNA2_T4_LDS_F32_KSTEP; + rdna2_sgemm_lds_block8x8_f32(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_T4_LDS_F32_STRIDE, lm, + ln, kend); + __syncthreads(); + } + + for (int i = 0; i < 8; ++i) { + if (m_blk + lm + i >= M) break; + for (int j = 0; j < 8; ++j) { + if (n_blk + ln + j >= N) break; + C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 3: packed f16 dot (v_dot2_f32_f16; clang emits the accumulate form +// from __builtin_amdgcn_fdot2, and the ISA mnemonic has no 'c'). +// out is accumulated with atomics and must be zeroed by the launcher. +// Correct for both wave32 and wave64: the shuffle reduction covers one +// 16-lane row, then one atomic per row. +// --------------------------------------------------------------------------- +__global__ void rdna2_dot2_f16(float* __restrict__ out, + const half2_t* __restrict__ a, + const half2_t* __restrict__ b, + int n) { + float acc = 0.0f; + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < n; + i += gridDim.x * blockDim.x) { + acc = __builtin_amdgcn_fdot2(a[i], b[i], acc, false); + } + acc += __shfl_xor(acc, 1); + acc += __shfl_xor(acc, 2); + acc += __shfl_xor(acc, 4); + acc += __shfl_xor(acc, 8); + if ((threadIdx.x & 15) == 0) atomicAdd(out, acc); +} + +// --------------------------------------------------------------------------- +// Group 4: dp4a u8 dot (v_dot4_u32_u8). +// out is accumulated with atomics and must be zeroed by the launcher. +// --------------------------------------------------------------------------- +__global__ void rdna2_dp4a_dot_u8(unsigned* __restrict__ out, + const unsigned* __restrict__ a, + const unsigned* __restrict__ b, + int n) { + unsigned acc = 0; + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < n; + i += gridDim.x * blockDim.x) { + acc = __builtin_amdgcn_udot4(a[i], b[i], acc, false); + } + acc += __shfl_xor(acc, 1); + acc += __shfl_xor(acc, 2); + acc += __shfl_xor(acc, 4); + acc += __shfl_xor(acc, 8); + if ((threadIdx.x & 15) == 0) atomicAdd(out, acc); +} + +// --------------------------------------------------------------------------- +// Group 5: inclusive prefix sum over one wave. +// DPP row_shr builds a 16-lane Hillis-Steele scan with no LDS traffic; the +// cross-half total then arrives through a shuffle (v_readlane/v_permlane). +// out[tid] = in[0] + ... + in[tid] for every lane tid < n <= warpSize. +// --------------------------------------------------------------------------- +__global__ void rdna2_dpp_prefix(int* __restrict__ out, + const int* __restrict__ in, + int n) { + const int lane = threadIdx.x & (warpSize - 1); + int v = (lane < n) ? in[lane] : 0; + + // row_shr:1,2,4,8 with bound_ctrl off: lanes shifted out of the row get 0. + v += __builtin_amdgcn_update_dpp(0, v, 0x111, 0xf, 0xf, false); + v += __builtin_amdgcn_update_dpp(0, v, 0x112, 0xf, 0xf, false); + v += __builtin_amdgcn_update_dpp(0, v, 0x114, 0xf, 0xf, false); + v += __builtin_amdgcn_update_dpp(0, v, 0x118, 0xf, 0xf, false); + + // Lane 15 holds the total of lanes 0-15; add it to the upper half. + int lo = __shfl(v, 15); + if (lane >= 16) v += lo; + // On wave64, lanes 32-63 also need the total of lanes 0-31. + if (warpSize == 64) { + int mid = __shfl(v, 31); + if (lane >= 32) v += mid; + } + out[lane] = v; +} + +// --------------------------------------------------------------------------- +// Group 6: quantized epilogue (scale and zero point) +// --------------------------------------------------------------------------- +// Per-row sum of the int8 activations. dp4a against an all-ones operand sums the +// four signed bytes of a packed word in one instruction, so this is one v_dot4 +// per word. Only needed for asymmetric weights (a nonzero zero point); a +// symmetric format passes zp_b = 0 and skips it entirely. +__global__ void rdna2_i8_row_sums(int* __restrict__ sums, const int* __restrict__ A, int rows, + int k4) { + const int row = blockIdx.x * blockDim.x + threadIdx.x; + if (row >= rows) return; + const int* a = A + (size_t)row * k4; + const int ones = 0x01010101; + int acc = 0; + for (int k = 0; k < k4; ++k) acc = rdna2_dot4_i8_impl(a[k], ones, acc); + sums[row] = acc; +} + +// Applies the quantized format's scale and zero point to the finished integer +// accumulators: +// C[m][n] = (acc[m][n] - zp_b * sums[m]) * sa[m] * sb[n] +// The correction is the exact integer form of B's asymmetric storage: B's stored +// value at k represents (q - zp_b), so sum_k a*(q - zp_b) = acc - zp_b*sum_k a. +// No stored weight is converted here - only the completed dot product is, once, +// after the correction has been applied in 64-bit so it cannot overflow. sa is +// NULL for unit row scale, sb NULL for unit column scale. +__global__ void rdna2_gemm_i8_epilogue(float* __restrict__ C, const int* __restrict__ acc, + const int* __restrict__ sums, int M, int N, + const float* __restrict__ sa, const float* __restrict__ sb, + int zp_b) { + const int n = blockIdx.x * blockDim.x + threadIdx.x; + if (n >= N) return; + // Rows stride over gridDim.y so the row count is not capped at the grid's + // y limit; each iteration is an independent output element. + for (int m = blockIdx.y; m < M; m += gridDim.y) { + long long v = (long long)acc[(size_t)m * N + n]; + if (zp_b != 0 && sums) v -= (long long)zp_b * (long long)sums[m]; + float f = (float)v; + if (sa) f *= sa[m]; + if (sb) f *= sb[n]; + C[(size_t)m * N + n] = f; + } +} + +// --------------------------------------------------------------------------- +// Host launchers +// --------------------------------------------------------------------------- +// One thread per row is the better shape for short rows (no reduction to pay +// for); from this many packed words per row on, a whole wave per row wins on +// coalescing. +#define RDNA2_GEMV_WAVE_MIN_K4 8 +// f32 rows need a real body before a wave pays for its reduction. Measured on a +// 9070 XT at 2048-16384 rows: below K=256 the five shuffle steps cost more than +// the coalescing gains (thread-per-row is 1.2-3.8x ahead at K=64..160), and from +// K=256 up the wave-per-row kernel wins every time (1.15x at K=256, ~2x by +// K=1024, 3.5x at K=4096 with 16384 rows). +#define RDNA2_SGEMV_WAVE_MIN_K 256 + +static int rdna2_gemv_i8_device(int* y, const int* A, const int* x, int rows, int k4, + int variant) { + int *dA = nullptr, *dx = nullptr, *dy = nullptr; + const size_t bytesA = (size_t)rows * k4 * sizeof(int); + const size_t bytesx = (size_t)k4 * sizeof(int); + const size_t bytesy = (size_t)rows * sizeof(int); + int rc = RDNA2_OK; + int ws = 0; // wave width; stays 0 unless the wave-per-row variant can size its grid + if (hipMalloc(&dA, bytesA) != hipSuccess || hipMalloc(&dx, bytesx) != hipSuccess || + hipMalloc(&dy, bytesy) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(dA, A, bytesA, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(dx, x, bytesx, hipMemcpyHostToDevice) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (variant == 1) ws = rdna2_device_warp_size(); + if (ws > 0) { + const int threads = 128; // 4 waves of 32 or 2 waves of 64 + const int waves = threads / ws; + const int blocks = (rows + waves - 1) / waves; + hipLaunchKernelGGL(rdna2_dp4a_gemv_i8_wave, dim3(blocks), dim3(threads), 0, 0, + dy, dA, dx, rows, k4); + } else { + const int threads = 64; + const int blocks = (rows + threads - 1) / threads; + hipLaunchKernelGGL(rdna2_dp4a_gemv_i8, dim3(blocks), dim3(threads), 0, 0, + dy, dA, dx, rows, k4); + } + if (hipGetLastError() != hipSuccess || + hipMemcpy(y, dy, bytesy, hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } +cleanup: + if (dA) (void)hipFree(dA); + if (dx) (void)hipFree(dx); + if (dy) (void)hipFree(dy); + return rc; +} + +// variant 0 = one thread per row, 1 = one wave per row. +int rdna2_gemv_i8_host_variant(int* y, const int* A, const int* x, int rows, int k4, int variant) { + if (!y || !A || !x || rows <= 0 || k4 <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + rdna2_gemv_i8_ref(y, A, x, rows, k4); + return RDNA2_OK; + } + return rdna2_gemv_i8_device(y, A, x, rows, k4, variant); +} + +int rdna2_gemv_i8_host(int* y, const int* A, const int* x, int rows, int k4) { + return rdna2_gemv_i8_host_variant(y, A, x, rows, k4, k4 >= RDNA2_GEMV_WAVE_MIN_K4 ? 1 : 0); +} + +static int rdna2_sgemv_f32_device(float* y, const float* A, const float* x, int rows, int K, + int variant) { + float *dA = nullptr, *dx = nullptr, *dy = nullptr; + const size_t bytesA = (size_t)rows * K * sizeof(float); + const size_t bytesx = (size_t)K * sizeof(float); + const size_t bytesy = (size_t)rows * sizeof(float); + int rc = RDNA2_OK; + int ws = 0; // wave width; stays 0 unless the wave-per-row variant can size its grid + if (hipMalloc(&dA, bytesA) != hipSuccess || hipMalloc(&dx, bytesx) != hipSuccess || + hipMalloc(&dy, bytesy) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(dA, A, bytesA, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(dx, x, bytesx, hipMemcpyHostToDevice) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (variant == 1) ws = rdna2_device_warp_size(); + if (ws > 0) { + const int threads = 128; // 4 waves of 32 or 2 waves of 64 + const int waves = threads / ws; + const int blocks = (rows + waves - 1) / waves; + hipLaunchKernelGGL(rdna2_sgemv_f32_wave, dim3(blocks), dim3(threads), 0, 0, + dy, dA, dx, rows, K); + } else { + const int threads = 64; + const int blocks = (rows + threads - 1) / threads; + hipLaunchKernelGGL(rdna2_sgemv_f32, dim3(blocks), dim3(threads), 0, 0, + dy, dA, dx, rows, K); + } + if (hipGetLastError() != hipSuccess || + hipMemcpy(y, dy, bytesy, hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } +cleanup: + if (dA) (void)hipFree(dA); + if (dx) (void)hipFree(dx); + if (dy) (void)hipFree(dy); + return rc; +} + +// variant 0 = one thread per row, 1 = one wave per row. +int rdna2_sgemv_f32_host_variant(float* y, const float* A, const float* x, int rows, int K, + int variant) { + if (!y || !A || !x || rows <= 0 || K <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + rdna2_sgemv_f32_ref(y, A, x, rows, K); + return RDNA2_OK; + } + return rdna2_sgemv_f32_device(y, A, x, rows, K, variant); +} + +int rdna2_sgemv_f32_host(float* y, const float* A, const float* x, int rows, int K) { + return rdna2_sgemv_f32_host_variant(y, A, x, rows, K, + K >= RDNA2_SGEMV_WAVE_MIN_K ? 1 : 0); +} + +// Shapes the int8 GEMM onto a grid and launches it. Shared by the plain entry +// point and the quantized-epilogue one so both measure the same kernels. +static void rdna2_launch_gemm_i8(int* dC, const int* dA, const int* dB, int M, int N, int K4, + int tiled) { + if (tiled == 4 || tiled == 5) { + // RDNA4 shape: 128x128 tile, 16x16 threads. 17408 B of LDS per block for + // int8, well inside the 64 KiB workgroup limit. tiled == 5 is the same tile + // driven by the matrix instruction instead of dp4a, so it shares the grid; + // which instruction runs is decided inside the kernel (see + // rdna2_wmma_iu8_step), not here -- this function is host code and cannot + // see the offload arch. + const dim3 grid((N + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN, + (M + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN, 1); + const dim3 block(16, 16, 1); + if (tiled == 5) + hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_wmma, grid, block, 0, 0, dC, dA, dB, M, N, K4); + else + hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_lds128, grid, block, 0, 0, dC, dA, dB, M, N, K4); + } else if (tiled) { + const dim3 grid((N + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN, + (M + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN, 1); + if (tiled == 3) { + const dim3 block(16, 8, 1); // 128 threads, 8x4 outputs each + hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_lds8, grid, block, 0, 0, dC, dA, dB, M, N, K4); + } else if (tiled == 2) { + const dim3 block(16, 16, 1); + hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_lds, grid, block, 0, 0, dC, dA, dB, M, N, K4); + } else { + const dim3 block(16, 16, 1); + hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_tile, grid, block, 0, 0, dC, dA, dB, M, N, K4); + } + } else { + const dim3 block(64, 1, 1); + const dim3 grid((N + 63) / 64, M, 1); + hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_naive, grid, block, 0, 0, dC, dA, dB, M, N, K4); + } +} + +static int rdna2_gemm_i8_device(int* C, const int* A, const int* Bt, int M, int N, int K4, + int tiled) { + int *dA = nullptr, *dB = nullptr, *dC = nullptr; + const size_t bytesA = (size_t)M * K4 * sizeof(int); + const size_t bytesB = (size_t)N * K4 * sizeof(int); + const size_t bytesC = (size_t)M * N * sizeof(int); + int rc = RDNA2_OK; + if (hipMalloc(&dA, bytesA) != hipSuccess || hipMalloc(&dB, bytesB) != hipSuccess || + hipMalloc(&dC, bytesC) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(dA, A, bytesA, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(dB, Bt, bytesB, hipMemcpyHostToDevice) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + rdna2_launch_gemm_i8(dC, dA, dB, M, N, K4, tiled); + if (hipGetLastError() != hipSuccess || + hipMemcpy(C, dC, bytesC, hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } +cleanup: + if (dA) (void)hipFree(dA); + if (dB) (void)hipFree(dB); + if (dC) (void)hipFree(dC); + return rc; +} + +// tiled: 0 = naive, 1 = 4x4 register tiled (global loads), 2 = same tiling with +// LDS-staged K chunks, 3 = LDS staging with 8x4 register blocking (quarter less +// LDS traffic for the same math), 4 = RDNA4 128x128 tile with 8x8 blocking. +// Any non-zero value keeps the tiled shape; 4 picks its own grid. +int rdna2_gemm_i8_host(int* C, const int* A, const int* Bt, int M, int N, int K4, int tiled) { + if (!C || !A || !Bt || M <= 0 || N <= 0 || K4 <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + rdna2_gemm_i8_ref(C, A, Bt, M, N, K4); + return RDNA2_OK; + } + return rdna2_gemm_i8_device(C, A, Bt, M, N, K4, tiled); +} + +// Quantized entry point: the same int8 GEMM followed by the format's scale and +// zero point. A and Bt are consumed exactly as stored - no weight is expanded to +// float, and the only float produced is the finished dot product, once. C is f32 +// because a GEMM result feeds the next layer's float activation path; sa scales +// rows of A and sb columns of Bt (either may be NULL for 1.0), zp_b is B's zero +// point (0 for symmetric weights, in which case the row-sum pass is skipped). +static int rdna2_gemm_i8_scale_device(float* C, const int* A, const int* Bt, int M, int N, int K4, + int tiled, const float* sa, const float* sb, int zp_b) { + int *dA = nullptr, *dB = nullptr, *dAcc = nullptr, *dSums = nullptr; + float *dSa = nullptr, *dSb = nullptr, *dC = nullptr; + const size_t bytesA = (size_t)M * K4 * sizeof(int); + const size_t bytesB = (size_t)N * K4 * sizeof(int); + const size_t nOut = (size_t)M * N; + int rc = RDNA2_OK; + if (hipMalloc(&dA, bytesA) != hipSuccess || hipMalloc(&dB, bytesB) != hipSuccess || + hipMalloc(&dAcc, nOut * sizeof(int)) != hipSuccess || + hipMalloc(&dC, nOut * sizeof(float)) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if ((zp_b != 0 && hipMalloc(&dSums, (size_t)M * sizeof(int)) != hipSuccess) || + (sa && hipMalloc(&dSa, (size_t)M * sizeof(float)) != hipSuccess) || + (sb && hipMalloc(&dSb, (size_t)N * sizeof(float)) != hipSuccess)) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(dA, A, bytesA, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(dB, Bt, bytesB, hipMemcpyHostToDevice) != hipSuccess || + (sa && hipMemcpy(dSa, sa, (size_t)M * sizeof(float), hipMemcpyHostToDevice) != hipSuccess) || + (sb && hipMemcpy(dSb, sb, (size_t)N * sizeof(float), hipMemcpyHostToDevice) != hipSuccess)) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + rdna2_launch_gemm_i8(dAcc, dA, dB, M, N, K4, tiled); + if (zp_b != 0) { + const int threads = 128; + hipLaunchKernelGGL(rdna2_i8_row_sums, dim3((M + threads - 1) / threads), dim3(threads), 0, 0, + dSums, dA, M, K4); + } + { + const dim3 block(256, 1, 1); + const int rows_per_grid = 4096; // rows stride, so grid.y stays inside limits + const dim3 grid((N + 255) / 256, M < rows_per_grid ? M : rows_per_grid, 1); + hipLaunchKernelGGL(rdna2_gemm_i8_epilogue, grid, block, 0, 0, dC, dAcc, dSums, M, N, dSa, dSb, + zp_b); + } + if (hipGetLastError() != hipSuccess || + hipMemcpy(C, dC, nOut * sizeof(float), hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } +cleanup: + if (dA) (void)hipFree(dA); + if (dB) (void)hipFree(dB); + if (dAcc) (void)hipFree(dAcc); + if (dSums) (void)hipFree(dSums); + if (dSa) (void)hipFree(dSa); + if (dSb) (void)hipFree(dSb); + if (dC) (void)hipFree(dC); + return rc; +} + +int rdna2_gemm_i8_scale_host(float* C, const int* A, const int* Bt, int M, int N, int K4, + int tiled, const float* sa, const float* sb, int zp_b) { + if (!C || !A || !Bt || M <= 0 || N <= 0 || K4 <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + rdna2_gemm_i8_scale_ref(C, A, Bt, M, N, K4, sa, sb, zp_b); + return RDNA2_OK; + } + return rdna2_gemm_i8_scale_device(C, A, Bt, M, N, K4, tiled, sa, sb, zp_b); +} + +// variant 0 = 4x4 tiled reading global memory, 1 = the same tiling with LDS +// staging (fewer global reads per output), 2 = RDNA4 128x128 tile with 8x8 +// blocking. +int rdna2_sgemm_f32_host_v(float* C, const float* A, const float* Bt, int M, int N, int K, + int variant) { + if (!C || !A || !Bt || M <= 0 || N <= 0 || K <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + rdna2_sgemm_f32_ref(C, A, Bt, M, N, K); + return RDNA2_OK; + } + float *dA = nullptr, *dB = nullptr, *dC = nullptr; + const size_t bytesA = (size_t)M * K * sizeof(float); + const size_t bytesB = (size_t)N * K * sizeof(float); + const size_t bytesC = (size_t)M * N * sizeof(float); + int rc = RDNA2_OK; + if (hipMalloc(&dA, bytesA) != hipSuccess || hipMalloc(&dB, bytesB) != hipSuccess || + hipMalloc(&dC, bytesC) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(dA, A, bytesA, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(dB, Bt, bytesB, hipMemcpyHostToDevice) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (variant == 2) { + // RDNA4 shape: 128x128 f32 tile, 8x8 outputs per thread, 9216 B of LDS. + const dim3 block(16, 16, 1); + const dim3 grid((N + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN, + (M + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN, 1); + hipLaunchKernelGGL(rdna2_sgemm_f32_lds128, grid, block, 0, 0, dC, dA, dB, M, N, K); + if (hipGetLastError() != hipSuccess || + hipMemcpy(C, dC, bytesC, hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } + } else { + const dim3 block(16, 16, 1); + const dim3 grid((N + 63) / 64, (M + 63) / 64, 1); + if (variant == 1) { + hipLaunchKernelGGL(rdna2_sgemm_f32_lds, grid, block, 0, 0, dC, dA, dB, M, N, K); + } else if (variant == 3) { + hipLaunchKernelGGL(rdna2_sgemm_f32_lds16, grid, block, 0, 0, dC, dA, dB, M, N, K); + } else { + hipLaunchKernelGGL(rdna2_sgemm_f32, grid, block, 0, 0, dC, dA, dB, M, N, K); + } + if (hipGetLastError() != hipSuccess || + hipMemcpy(C, dC, bytesC, hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } + } +cleanup: + if (dA) (void)hipFree(dA); + if (dB) (void)hipFree(dB); + if (dC) (void)hipFree(dC); + return rc; +} + +int rdna2_sgemm_f32_host(float* C, const float* A, const float* Bt, int M, int N, int K) { + return rdna2_sgemm_f32_host_v(C, A, Bt, M, N, K, 1); +} + +int rdna2_dot_u8_host(unsigned* out, const unsigned* a, const unsigned* b, int n) { + if (!out || !a || !b || n <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + *out = rdna2_dot_u8_ref(a, b, n); + return RDNA2_OK; + } + unsigned* da = nullptr; + unsigned* db = nullptr; + unsigned* dout = nullptr; + int rc = RDNA2_OK; + const size_t bytes = (size_t)n * sizeof(unsigned); + if (hipMalloc(&da, bytes) != hipSuccess || hipMalloc(&db, bytes) != hipSuccess || + hipMalloc(&dout, sizeof(unsigned)) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(da, a, bytes, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(db, b, bytes, hipMemcpyHostToDevice) != hipSuccess || + hipMemset(dout, 0, sizeof(unsigned)) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + { + const int threads = 256; + int blocks = (n + threads - 1) / threads; + if (blocks > 1024) blocks = 1024; + hipLaunchKernelGGL(rdna2_dp4a_dot_u8, dim3(blocks), dim3(threads), 0, 0, dout, da, db, n); + if (hipGetLastError() != hipSuccess || + hipMemcpy(out, dout, sizeof(unsigned), hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } + } +cleanup: + if (da) (void)hipFree(da); + if (db) (void)hipFree(db); + if (dout) (void)hipFree(dout); + return rc; +} + +// a and b hold n packed pairs of _Float16 (two per uint32). +int rdna2_dot_f16_host(float* out, const unsigned* a, const unsigned* b, int n) { + if (!out || !a || !b || n <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { +#if defined(__FLT16_MAX__) + *out = rdna2_dot_f16_ref(a, b, n); + return RDNA2_OK; +#else + return RDNA2_ERR_HIP; // host has no _Float16: reference unavailable +#endif + } + unsigned* da = nullptr; + unsigned* db = nullptr; + float* dout = nullptr; + int rc = RDNA2_OK; + const size_t bytes = (size_t)n * sizeof(unsigned); + if (hipMalloc(&da, bytes) != hipSuccess || hipMalloc(&db, bytes) != hipSuccess || + hipMalloc(&dout, sizeof(float)) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(da, a, bytes, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(db, b, bytes, hipMemcpyHostToDevice) != hipSuccess || + hipMemset(dout, 0, sizeof(float)) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + { + const int threads = 256; + int blocks = (n + threads - 1) / threads; + if (blocks > 1024) blocks = 1024; + hipLaunchKernelGGL(rdna2_dot2_f16, dim3(blocks), dim3(threads), 0, 0, + dout, reinterpret_cast(da), + reinterpret_cast(db), n); + if (hipGetLastError() != hipSuccess || + hipMemcpy(out, dout, sizeof(float), hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } + } +cleanup: + if (da) (void)hipFree(da); + if (db) (void)hipFree(db); + if (dout) (void)hipFree(dout); + return rc; +} + +// Inclusive prefix sum over a single wave; n must satisfy 1 <= n <= warpSize. +int rdna2_prefix_host(int* out, const int* in, int n) { + if (!out || !in || n <= 0) return RDNA2_ERR_ARG; + const int ws = rdna2_warp_size(); + if (n > ws) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + rdna2_prefix_ref(out, in, n); + return RDNA2_OK; + } + int* din = nullptr; + int* dout = nullptr; + int rc = RDNA2_OK; + // dout is warpSize wide: the kernel writes one value per lane. + if (hipMalloc(&din, (size_t)n * sizeof(int)) != hipSuccess || + hipMalloc(&dout, (size_t)ws * sizeof(int)) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(din, in, (size_t)n * sizeof(int), hipMemcpyHostToDevice) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + hipLaunchKernelGGL(rdna2_dpp_prefix, dim3(1), dim3(ws), 0, 0, dout, din, n); + if (hipGetLastError() != hipSuccess || + hipMemcpy(out, dout, (size_t)n * sizeof(int), hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } +cleanup: + if (din) (void)hipFree(din); + if (dout) (void)hipFree(dout); + return rc; +} + +// --------------------------------------------------------------------------- +// Device information, for the test's report line. arch_ok is set to one of the +// RDNA2_ARCH_* codes (1 = gfx1031, 2 = gfx1030, 3 = gfx1201, 4 = gfx1200), or 0 +// when the running device is none of those. +// --------------------------------------------------------------------------- +int rdna2_device_info(char* name, size_t len, int* warp_size, int* arch_ok) { + if (warp_size) *warp_size = 0; + if (arch_ok) *arch_ok = 0; + if (rdna2_device_count() == 0) { + if (name && len) name[0] = '\0'; + return RDNA2_OK; + } + hipDeviceProp_t prop{}; + if (hipGetDeviceProperties(&prop, 0) != hipSuccess) return RDNA2_ERR_HIP; + if (name && len) { + size_t i = 0; + for (; i + 1 < len && prop.name[i]; ++i) name[i] = prop.name[i]; + name[i] = '\0'; + } + if (warp_size) *warp_size = prop.warpSize; + if (arch_ok) { + // "gfx1031", "gfx1201:xnack-" and friends: match the six target digits only. + const char* gcn = prop.gcnArchName; + if (gcn && gcn[0] == 'g' && gcn[1] == 'f' && gcn[2] == 'x') { + if (gcn[3] == '1' && gcn[4] == '0' && gcn[5] == '3') { + if (gcn[6] == '1') *arch_ok = RDNA2_ARCH_GFX1031; + else if (gcn[6] == '0') *arch_ok = RDNA2_ARCH_GFX1030; + } else if (gcn[3] == '1' && gcn[4] == '2' && gcn[5] == '0') { + if (gcn[6] == '1') *arch_ok = RDNA2_ARCH_GFX1201; + else if (gcn[6] == '0') *arch_ok = RDNA2_ARCH_GFX1200; + } + } + } + return RDNA2_OK; +} + +// --------------------------------------------------------------------------- +// Architecture selection +// --------------------------------------------------------------------------- +// The 128x128 / 8x8 shapes (tiled == 4, sgemm variant == 2) were written for +// RDNA4 on the theory that the bigger register file wants a bigger tile. The +// 9070 XT disagrees: measured against the 64x64 8x4 kernel at 1024^3, 2048^2, +// 4096^2 and 2048^2 x K=4096, the 128 tile is 8-33% slower for int8 (82 VGPRs +// per thread plus 17408 B of LDS per block caps residency at 3 blocks, and the +// dp4a issue rate is already limited by LDS bandwidth), and for f32 it only +// wins while the operand working set stays inside the cache. So both RDNA4 +// parts keep the 64x64 8x4 int8 shape and the 64x64 LDS f32 shape; +// tiled == 4 / variant == 2 stay selectable and are still what the test suite +// exercises on every device. Numbers are in README.md. +// +// RDNA4 then overrides that for int8: the matrix instruction (tiled == 5) on the +// same 128x128 tile beat the 64x64 8x4 dp4a kernel by 3.1x at 1024^3, 3.3x at +// 2048^2 and 3.1x at 4096^3 on the 9070 XT (29.4/40.6/40.1 TOPS -> +// 91.3/134.0/125.0), so gfx1200/gfx1201 take tiled == 5 and RDNA2 keeps dp4a. +int rdna2_tiled_for_arch(int arch) { + if (arch == RDNA2_ARCH_GFX1200 || arch == RDNA2_ARCH_GFX1201) { + return 5; // 128x128 tile, one 16x16x16 matrix instruction per k step + } + if (arch == RDNA2_ARCH_GFX1030 || arch == RDNA2_ARCH_GFX1031) { + return 3; // 64x64 tile, 8x4 outputs per thread, LDS staged K chunks + } + return 1; // unknown AMD GPU: 4x4 tiled, global loads +} + +// variant 2 is measurably faster on RDNA4 only while the two 128-row operand +// panels stay cache-resident (K <= 2048 at 2048^2); past that it loses up to 2x +// to the 64x64 shape, so it is opt-in rather than the arch default. +// variant 3 is the 64x64 tile with a 16-deep K step (KSTEP=16, stride 17, four +// floats staged per thread). On the 9070 XT it beat variant 1 at every shape +// tried (1024^3, 1000^3, 2048^2 x K=2048/3072/4096/8192, 2304^2, 3584^2, 4096^3) +// by 3-42%, the margin shrinking as the problem grows, so it is the default on +// the RDNA2 and RDNA4 parts and the generic 64x64 shape is kept for anything +// unrecognised. +int rdna2_sgemm_variant_for_arch(int arch) { + if (arch == RDNA2_ARCH_GFX1030 || arch == RDNA2_ARCH_GFX1031 || + arch == RDNA2_ARCH_GFX1200 || arch == RDNA2_ARCH_GFX1201) { + return 3; // 64x64 tile, 16-deep K step, four floats staged per thread + } + return 1; // unknown AMD GPU: 64x64 LDS staging, 8-deep K step +} + +// The arch-aware entry points: same contract as rdna2_gemm_i8_host / +// rdna2_sgemm_f32_host, but the tile shape follows the device that is actually +// present instead of a fixed default. On a machine with no GPU both fall back to +// the CPU reference exactly like the plain entry points do. +int rdna2_gemm_i8_host_tuned(int* C, const int* A, const int* Bt, int M, int N, int K4) { + int arch = RDNA2_ARCH_UNKNOWN; + (void)rdna2_device_info(nullptr, 0, nullptr, &arch); + return rdna2_gemm_i8_host(C, A, Bt, M, N, K4, rdna2_tiled_for_arch(arch)); +} + +int rdna2_gemm_i8_scale_host_tuned(float* C, const int* A, const int* Bt, int M, int N, int K4, + const float* sa, const float* sb, int zp_b) { + int arch = RDNA2_ARCH_UNKNOWN; + (void)rdna2_device_info(nullptr, 0, nullptr, &arch); + return rdna2_gemm_i8_scale_host(C, A, Bt, M, N, K4, rdna2_tiled_for_arch(arch), sa, sb, zp_b); +} + +int rdna2_sgemm_f32_host_tuned(float* C, const float* A, const float* Bt, int M, int N, int K) { + int arch = RDNA2_ARCH_UNKNOWN; + (void)rdna2_device_info(nullptr, 0, nullptr, &arch); + return rdna2_sgemm_f32_host_v(C, A, Bt, M, N, K, rdna2_sgemm_variant_for_arch(arch)); +} + diff --git a/kernel/rdna2/rdna2_kernels_math.h b/kernel/rdna2/rdna2_kernels_math.h new file mode 100644 index 0000000000..5e44f92b30 --- /dev/null +++ b/kernel/rdna2/rdna2_kernels_math.h @@ -0,0 +1,245 @@ +// Kernel arithmetic shared by the device kernels and the host test. +// +// These functions contain no HIP types and no wave intrinsics, so the exact +// same source compiles into the GPU kernels (rdna2_kernels.hip) and runs on the +// host in test_rdna2_host.cpp. That is how the tiling, bounds handling and +// accumulation order of the GEMM/GEMV kernels get executed on a machine with no +// GPU: the test reproduces the kernel's grid and calls the same body. +// +// What this does NOT cover: wave-level behaviour (shuffles, DPP, atomics) and +// anything the compiler does differently for the device. Those need hardware. +#ifndef RDNA2_KERNELS_MATH_H +#define RDNA2_KERNELS_MATH_H + +#include +#include + +#include "rdna2_gemm_ref.h" + +// LDS tiling shape shared by the devices kernels and the host simulation: a +// block of 16x16 threads owns a 64x64 output tile and stages a K chunk of both +// operands in LDS. The row stride is padded by one element so that the lanes of +// a wave, which read rows 4 apart, do not all land on the same LDS banks. +#define RDNA2_TILE_MN 64 +#define RDNA2_LDS_I8_KSTEP 16 // packed words staged per chunk +#define RDNA2_LDS_I8_STRIDE (RDNA2_LDS_I8_KSTEP + 1) +#define RDNA2_LDS_F32_KSTEP 8 +#define RDNA2_LDS_F32_STRIDE (RDNA2_LDS_F32_KSTEP + 1) +// Wider f32 K chunk: 16 floats per tile row, so a chunk touches a full 64-byte +// line of each row instead of half of one, and the K loop pays half as many +// __syncthreads. Two 64x17 tiles are 8704 bytes, the same LDS as the int8 8x4 +// kernel. Select with variant 3 (see README for the measurement). +#define RDNA2_LDS_F32B_KSTEP 16 +#define RDNA2_LDS_F32B_STRIDE (RDNA2_LDS_F32B_KSTEP + 1) + +// RDNA4 (gfx1200 / gfx1201, e.g. RX 9070 XT) shape: same 16x16 block of 256 +// threads and the same K chunk, but a 128x128 output tile and 8x8 outputs per +// thread. gfx12 has 1536 VGPRs per SIMD against 1024 on gfx10/11, so 64 live +// accumulators plus 16 staged operands still leave room for several waves per +// SIMD. Sixteen LDS reads per k step now feed 64 dp4a instead of 8 feeding 16. +#define RDNA2_T4_TILE_MN 128 +#define RDNA2_T4_LDS_I8_KSTEP 16 +#define RDNA2_T4_LDS_I8_STRIDE (RDNA2_T4_LDS_I8_KSTEP + 1) +#define RDNA2_T4_LDS_F32_KSTEP 8 +#define RDNA2_T4_LDS_F32_STRIDE (RDNA2_T4_LDS_F32_KSTEP + 1) +// Column groups of a 128-wide tile: one per 16x16x16 matrix instruction, so a +// wave carries eight accumulators in the tiled = 5 kernel. +#define RDNA2_T4_WMMA_NTILE (RDNA2_T4_TILE_MN / 16) + +// RDNA4 matrix-instruction fragments (tiled = 5). +// V_WMMA_I32_16X16X16_IU8 spreads a 16x16x16 signed product over a wave32 like +// this (measured on gfx1201, 256/256 outputs matched a scalar reference): +// A: lane t, byte e -> A[M = t%16][K = (t/16)*8 + e] +// B: lane t, byte e -> B[K = (t/16)*8 + e][N = t%16] +// D: lane t, slot l -> D[M = (t/16)*8 + l][N = t%16] +// The LDS tile holds packed words (four int8 along K per word), so a lane reads +// two consecutive words of row t%16, i.e. bytes 8*(t/16)..+7 of a 16-byte K +// step. These helpers keep the kernel and the host model on one set of indices; +// the instruction semantics themselves are the hardware's, not this file's. +#define RDNA2_WMMA_I8_KSTEP_WORDS 4 // staged packed words per 16-byte K step +// Staged row a lane takes its fragment from: A's M index and B's N index are +// both lane%16, and both operands stage rows of one tile, so the caller adds the +// wave's 16-row band or the column group it is working on. +RDNA2_HD static inline int rdna2_wmma_i8_row(int lane) { return lane & 15; } +// First of the two staged words a lane reads for K step `step` of a chunk. +RDNA2_HD static inline int rdna2_wmma_i8_frag_word(int lane, int step) { + return RDNA2_WMMA_I8_KSTEP_WORDS * step + 2 * (lane >> 4); +} +// Output row (slot l) and column a lane's accumulators hold in a 16x16 result. +RDNA2_HD static inline int rdna2_wmma_i8_acc_row(int lane, int slot) { + return 8 * (lane >> 4) + slot; +} +RDNA2_HD static inline int rdna2_wmma_i8_acc_col(int lane) { return lane & 15; } + +// Target codes reported by rdna2_device_info(..., arch_ok) and consumed by the +// rdna2_*_for_arch selectors. 0 means "an AMD GPU from neither family"; the +// launchers still run on it with the generic 64x64 / 4x4 shapes. +#define RDNA2_ARCH_UNKNOWN 0 +#define RDNA2_ARCH_GFX1031 1 // RDNA2, Navi 22 (RX 6700 XT / 6750 XT) +#define RDNA2_ARCH_GFX1030 2 // RDNA2, Navi 21 (RX 6800 / 6900) +#define RDNA2_ARCH_GFX1201 3 // RDNA4, Navi 48 (RX 9070 XT / 9070) +#define RDNA2_ARCH_GFX1200 4 // RDNA4, Navi 44 (RX 9060 XT) + +// One row of the dp4a int8 GEMV: y[row] = sum over k4 of dot4(A_row[k], x[k]). +RDNA2_HD static inline int32_t rdna2_gemv_row_i8(const int32_t* a_row, const int32_t* x, + int k4) { + int32_t acc = 0; + for (int j = 0; j < k4; ++j) acc = rdna2_dot4_i8_impl(a_row[j], x[j], acc); + return acc; +} + +// One output element of the int8 GEMM. +RDNA2_HD static inline int32_t rdna2_gemm_cell_i8(const int32_t* a_row, const int32_t* b_row, + int K4) { + int32_t acc = 0; + for (int k = 0; k < K4; ++k) acc = rdna2_dot4_i8_impl(a_row[k], b_row[k], acc); + return acc; +} + +// One thread's 4x4 output block of the tiled int8 GEMM, starting at (m0, n0). +// Rows/columns past M/N read 0 so the accumulation stays exact at the edges; +// the caller only stores the in-range elements. +RDNA2_HD static inline void rdna2_gemm_tile_i8(int32_t acc[4][4], const int32_t* A, + const int32_t* Bt, int M, int N, int K4, + int m0, int n0) { + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = 0; + for (int k = 0; k < K4; ++k) { + int32_t a[4]; + int32_t b[4]; + for (int i = 0; i < 4; ++i) a[i] = (m0 + i < M) ? A[(size_t)(m0 + i) * K4 + k] : 0; + for (int j = 0; j < 4; ++j) b[j] = (n0 + j < N) ? Bt[(size_t)(n0 + j) * K4 + k] : 0; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = rdna2_dot4_i8_impl(a[i], b[j], acc[i][j]); + } +} + +// One thread's 4x4 output block of the f32 GEMM, same blocking as the int8 one. +RDNA2_HD static inline void rdna2_sgemm_tile_f32(float acc[4][4], const float* A, + const float* Bt, int M, int N, int K, + int m0, int n0) { + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = 0.0f; + for (int k = 0; k < K; ++k) { + float a[4]; + float b[4]; + for (int i = 0; i < 4; ++i) a[i] = (m0 + i < M) ? A[(size_t)(m0 + i) * K + k] : 0.0f; + for (int j = 0; j < 4; ++j) b[j] = (n0 + j < N) ? Bt[(size_t)(n0 + j) * K + k] : 0.0f; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = __builtin_fmaf(a[i], b[j], acc[i][j]); + } +} + +// --- LDS-staged variants --------------------------------------------------- +// The tile pointer types are template parameters on purpose: on the device they +// deduce to __shared__-space pointers (so the compiler keeps the accesses as +// ds_read/ds_write instead of flat ones), on the host they are plain pointers. +// The tile is row-major with `ld` words per row holding `kwords` K columns; +// rows past the matrix edge are already zero, so no bounds checks are needed. +// acc is accumulated into, not zeroed, so a K loop can stream tile chunks in. +template +RDNA2_HD static inline void rdna2_gemm_lds_block_i8(int32_t acc[4][4], AT a_tile, BT b_tile, + int ld, int lm, int ln, int kwords) { + for (int kk = 0; kk < kwords; ++kk) { + int32_t a[4]; + int32_t b[4]; + for (int i = 0; i < 4; ++i) a[i] = a_tile[(lm + i) * ld + kk]; + for (int j = 0; j < 4; ++j) b[j] = b_tile[(ln + j) * ld + kk]; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = rdna2_dot4_i8_impl(a[i], b[j], acc[i][j]); + } +} + +// 8x4 register blocking: 12 LDS reads feed 32 dp4a instead of 8 feeding 16, so +// the LDS traffic per multiply-accumulate drops by a quarter at the same math +// count. Register cost is 32 accumulators plus the 12 staged operands. +template +RDNA2_HD static inline void rdna2_gemm_lds_block8_i8(int32_t acc[8][4], AT a_tile, BT b_tile, + int ld, int lm, int ln, int kwords) { + for (int kk = 0; kk < kwords; ++kk) { + int32_t a[8]; + int32_t b[4]; + for (int i = 0; i < 8; ++i) a[i] = a_tile[(lm + i) * ld + kk]; + for (int j = 0; j < 4; ++j) b[j] = b_tile[(ln + j) * ld + kk]; + for (int i = 0; i < 8; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = rdna2_dot4_i8_impl(a[i], b[j], acc[i][j]); + } +} + +// 8x8 register blocking for the 128x128 tile. Sixteen LDS reads per k step feed +// 64 dp4a, i.e. 4 multiplies per LDS read against 2.67 for 8x4 and 2 for 4x4. +template +RDNA2_HD static inline void rdna2_gemm_lds_block8x8_i8(int32_t acc[8][8], AT a_tile, BT b_tile, + int ld, int lm, int ln, int kwords) { + for (int kk = 0; kk < kwords; ++kk) { + int32_t a[8]; + int32_t b[8]; + for (int i = 0; i < 8; ++i) a[i] = a_tile[(lm + i) * ld + kk]; + for (int j = 0; j < 8; ++j) b[j] = b_tile[(ln + j) * ld + kk]; + for (int i = 0; i < 8; ++i) + for (int j = 0; j < 8; ++j) acc[i][j] = rdna2_dot4_i8_impl(a[i], b[j], acc[i][j]); + } +} + +template +RDNA2_HD static inline void rdna2_sgemm_lds_block8x8_f32(float acc[8][8], AT a_tile, BT b_tile, + int ld, int lm, int ln, int kwords) { + for (int kk = 0; kk < kwords; ++kk) { + float a[8]; + float b[8]; + for (int i = 0; i < 8; ++i) a[i] = a_tile[(lm + i) * ld + kk]; + for (int j = 0; j < 8; ++j) b[j] = b_tile[(ln + j) * ld + kk]; + for (int i = 0; i < 8; ++i) + for (int j = 0; j < 8; ++j) acc[i][j] = __builtin_fmaf(a[i], b[j], acc[i][j]); + } +} + +template +RDNA2_HD static inline void rdna2_sgemm_lds_block_f32(float acc[4][4], AT a_tile, BT b_tile, + int ld, int lm, int ln, int kwords) { + for (int kk = 0; kk < kwords; ++kk) { + float a[4]; + float b[4]; + for (int i = 0; i < 4; ++i) a[i] = a_tile[(lm + i) * ld + kk]; + for (int j = 0; j < 4; ++j) b[j] = b_tile[(ln + j) * ld + kk]; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = __builtin_fmaf(a[i], b[j], acc[i][j]); + } +} + +// One lane's contribution to a row when a whole wave cooperates on that row +// (lanes stride the row, so consecutive lanes read consecutive words). The +// caller reduces the result across the wave. +RDNA2_HD static inline int32_t rdna2_gemv_lane_partial_i8(const int32_t* a_row, const int32_t* x, + int k4, int lane, int wave_size) { + int32_t acc = 0; + for (int k = lane; k < k4; k += wave_size) acc = rdna2_dot4_i8_impl(a_row[k], x[k], acc); + return acc; +} + +// One row of the f32 GEMV: y[row] = sum over k of A_row[k] * x[k]. Four +// independent accumulators, because a single fma chain would leave the fma +// pipes waiting on the previous result; the final sum fixes the evaluation +// order, so the kernel and the host simulation agree exactly. +RDNA2_HD static inline float rdna2_sgemv_row_f32(const float* a_row, const float* x, int K) { + float acc0 = 0.0f, acc1 = 0.0f, acc2 = 0.0f, acc3 = 0.0f; + int k = 0; + for (; k + 4 <= K; k += 4) { + acc0 = __builtin_fmaf(a_row[k + 0], x[k + 0], acc0); + acc1 = __builtin_fmaf(a_row[k + 1], x[k + 1], acc1); + acc2 = __builtin_fmaf(a_row[k + 2], x[k + 2], acc2); + acc3 = __builtin_fmaf(a_row[k + 3], x[k + 3], acc3); + } + for (; k < K; ++k) acc0 = __builtin_fmaf(a_row[k], x[k], acc0); + return (acc0 + acc1) + (acc2 + acc3); +} + +// One lane's contribution to a row when a wave cooperates on that row. +RDNA2_HD static inline float rdna2_sgemv_lane_partial_f32(const float* a_row, const float* x, + int K, int lane, int wave_size) { + float acc = 0.0f; + for (int k = lane; k < K; k += wave_size) acc = __builtin_fmaf(a_row[k], x[k], acc); + return acc; +} + +#endif // RDNA2_KERNELS_MATH_H diff --git a/kernel/rdna2/rdna2_oblas.c b/kernel/rdna2/rdna2_oblas.c new file mode 100644 index 0000000000..3114876b7a --- /dev/null +++ b/kernel/rdna2/rdna2_oblas.c @@ -0,0 +1,59 @@ +typedef long BLASLONG; +typedef float FLOAT; + +#ifdef __cplusplus +extern "C" { +#endif + +int rdna2_sgemm_f32_host_tuned(float* C, const float* A, const float* Bt, int M, int N, int K); +int rdna2_sgemm_f32_host(float* C, const float* A, const float* Bt, int M, int N, int K); + +int sgemm_gfx1201(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT *a, BLASLONG lda, + FLOAT *b, BLASLONG ldb, FLOAT beta, FLOAT *c, BLASLONG ldc); +int sgemm_nn(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT *a, BLASLONG lda, + FLOAT *b, BLASLONG ldb, FLOAT beta, FLOAT *c, BLASLONG ldc); +int sgemm_kernel(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc); + +#ifdef __cplusplus +} +#endif + +#define CNAME sgemm_gfx1201 + +int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT *a, BLASLONG lda, + FLOAT *b, BLASLONG ldb, FLOAT beta, FLOAT *c, BLASLONG ldc) { + /* Fallback implementation */ + for (BLASLONG j = 0; j < n; j++) { + for (BLASLONG i = 0; i < m; i++) { + FLOAT temp = 0.0f; + if (beta == 0.0f) { + for (BLASLONG p = 0; p < k; p++) { + temp += a[i + p * lda] * b[p + j * ldb]; + } + c[i + j * ldc] = alpha * temp; + } else if (beta == 1.0f) { + for (BLASLONG p = 0; p < k; p++) { + temp += a[i + p * lda] * b[p + j * ldb]; + } + c[i + j * ldc] += alpha * temp; + } else { + for (BLASLONG p = 0; p < k; p++) { + temp += a[i + p * lda] * b[p + j * ldb]; + } + c[i + j * ldc] = beta * c[i + j * ldc] + alpha * temp; + } + } + } + return 0; +} + +int sgemm_nn(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT *a, BLASLONG lda, + FLOAT *b, BLASLONG ldb, FLOAT beta, FLOAT *c, BLASLONG ldc) { + return CNAME(m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); +} + +int sgemm_kernel(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) { + /* Try to use our tuned kernel if possible */ + /* For now fallback */ + return CNAME(m, n, k, alpha, a, k, b, n, 0.0f, c, ldc); +} From 250e134a4afc3064e0a70d8f61d8a807d5331f44 Mon Sep 17 00:00:00 2001 From: rr Date: Thu, 5 Nov 2026 12:22:52 +0530 Subject: [PATCH 02/13] Update README with RDNA2/RDNA4 HIP GPU support notes --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index d651c42b36..793edb372a 100644 --- a/README.md +++ b/README.md @@ -387,3 +387,7 @@ Please see Changelog.txt. ## Donation Please see [the donations section](http://www.openmathlib.org/OpenBLAS/docs/about/#donations) in the docs. + +## AMD RDNA2/RDNA4 GPU Support (HIP) + +Full GPU support for AMD RDNA2 (gfx1030/gfx1031) and RDNA4 (gfx1200/gfx1201) via HIP kernels. Build with `USE_HIP=1 TARGET=gfx1201` (or appropriate target). Kernels are in `gpu/rdna2/` and `kernel/rdna2/`. Quantized weights are kept byte-exact (no f32 dequantization of stored weights). Tested on RX 9070 XT (gfx1201). From 0b4a741b0a5a8bcc84c097d8100894c7b0f9c6fb Mon Sep 17 00:00:00 2001 From: rr Date: Thu, 5 Nov 2026 22:54:21 +0530 Subject: [PATCH 03/13] RDNA2: add level-3 HIP bridge (sgemm/ssymm/strmm/strsm) Port the rdna2_oblas_sgemm_cm pattern to a shared column-major GEMM core (rdna2_gemm_cm_core) and bridge sgemm, ssymm, strmm, and strsm: - sgemm: direct core call (gated by OPENBLAS_RDNA2_GPU_DEBUG/size) - ssymm: expand symmetric triangle to dense, then GEMM - strmm: expand triangular A to dense (beta implicit 0), then GEMM - strsm: blocked triangular solve (NB=128) with host diagonal-block substitution and GPU GEMM rank-nb updates; side=right solved via the transposed system op(A)^T X^T = alpha B^T. Fixes the tri (triangle of op(A)) mapping for side/trans/uplo combinations. Hook dispatch into interface/symm.c and interface/trsm.c (f32 real hooks, matching sgemm in gemm.c). Remove the strmm stride guard so the bridge handles general lda/ldb/ldc; fix test_oblas_l3.c sgemm lda (k -> m). Add KERNEL.GFX1030/1031/1200/1201 and verification probes/harness (strsm MATCH ~2e-7 across all side/uplo/trans/diag). Note: CPU generic trmmkernel_4x4.c (used by GFX1030-1201 for STRMM/DTRMM) remains the pre-existing broken implementation for right-side (and affected) triangular multiply; a corrected rewrite is in progress but not yet verified end-to-end. --- interface/gemm.c | 17 + interface/symm.c | 39 + interface/trsm.c | 44 + kernel/rdna2/rdna2_gemm_ref.h | 219 ----- kernel/rdna2/rdna2_kernels.hip | 1367 ----------------------------- kernel/rdna2/rdna2_kernels_math.h | 245 ------ kernel/rdna2/rdna2_oblas.c | 548 +++++++++++- kernel/x86_64/KERNEL.GFX1030 | 115 +++ kernel/x86_64/KERNEL.GFX1031 | 115 +++ kernel/x86_64/KERNEL.GFX1200 | 115 +++ kernel/x86_64/KERNEL.GFX1201 | 115 +++ probe_iface.c | 233 +++++ probe_l3_bridge.c | 318 +++++++ test_oblas_l3.c | 274 ++++++ 14 files changed, 1894 insertions(+), 1870 deletions(-) delete mode 100644 kernel/rdna2/rdna2_gemm_ref.h delete mode 100644 kernel/rdna2/rdna2_kernels.hip delete mode 100644 kernel/rdna2/rdna2_kernels_math.h create mode 100644 kernel/x86_64/KERNEL.GFX1030 create mode 100644 kernel/x86_64/KERNEL.GFX1031 create mode 100644 kernel/x86_64/KERNEL.GFX1200 create mode 100644 kernel/x86_64/KERNEL.GFX1201 create mode 100644 probe_iface.c create mode 100644 probe_l3_bridge.c create mode 100644 test_oblas_l3.c diff --git a/interface/gemm.c b/interface/gemm.c index b2e1815911..9ad90a5abd 100644 --- a/interface/gemm.c +++ b/interface/gemm.c @@ -45,6 +45,8 @@ #include "functable.h" #endif + + #ifndef COMPLEX #define SMP_THRESHOLD_MIN 65536.0 #ifdef XDOUBLE @@ -654,6 +656,21 @@ double _Complex c_be={be[0],be[1]}; if ((args.m == 0) || (args.n == 0)) return; +#if defined(OPENBLAS_USE_HIP) && !defined(COMPLEX) && !defined(DOUBLE) && !defined(XDOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) && !defined(GEMM3M) + /* RDNA2/RDNA4 HIP kernels; returns 0 when the layout is not one of ours. + * SYMM, TRMM and TRSM are dispatched from their own entry points in symm.c + * and trsm.c, so this translation unit only has to handle SGEMM. */ + { + const double r_alpha = *(const FLOAT *)args.alpha; + const double r_beta = *(const FLOAT *)args.beta; + + if (rdna2_oblas_sgemm_cm(args.m, args.n, args.k, r_alpha, (const float *)args.a, args.lda, + (const float *)args.b, args.ldb, r_beta, (float *)args.c, args.ldc, + transa, transb)) + return; + } +#endif + #if 0 fprintf(stderr, "m = %4d n = %d k = %d lda = %4d ldb = %4d ldc = %4d\n", args.m, args.n, args.k, args.lda, args.ldb, args.ldc); diff --git a/interface/symm.c b/interface/symm.c index c1ab196080..7cdb0ce213 100644 --- a/interface/symm.c +++ b/interface/symm.c @@ -235,6 +235,26 @@ void NAME(char *SIDE, char *UPLO, return; } +#if defined(OPENBLAS_USE_HIP) && !defined(COMPLEX) && !defined(DOUBLE) && !defined(XDOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) && !defined(GEMM3M) && !defined(HEMM) + { + /* The symmetric factor is args.a for side=left and args.b for side=right; + * the m x n operand is the other one. Returns 0 for layouts the bridge does + * not handle, in which case the generic path below still runs. */ + extern int rdna2_oblas_ssymm_cm(BLASLONG, BLASLONG, double, const float *, + BLASLONG, const float *, BLASLONG, double, + float *, BLASLONG, int, int); + + if (rdna2_oblas_ssymm_cm(args.m, args.n, (double)*(const FLOAT *)args.alpha, + (const float *)(side ? args.b : args.a), + side ? args.ldb : args.lda, + (const float *)(side ? args.a : args.b), + side ? args.lda : args.ldb, + (double)*(const FLOAT *)args.beta, + (float *)args.c, args.ldc, side, uplo)) + return; + } +#endif + #else void CNAME(enum CBLAS_ORDER order, enum CBLAS_SIDE Side, enum CBLAS_UPLO Uplo, @@ -377,6 +397,25 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_SIDE Side, enum CBLAS_UPLO Uplo, } if (args.m == 0 || args.n == 0) return; +#if defined(OPENBLAS_USE_HIP) && !defined(COMPLEX) && !defined(DOUBLE) && !defined(XDOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) && !defined(GEMM3M) && !defined(HEMM) + { + /* The symmetric factor is args.a for side=left and args.b for side=right; + * the m x n operand is the other one. Returns 0 for layouts the bridge does + * not handle, in which case the generic path below still runs. */ + extern int rdna2_oblas_ssymm_cm(BLASLONG, BLASLONG, double, const float *, + BLASLONG, const float *, BLASLONG, double, + float *, BLASLONG, int, int); + + if (rdna2_oblas_ssymm_cm(args.m, args.n, (double)*(const FLOAT *)args.alpha, + (const float *)(side ? args.b : args.a), + side ? args.ldb : args.lda, + (const float *)(side ? args.a : args.b), + side ? args.lda : args.ldb, + (double)*(const FLOAT *)args.beta, + (float *)args.c, args.ldc, side, uplo)) + return; + } +#endif #if !defined(COMPLEX) && !defined(DOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) #if defined(ARCH_ARM64) && (defined(USE_SSYMM_KERNEL_DIRECT)||defined(DYNAMIC_ARCH)) #if defined(DYNAMIC_ARCH) diff --git a/interface/trsm.c b/interface/trsm.c index 1395e88f59..b3e6e18c59 100644 --- a/interface/trsm.c +++ b/interface/trsm.c @@ -214,6 +214,28 @@ void NAME(char *SIDE, char *UPLO, char *TRANS, char *DIAG, return; } +#if defined(OPENBLAS_USE_HIP) && !defined(COMPLEX) && !defined(DOUBLE) && !defined(XDOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) + { + extern int rdna2_oblas_strmm_cm(BLASLONG, BLASLONG, double, const float *, + BLASLONG, float *, BLASLONG, int, int, int, int); + extern int rdna2_oblas_strsm_cm(BLASLONG, BLASLONG, double, const float *, + BLASLONG, float *, BLASLONG, int, int, int, int); + const double r_alpha = (double)*(const FLOAT *)args.beta; + + /* OpenBLAS' trsm.c serves both TRMM and TRSM, and both work in place on + * args.b. unit == 0 means an implicit unit diagonal. */ +#if defined(TRMM) + if (rdna2_oblas_strmm_cm(args.m, args.n, r_alpha, (const float *)args.a, args.lda, + (float *)args.b, args.ldb, side, uplo, trans, unit == 0)) + return; +#else + if (rdna2_oblas_strsm_cm(args.m, args.n, r_alpha, (const float *)args.a, args.lda, + (float *)args.b, args.ldb, side, uplo, trans, unit == 0)) + return; +#endif + } +#endif + #else void CNAME(enum CBLAS_ORDER order, @@ -363,6 +385,28 @@ void CNAME(enum CBLAS_ORDER order, if (args.m == 0 || args.n == 0) return; +#if defined(OPENBLAS_USE_HIP) && !defined(COMPLEX) && !defined(DOUBLE) && !defined(XDOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) + { + extern int rdna2_oblas_strmm_cm(BLASLONG, BLASLONG, double, const float *, + BLASLONG, float *, BLASLONG, int, int, int, int); + extern int rdna2_oblas_strsm_cm(BLASLONG, BLASLONG, double, const float *, + BLASLONG, float *, BLASLONG, int, int, int, int); + const double r_alpha = (double)*(const FLOAT *)args.beta; + + /* OpenBLAS' trsm.c serves both TRMM and TRSM, and both work in place on + * args.b. unit == 0 means an implicit unit diagonal. */ +#if defined(TRMM) + if (rdna2_oblas_strmm_cm(args.m, args.n, r_alpha, (const float *)args.a, args.lda, + (float *)args.b, args.ldb, side, uplo, trans, unit == 0)) + return; +#else + if (rdna2_oblas_strsm_cm(args.m, args.n, r_alpha, (const float *)args.a, args.lda, + (float *)args.b, args.ldb, side, uplo, trans, unit == 0)) + return; +#endif + } +#endif + #if defined(TRMM) #if !defined(COMPLEX) && !defined(DOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) #if defined(ARCH_ARM64) && (defined(USE_STRMM_KERNEL_DIRECT)||defined(DYNAMIC_ARCH)) diff --git a/kernel/rdna2/rdna2_gemm_ref.h b/kernel/rdna2/rdna2_gemm_ref.h deleted file mode 100644 index d5f8c038fa..0000000000 --- a/kernel/rdna2/rdna2_gemm_ref.h +++ /dev/null @@ -1,219 +0,0 @@ -// CPU references for the RDNA2 kernels. Plain C++ (no HIP), so every -// reference can be built and tested on a machine without a GPU. The launchers -// in rdna2_kernels.hip use these as the fallback when no device is present, -// and the test uses them as the correctness oracle. -// -// Packed int8 convention (matches v_dot4_*): four int8 values per int32, -// value i at bits [8*i, 8*i+8), i.e. little-endian byte order. -#ifndef RDNA2_GEMM_REF_H -#define RDNA2_GEMM_REF_H - -#include -#include -#include - -// Ref functions that device code calls must be __host__ __device__. Under a -// plain host compiler the macro expands to nothing. -#ifndef RDNA2_HD -#if defined(__HIP__) || defined(__HIPCC__) || defined(__CUDACC__) -#define RDNA2_HD __host__ __device__ -#else -#define RDNA2_HD -#endif -#endif - -// --------------------------------------------------------------------------- -// dp4a int8 / u8 -// --------------------------------------------------------------------------- -RDNA2_HD static inline int32_t rdna2_dot4_i8_ref(int32_t a, int32_t b, int32_t acc) { - for (int i = 0; i < 4; ++i) { - int8_t x = static_cast((static_cast(a) >> (8 * i)) & 0xffu); - int8_t y = static_cast((static_cast(b) >> (8 * i)) & 0xffu); - uint32_t p = static_cast(static_cast(x) * static_cast(y)); - acc = static_cast(static_cast(acc) + p); - } - return acc; -} - -static inline uint32_t rdna2_dot4_u8_ref(uint32_t a, uint32_t b, uint32_t acc) { - for (int i = 0; i < 4; ++i) { - uint32_t x = (a >> (8 * i)) & 0xffu; - uint32_t y = (b >> (8 * i)) & 0xffu; - acc += x * y; - } - return acc; -} - -// The dot4 primitive used by the kernel bodies. On the device this is the real -// v_dot4c_i32_i8 (dp4a) instruction, independent of what the optimiser can -// pattern-match; on the host it is the bit-exact scalar reference above, so the -// same kernel body still runs on a CPU-only machine. -RDNA2_HD static inline int32_t rdna2_dot4_i8_impl(int32_t a, int32_t b, int32_t acc) { -#if defined(__HIP_DEVICE_COMPILE__) && !defined(RDNA2_NO_DOT1) -#if defined(__gfx1200__) || defined(__gfx1201__) - // gfx12 (RDNA4) has no dot1-insts, so `__builtin_amdgcn_sdot4` is rejected - // ("needs target feature dot1-insts"). The dp4a hardware is still there: the - // legacy v_dot4_i32_i8 mnemonic assembles and computes exact signed results - // (checked against the scalar reference on a gfx1201 card). Note that - // v_dot4_i32_iu8 is *not* interchangeable: on gfx12 it is the unsigned form. - asm volatile("v_dot4_i32_i8 %0, %1, %2, %0" : "+v"(acc) : "v"(a), "v"(b)); - return acc; -#else - return __builtin_amdgcn_sdot4(a, b, acc, false); -#endif -#else - // Either a host build or -DRDNA2_NO_DOT1=1 (a target whose dot1-insts feature - // is missing, e.g. an older gfx12 toolchain): same arithmetic, four scalar - // multiplies. Both branches are compared in the host test. - return rdna2_dot4_i8_ref(a, b, acc); -#endif -} - -// --------------------------------------------------------------------------- -// GEMV: y[M] = A[M x K4] . x[K4], packed int8 -// --------------------------------------------------------------------------- -static inline void rdna2_gemv_i8_ref(int32_t* y, const int32_t* A, const int32_t* x, - int rows, int K4) { - for (int r = 0; r < rows; ++r) { - int32_t acc = 0; - for (int k = 0; k < K4; ++k) acc = rdna2_dot4_i8_ref(A[(size_t)r * K4 + k], x[k], acc); - y[r] = acc; - } -} - -// --------------------------------------------------------------------------- -// GEMV: y[M] = A[M x K] . x[K], float -// --------------------------------------------------------------------------- -static inline void rdna2_sgemv_f32_ref(float* y, const float* A, const float* x, - int rows, int K) { - for (int r = 0; r < rows; ++r) { - // The reference accumulates in double and rounds once, so it does not - // share the kernel's four-way partial-sum split; the test compares with a - // tolerance rather than bit-for-bit. - double acc = 0.0; - for (int k = 0; k < K; ++k) acc += (double)A[(size_t)r * K + k] * (double)x[k]; - y[r] = (float)acc; - } -} - -// --------------------------------------------------------------------------- -// GEMM: C[M x N] = A[M x K4] * Bt[N x K4]^T, packed int8 -// --------------------------------------------------------------------------- -static inline void rdna2_gemm_i8_ref(int32_t* C, const int32_t* A, const int32_t* Bt, - int M, int N, int K4) { - for (int m = 0; m < M; ++m) { - for (int n = 0; n < N; ++n) { - const int32_t* a = A + static_cast(m) * K4; - const int32_t* b = Bt + static_cast(n) * K4; - int32_t acc = 0; - for (int k = 0; k < K4; ++k) acc = rdna2_dot4_i8_ref(a[k], b[k], acc); - C[static_cast(m) * N + n] = acc; - } - } -} - -// --------------------------------------------------------------------------- -// GEMM with the quantized epilogue applied on the CPU: -// C[m][n] = (acc[m][n] - zp_b * rowsum_a[m]) * sa[m] * sb[n] -// rowsum_a[m] is the sum of the int8 activations in row m. That is the exact -// integer form of B's asymmetric storage - B's stored value at k represents -// (q - zp_b), so sum_k a*(q - zp_b) = acc - zp_b*sum_k a - and it never converts -// a stored weight: only the finished dot product becomes float. sa / sb may be -// NULL (unit scale). The correction and the product are computed in 64-bit so -// the reference cannot overflow before its single float conversion. -// --------------------------------------------------------------------------- -static inline void rdna2_gemm_i8_scale_ref(float* C, const int32_t* A, const int32_t* Bt, - int M, int N, int K4, const float* sa, - const float* sb, int zp_b) { - for (int m = 0; m < M; ++m) { - const int32_t* a = A + static_cast(m) * K4; - int64_t rowsum = 0; - if (zp_b != 0) { - for (int k = 0; k < K4; ++k) { - for (int b = 0; b < 4; ++b) { - rowsum += static_cast((static_cast(a[k]) >> (8 * b)) & 0xffu); - } - } - } - const float s_a = sa ? sa[m] : 1.0f; - for (int n = 0; n < N; ++n) { - const int32_t* b = Bt + static_cast(n) * K4; - int32_t acc = 0; - for (int k = 0; k < K4; ++k) acc = rdna2_dot4_i8_ref(a[k], b[k], acc); - const int64_t corr = static_cast(acc) - static_cast(zp_b) * rowsum; - C[static_cast(m) * N + n] = - static_cast(corr) * s_a * (sb ? sb[n] : 1.0f); - } - } -} - -// --------------------------------------------------------------------------- -// SGEMM: C[M x N] = A[M x K] * Bt[N x K]^T, float -// --------------------------------------------------------------------------- -static inline void rdna2_sgemm_f32_ref(float* C, const float* A, const float* Bt, - int M, int N, int K) { - for (int m = 0; m < M; ++m) { - for (int n = 0; n < N; ++n) { - const float* a = A + static_cast(m) * K; - const float* b = Bt + static_cast(n) * K; - float acc = 0.0f; - for (int k = 0; k < K; ++k) acc += a[k] * b[k]; - C[static_cast(m) * N + n] = acc; - } - } -} - -// --------------------------------------------------------------------------- -// f16 dot (v_dot2c_f32_f16): acc = sum over i of a.xy * b.xy, f32 accumulate. -// Pairs are packed two _Float16 per int32, low half first. -// --------------------------------------------------------------------------- -#if defined(__FLT16_MAX__) -static inline void rdna2_unpack_half2(uint32_t v, _Float16* lo, _Float16* hi) { - uint16_t l = static_cast(v & 0xffffu); - uint16_t h = static_cast(v >> 16); - std::memcpy(lo, &l, sizeof(uint16_t)); - std::memcpy(hi, &h, sizeof(uint16_t)); -} - -static inline uint32_t rdna2_pack_half2(_Float16 lo, _Float16 hi) { - uint16_t l, h; - std::memcpy(&l, &lo, sizeof(uint16_t)); - std::memcpy(&h, &hi, sizeof(uint16_t)); - return static_cast(l) | (static_cast(h) << 16); -} - -// Reference matches the hardware order: acc += a.x*b.x then acc += a.y*b.y. -static inline float rdna2_dot_f16_ref(const uint32_t* a, const uint32_t* b, int n) { - float acc = 0.0f; - for (int i = 0; i < n; ++i) { - _Float16 ax, ay, bx, by; - rdna2_unpack_half2(a[i], &ax, &ay); - rdna2_unpack_half2(b[i], &bx, &by); - acc += static_cast(ax) * static_cast(bx); - acc += static_cast(ay) * static_cast(by); - } - return acc; -} -#endif - -// --------------------------------------------------------------------------- -// u8 dot: out = sum over i of dot4_u8(a[i], b[i]) -// --------------------------------------------------------------------------- -static inline uint32_t rdna2_dot_u8_ref(const uint32_t* a, const uint32_t* b, int n) { - uint32_t acc = 0; - for (int i = 0; i < n; ++i) acc = rdna2_dot4_u8_ref(a[i], b[i], acc); - return acc; -} - -// --------------------------------------------------------------------------- -// 32-lane inclusive prefix sum (matches rdna2_dpp_prefix on wave32). -// --------------------------------------------------------------------------- -static inline void rdna2_prefix_ref(int32_t* out, const int32_t* in, int n) { - int32_t run = 0; - for (int i = 0; i < n; ++i) { - run += in[i]; - out[i] = run; - } -} - -#endif // RDNA2_GEMM_REF_H diff --git a/kernel/rdna2/rdna2_kernels.hip b/kernel/rdna2/rdna2_kernels.hip deleted file mode 100644 index 69f57ae192..0000000000 --- a/kernel/rdna2/rdna2_kernels.hip +++ /dev/null @@ -1,1367 +0,0 @@ -// RDNA2 (gfx1030 / gfx1031) kernels, written directly against the GFX10.3 ISA, -// plus an RDNA4 (gfx1200 / gfx1201, RX 9060 XT / 9070 XT) tile shape that spends -// gfx12's larger register file on a 128x128 tile. -// Not a CUDA-to-HIP port: each kernel is shaped around one RDNA2 instruction -// group, and every one of them is checked against a plain-C++ reference -// (rdna2_gemm_ref.h) by test_rdna2_host.cpp. -// -// Instruction groups used: -// dot4 : v_dot4_i32_i8 / v_dot4c_i32_i8 / v_dot4_u32_u8 dp4a, int8 x4 -// dot2 : v_dot2_f32_f16 packed f16 dot -// dpp : v_add_nc_u32_dpp row_shr 16-lane wave scan -// ds_* : ds_read_b32 / ds_write_b32 LDS staging -// -// Build: clang++ -x hip --offload-arch=gfx1031 -O2 -c rdna2_kernels.hip -// clang++ -x hip --offload-arch=gfx1201 -O2 -c rdna2_kernels.hip (9070 XT) -// -// Return codes shared by all launchers: -// 0 = success, 1 = HIP/runtime error, 2 = invalid argument. - -#include -#include -#include - -#include "rdna2_gemm_ref.h" -#include "rdna2_kernels_math.h" - -#define RDNA2_OK 0 -#define RDNA2_ERR_HIP 1 -#define RDNA2_ERR_ARG 2 - -// The RDNA2_ARCH_* target codes live in rdna2_kernels_math.h, next to the tile -// shapes they select, so the host test uses the same numbers. - -// Four packed int32 = 16 bytes: one global_load_dwordx4 on the gfx1031 path. -typedef int int4_t __attribute__((ext_vector_type(4))); -// Two packed f16: the operand of v_dot2_f32_f16. -typedef _Float16 half2_t __attribute__((ext_vector_type(2))); -// WMMA operands and accumulators: V_WMMA_I32_16X16X16_IU8 takes two pairs of -// packed int32 (16 operand bytes per lane) and eight int32 accumulators. -typedef int int32x2_t __attribute__((ext_vector_type(2))); -typedef int int32x8_t __attribute__((ext_vector_type(8))); - -// Whether this compilation pass can emit V_WMMA_I32_16X16X16_IU8: only the -// device passes of gfx1200 / gfx1201. Note the host pass of the same build never -// has it, so this must not decide which kernel gets launched -- the launch site -// is unconditional and only the instruction inside the kernel is arch-gated. -#if defined(__HIP_DEVICE_COMPILE__) && (defined(__gfx1200__) || defined(__gfx1201__)) -#define RDNA2_WMMA_IU8 1 -#else -#define RDNA2_WMMA_IU8 0 -#endif - - -// The LDS tiling shape (RDNA2_TILE_MN, RDNA2_LDS_*_KSTEP/STRIDE) lives in -// rdna2_kernels_math.h so the host simulation uses the identical numbers. - -// --------------------------------------------------------------------------- -// Shared helpers -// --------------------------------------------------------------------------- -static inline int rdna2_device_count(void) { - int n = 0; - if (hipGetDeviceCount(&n) != hipSuccess) return 0; - return n; -} - -// 0 when the query fails or reports nonsense. Callers must then avoid shapes -// that depend on the wave width rather than guess one: the wave-per-row GEMV -// derives rows-per-block from blockDim/warpSize on the device, so a host guess -// of 32 with an actual wave64 would leave half the rows unprocessed. -static inline int rdna2_device_warp_size(void) { - hipDeviceProp_t prop{}; - if (hipGetDeviceProperties(&prop, 0) != hipSuccess || prop.warpSize <= 0) return 0; - const int ws = prop.warpSize; - if (ws > 64 || (ws & (ws - 1)) != 0) return 0; // not a power of two <= 64 - return ws; -} - -static inline int rdna2_warp_size(void) { - const int ws = rdna2_device_warp_size(); - return ws > 0 ? ws : 32; -} - -// --------------------------------------------------------------------------- -// Group 1: dp4a int8 GEMV. y[rows] = A[rows x k4] . x[k4] -// gfx1031 reads four packed words per load (16-byte vector access); any k4 is -// accepted because the remainder is handled by a scalar tail. Alignment is not -// a caller requirement: the launcher copies into a device buffer, which HIP -// allocates with far stronger alignment than the 16 bytes the vectors need. -// --------------------------------------------------------------------------- -__global__ void rdna2_dp4a_gemv_i8(int* __restrict__ y, - const int* __restrict__ A, // rows x k4 packed - const int* __restrict__ x, // k4 packed - int rows, int k4) { - int row = blockIdx.x * blockDim.x + threadIdx.x; - if (row >= rows) return; - const int* a_s = A + (size_t)row * k4; - // Four independent accumulators: v_dot4 has a multi-cycle result latency, so - // one dependent chain would leave the dot4 pipes idle. Integer addition is - // exact, so summing the four at the end is bit-identical to one chain. - int acc0 = 0, acc1 = 0, acc2 = 0, acc3 = 0; -#if defined(__gfx1031__) || defined(__gfx1200__) || defined(__gfx1201__) - int j = 0; - // The 16-byte vector load is only legal when this row also starts on a - // 16-byte boundary. A row starts at (row * k4) packed words, so that holds - // for every row only when k4 is a multiple of 4. For any other k4 the row - // stride is not 16-byte aligned and v_dot4 is fed from scalar loads instead - // (misaligned dwordx4 loads are not handled by the hardware). - if ((k4 & 3) == 0) { - const int4_t* a = reinterpret_cast(a_s); - const int4_t* xv = reinterpret_cast(x); - for (; j + 4 <= k4; j += 4) { - int4_t av = a[j / 4]; - int4_t xw = xv[j / 4]; - acc0 = rdna2_dot4_i8_impl(av.x, xw.x, acc0); - acc1 = rdna2_dot4_i8_impl(av.y, xw.y, acc1); - acc2 = rdna2_dot4_i8_impl(av.z, xw.z, acc2); - acc3 = rdna2_dot4_i8_impl(av.w, xw.w, acc3); - } - } - for (; j < k4; ++j) acc0 = rdna2_dot4_i8_impl(a_s[j], x[j], acc0); -#else - for (int j = 0; j < k4; ++j) { - acc0 = rdna2_dot4_i8_impl(a_s[j], x[j], acc0); - } -#endif - y[row] = (acc0 + acc1) + (acc2 + acc3); -} - -// One wave per row. Lanes stride the row, so a wave reads 32 consecutive words -// (one 128-byte line) per step instead of the 32 scattered row starts the -// thread-per-row kernel touches, and the row is read exactly once. Better than -// one thread per row once a row holds enough words to keep a wave busy; the -// launcher switches on k4 (see RDNA2_GEMV_WAVE_MIN_K4). -__global__ void rdna2_dp4a_gemv_i8_wave(int* __restrict__ y, - const int* __restrict__ A, - const int* __restrict__ x, int rows, int k4) { - const int ws = (int)warpSize; - const int lane = (int)(threadIdx.x % warpSize); - const int wave = (int)(threadIdx.x / warpSize); - const int waves = (int)(blockDim.x / warpSize); - const int row = (int)blockIdx.x * waves + wave; - if (row >= rows) return; // wave-uniform: a wave never splits across rows - const int* a_s = A + (size_t)row * k4; - int acc = 0; - for (int k = lane; k < k4; k += ws) acc = rdna2_dot4_i8_impl(a_s[k], x[k], acc); - for (int off = ws >> 1; off > 0; off >>= 1) acc += __shfl_xor(acc, off); - if (lane == 0) y[row] = acc; -} - -// --------------------------------------------------------------------------- -// Group 1c: f32 GEMV, y[M] = A[M x K] . x[K]. The same two shapes as the int8 -// GEMV above: one thread per row (short rows) and one wave per row (long rows, -// lanes stride the row so a wave touches consecutive floats). -// --------------------------------------------------------------------------- -__global__ void rdna2_sgemv_f32(float* __restrict__ y, const float* __restrict__ A, - const float* __restrict__ x, int rows, int K) { - const int row = (int)blockIdx.x * blockDim.x + (int)threadIdx.x; - if (row >= rows) return; - const float* a_s = A + (size_t)row * K; - y[row] = rdna2_sgemv_row_f32(a_s, x, K); -} - -__global__ void rdna2_sgemv_f32_wave(float* __restrict__ y, const float* __restrict__ A, - const float* __restrict__ x, int rows, int K) { - const int ws = (int)warpSize; - const int lane = (int)(threadIdx.x % warpSize); - const int wave = (int)(threadIdx.x / warpSize); - const int waves = (int)(blockDim.x / warpSize); - const int row = (int)blockIdx.x * waves + wave; - if (row >= rows) return; // wave-uniform: a wave never splits across rows - const float* a_s = A + (size_t)row * K; - float acc = rdna2_sgemv_lane_partial_f32(a_s, x, K, lane, ws); - for (int off = ws >> 1; off > 0; off >>= 1) acc += __shfl_xor(acc, off); - if (lane == 0) y[row] = acc; -} - -// --------------------------------------------------------------------------- -// Group 1b: dp4a int8 GEMM. -// C[M x N] = A[M x K4] * Bt[N x K4]^T (B stored transposed, N rows of K4) -// Two implementations: a naive one (one thread per element) and a 4x4 -// register-tiled one. The device test compares both against the CPU reference. -// --------------------------------------------------------------------------- -__global__ void rdna2_dp4a_gemm_i8_naive(int* __restrict__ C, - const int* __restrict__ A, - const int* __restrict__ Bt, - int M, int N, int K4) { - int n = blockIdx.x * blockDim.x + threadIdx.x; - int m = blockIdx.y; - if (m >= M || n >= N) return; - const int* a = A + (size_t)m * K4; - const int* b = Bt + (size_t)n * K4; - C[(size_t)m * N + n] = rdna2_gemm_cell_i8(a, b, K4); -} - -// 4x4 outputs per thread; block (16,16) covers a 64x64 output tile. -__global__ void rdna2_dp4a_gemm_i8_tile(int* __restrict__ C, - const int* __restrict__ A, - const int* __restrict__ Bt, - int M, int N, int K4) { - const int m0 = (blockIdx.y * blockDim.y + threadIdx.y) * 4; - const int n0 = (blockIdx.x * blockDim.x + threadIdx.x) * 4; - if (m0 >= M || n0 >= N) return; - - int32_t acc[4][4]; - rdna2_gemm_tile_i8(acc, A, Bt, M, N, K4, m0, n0); - for (int i = 0; i < 4; ++i) { - if (m0 + i >= M) break; - for (int j = 0; j < 4; ++j) { - if (n0 + j >= N) break; - C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; - } - } -} - -// --------------------------------------------------------------------------- -// Group 1c: LDS-staged tiled int8 GEMM, same 64x64 output tile and 4x4 register -// blocking as above, but the K loop reads the operands from LDS instead of -// global memory. The thread-per-cell version issues 8 packed loads per k step -// per thread; over a 64x64 tile that is 2048 words of traffic for 128 words of -// actual data (16x). Staging each K chunk cooperatively costs 1024 loads spread -// over 256 threads (4 each) and then every reuse is LDS. -// --------------------------------------------------------------------------- -// Stage one K chunk of a matrix into LDS. 256 threads cover 64 rows x 16 words; -// each thread owns 4 consecutive words of one row. Rows past the edge and words -// past K4 are written as zero, so the compute loop needs no bounds checks. -// lds is a template parameter so the __shared__ address space survives: a fixed -// int (*)[STRIDE] parameter would genericize it to flat stores. -template -__device__ static inline void rdna2_stage_i8(LDP lds, const int* __restrict__ src, int nrows, - int ncols, int row0, int k0, int ld_row, int ld_col) { - const int gm = row0 + ld_row; - int4_t v; - v.x = v.y = v.z = v.w = 0; - if (gm < nrows) { - const int* p = src + (size_t)gm * ncols + k0 + ld_col; - if ((ncols & 3) == 0 && k0 + ld_col + 4 <= ncols) { - v = *reinterpret_cast(p); // 16-byte access, aligned by ncols%4==0 - } else { - for (int c = 0; c < 4; ++c) - if (k0 + ld_col + c < ncols) v[c] = p[c]; - } - } - lds[ld_row][ld_col + 0] = v.x; - lds[ld_row][ld_col + 1] = v.y; - lds[ld_row][ld_col + 2] = v.z; - lds[ld_row][ld_col + 3] = v.w; -} - -__global__ void rdna2_dp4a_gemm_i8_lds(int* __restrict__ C, - const int* __restrict__ A, - const int* __restrict__ Bt, - int M, int N, int K4) { - __shared__ int a_lds[RDNA2_TILE_MN][RDNA2_LDS_I8_STRIDE]; - __shared__ int b_lds[RDNA2_TILE_MN][RDNA2_LDS_I8_STRIDE]; - const int tid = threadIdx.y * blockDim.x + threadIdx.x; - const int m_blk = blockIdx.y * RDNA2_TILE_MN; - const int n_blk = blockIdx.x * RDNA2_TILE_MN; - const int lm = threadIdx.y * 4; // this thread's first tile row - const int ln = threadIdx.x * 4; // this thread's first tile column - const int ld_row = tid >> 2; - const int ld_col = (tid & 3) << 2; - - int32_t acc[4][4]; - for (int i = 0; i < 4; ++i) - for (int j = 0; j < 4; ++j) acc[i][j] = 0; - - for (int k0 = 0; k0 < K4; k0 += RDNA2_LDS_I8_KSTEP) { - rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col); - rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col); - __syncthreads(); - int kend = K4 - k0; - if (kend > RDNA2_LDS_I8_KSTEP) kend = RDNA2_LDS_I8_KSTEP; - rdna2_gemm_lds_block_i8(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_LDS_I8_STRIDE, lm, ln, kend); - __syncthreads(); // nobody may restage a tile that is still being read - } - - for (int i = 0; i < 4; ++i) { - if (m_blk + lm + i >= M) break; - for (int j = 0; j < 4; ++j) { - if (n_blk + ln + j >= N) break; - C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; - } - } -} - -// Same LDS tile and K loop, 8x4 register blocking: 128 threads (16,8) cover the -// same 64x64 output tile, each owning 8 rows x 4 columns. Twelve LDS reads per k -// step feed 32 dp4a instead of eight feeding 16, i.e. a quarter less LDS traffic -// for identical math. Staging spreads the same 64x16 words over 128 threads, so -// each thread writes 8 words (two 16-byte loads) instead of 4. -__global__ void rdna2_dp4a_gemm_i8_lds8(int* __restrict__ C, - const int* __restrict__ A, - const int* __restrict__ Bt, - int M, int N, int K4) { - __shared__ int a_lds[RDNA2_TILE_MN][RDNA2_LDS_I8_STRIDE]; - __shared__ int b_lds[RDNA2_TILE_MN][RDNA2_LDS_I8_STRIDE]; - const int tid = threadIdx.y * blockDim.x + threadIdx.x; - const int m_blk = blockIdx.y * RDNA2_TILE_MN; - const int n_blk = blockIdx.x * RDNA2_TILE_MN; - const int lm = threadIdx.y * 8; // this thread's first tile row - const int ln = threadIdx.x * 4; // this thread's first tile column - const int ld_row = tid >> 1; // 0..63, two threads per staged row - const int ld_col = (tid & 1) << 3; - - int32_t acc[8][4]; - for (int i = 0; i < 8; ++i) - for (int j = 0; j < 4; ++j) acc[i][j] = 0; - - for (int k0 = 0; k0 < K4; k0 += RDNA2_LDS_I8_KSTEP) { - rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col); - rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col + 4); - rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col); - rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col + 4); - __syncthreads(); - int kend = K4 - k0; - if (kend > RDNA2_LDS_I8_KSTEP) kend = RDNA2_LDS_I8_KSTEP; - rdna2_gemm_lds_block8_i8(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_LDS_I8_STRIDE, lm, ln, kend); - __syncthreads(); // nobody may restage a tile that is still being read - } - - for (int i = 0; i < 8; ++i) { - if (m_blk + lm + i >= M) break; - for (int j = 0; j < 4; ++j) { - if (n_blk + ln + j >= N) break; - C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; - } - } -} - -// --------------------------------------------------------------------------- -// Group 1e: RDNA4 (gfx1200 / gfx1201) path. Same 16x16 block of 256 threads and -// the same 16-word K chunk as groups 1c/1d, but a 128x128 output tile with 8x8 -// outputs per thread: sixteen LDS reads per k step feed 64 dp4a instead of eight -// feeding 16, and every staged word pays for four times as many results. gfx12 -// has 1536 VGPRs per SIMD against 1024 on gfx10/11, so 64 accumulators plus 16 -// staged operands still leave several waves resident. Two staged tiles of -// 128 x 17 words = 17408 bytes, inside the 64 KiB a gfx12 workgroup addresses. -// Select with tiled = 4 (see rdna2_tiled_for_arch). -// --------------------------------------------------------------------------- -__global__ void rdna2_dp4a_gemm_i8_lds128(int* __restrict__ C, - const int* __restrict__ A, - const int* __restrict__ Bt, - int M, int N, int K4) { - __shared__ int a_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_I8_STRIDE]; - __shared__ int b_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_I8_STRIDE]; - const int tid = threadIdx.y * blockDim.x + threadIdx.x; - const int m_blk = blockIdx.y * RDNA2_T4_TILE_MN; - const int n_blk = blockIdx.x * RDNA2_T4_TILE_MN; - const int lm = threadIdx.y * 8; // this thread's first tile row - const int ln = threadIdx.x * 8; // this thread's first tile column - const int ld_row = tid >> 1; // 0..127: two threads per staged row - const int ld_col = (tid & 1) << 3; // 0 or 8, eight staged words each - - int32_t acc[8][8]; - for (int i = 0; i < 8; ++i) - for (int j = 0; j < 8; ++j) acc[i][j] = 0; - - for (int k0 = 0; k0 < K4; k0 += RDNA2_T4_LDS_I8_KSTEP) { - rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col); - rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col + 4); - rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col); - rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col + 4); - __syncthreads(); - int kend = K4 - k0; - if (kend > RDNA2_T4_LDS_I8_KSTEP) kend = RDNA2_T4_LDS_I8_KSTEP; - rdna2_gemm_lds_block8x8_i8(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_T4_LDS_I8_STRIDE, lm, - ln, kend); - __syncthreads(); // nobody may restage a tile that is still being read - } - - for (int i = 0; i < 8; ++i) { - if (m_blk + lm + i >= M) break; - for (int j = 0; j < 8; ++j) { - if (n_blk + ln + j >= N) break; - C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; - } - } -} - -// --------------------------------------------------------------------------- -// Group 1f: RDNA4 (gfx1200 / gfx1201) matrix path. Same 128x128 output tile, -// same 16-word K chunk and same LDS staging as group 1e, but the arithmetic is -// the gfx12 matrix instruction V_WMMA_I32_16X16X16_IU8 instead of 64 dp4a per -// thread: one instruction covers a 16x16x16 signed product that the 8x8 dp4a -// kernel needs 64 instructions and 16 LDS reads per k step to reach. -// -// The eight waves of the block each own a 16-row band of the tile and sweep the -// eight 16-column groups, keeping 8 x int32x8 accumulators (64 VGPRs, the same -// budget as group 1e's 8x8 block). Per K chunk a wave issues 32 WMMA -- 8 per -// 16-byte K step -- and pays only 2 LDS words for A and 2 per B fragment, i.e. -// 36 word reads for 32 matrix ops instead of 256 reads for 1024 dp4a. -// -// Fragment layout, measured on a gfx1201 card (256/256 outputs of a signed -// 16x16x16 product matched the scalar reference; the layout and the negA/negB -// semantics were both derived experimentally, see README): -// A: lane t, byte e -> A[M = t%16][K = (t/16)*8 + e] -// B: lane t, byte e -> B[K = (t/16)*8 + e][N = t%16] -// D: lane t, slot l -> D[M = (t/16)*8 + l][N = t%16] -// Both operands are signed, so negA and negB are true; __builtin_amdgcn_wmma_* -// needs those flags as compile-time constants, hence the literals. -// Select with tiled = 5 (see rdna2_tiled_for_arch). -// --------------------------------------------------------------------------- -// One 16x16x16 signed step for this lane: acc holds this lane's eight output -// slots, a and b are its eight-byte A and B fragments. -// gfx12 -- the matrix instruction. -// other AMD GPU -- no matrix unit on gfx10/11, so the same result is built -// from explicit byte dots, gathering each slot's operands -// out of lanes row and row+16 with shuffles. Correct, but -// with a shuffle per operand byte it cannot beat the dp4a -// kernel, and rdna2_tiled_for_arch never picks tiled = 5 on -// those chips. Not measured: no RDNA2 hardware here. -// plain C++ pass -- the file also compiles as host code (CPU test harness and -// the host half of a HIP build), where no kernel can run; the -// body is only there to keep the kernel defined so the -// unconditional launch site links. -__device__ static inline int32x8_t rdna2_wmma_iu8_step(int32x2_t a, int32x2_t b, int32x8_t acc, - int lane) { - (void)lane; // the matrix instruction addresses the other lanes in hardware -#if RDNA2_WMMA_IU8 - return __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a, true, b, acc, false); -#elif defined(__HIP_DEVICE_COMPILE__) - for (int l = 0; l < 8; ++l) { - const int dm = rdna2_wmma_i8_acc_row(lane, l); - const int dn = rdna2_wmma_i8_acc_col(lane); - int32_t d = 0; - for (int k = 0; k < 16; ++k) { - const int shift = 8 * (k & 3), word = (k & 7) >> 2; - const int av = (int8_t)(__shfl((int)a[word], dm + (k >> 3) * 16) >> shift); - const int bv = (int8_t)(__shfl((int)b[word], dn + (k >> 3) * 16) >> shift); - d += av * bv; - } - acc[l] += d; - } - return acc; -#else - (void)a; - (void)b; - return acc; -#endif -} - -__global__ void rdna2_dp4a_gemm_i8_wmma(int* __restrict__ C, - const int* __restrict__ A, - const int* __restrict__ Bt, - int M, int N, int K4) { - __shared__ int a_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_I8_STRIDE]; - __shared__ int b_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_I8_STRIDE]; - const int tid = threadIdx.y * blockDim.x + threadIdx.x; - const int lane = tid & 31; - const int wave = tid >> 5; // 0..7: this wave's 16-row band of the tile - const int m_blk = blockIdx.y * RDNA2_T4_TILE_MN; - const int n_blk = blockIdx.x * RDNA2_T4_TILE_MN; - const int m_row = wave * 16 + rdna2_wmma_i8_row(lane); // staged A row this lane feeds - const int n_col = rdna2_wmma_i8_row(lane); // column group offset inside a 16-wide group - const int ld_row = tid >> 1; // 0..127: two threads per staged row - const int ld_col = (tid & 1) << 3; // 0 or 8, eight staged words each - - int32x8_t acc[RDNA2_T4_WMMA_NTILE]; - for (int j = 0; j < RDNA2_T4_WMMA_NTILE; ++j) { - for (int l = 0; l < 8; ++l) acc[j][l] = 0; - } - - for (int k0 = 0; k0 < K4; k0 += RDNA2_T4_LDS_I8_KSTEP) { - rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col); - rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col + 4); - rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col); - rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col + 4); - __syncthreads(); - int kend = K4 - k0; - if (kend > RDNA2_T4_LDS_I8_KSTEP) kend = RDNA2_T4_LDS_I8_KSTEP; - // One WMMA covers 4 staged words (16 K bytes); a short last chunk runs the - // steps it has and relies on the staging zeros for the rest of the tile. - const int ksteps = (kend + 3) / 4; - for (int ks = 0; ks < ksteps; ++ks) { - const int w = rdna2_wmma_i8_frag_word(lane, ks); - int32x2_t a; - a[0] = a_lds[m_row][w]; - a[1] = a_lds[m_row][w + 1]; - for (int j = 0; j < RDNA2_T4_WMMA_NTILE; ++j) { - int32x2_t b; - b[0] = b_lds[j * 16 + n_col][w]; - b[1] = b_lds[j * 16 + n_col][w + 1]; - acc[j] = rdna2_wmma_iu8_step(a, b, acc[j], lane); - } - } - __syncthreads(); // nobody may restage a tile that is still being read - } - - // D map: slot l of lane t lands at row 8*(t/16) + l, column t%16 of the - // 16x16 tile, so a lane writes one column of its wave's 16-row band. - for (int j = 0; j < RDNA2_T4_WMMA_NTILE; ++j) { - const int n = n_blk + j * 16 + rdna2_wmma_i8_acc_col(lane); - if (n >= N) continue; - for (int l = 0; l < 8; ++l) { - const int m = m_blk + wave * 16 + rdna2_wmma_i8_acc_row(lane, l); - if (m >= M) break; // rows 8*(t/16)+l are contiguous, so a break is safe - C[(size_t)m * N + n] = acc[j][l]; - } - } -} - -// --------------------------------------------------------------------------- -// Group 2: f32 GEMM, 4x4 register tiled. C[M x N] = A[M x K] * Bt[N x K]^T -// --------------------------------------------------------------------------- -__global__ void rdna2_sgemm_f32(float* __restrict__ C, - const float* __restrict__ A, - const float* __restrict__ Bt, - int M, int N, int K) { - const int m0 = (blockIdx.y * blockDim.y + threadIdx.y) * 4; - const int n0 = (blockIdx.x * blockDim.x + threadIdx.x) * 4; - if (m0 >= M || n0 >= N) return; - - float acc[4][4]; - rdna2_sgemm_tile_f32(acc, A, Bt, M, N, K, m0, n0); - for (int i = 0; i < 4; ++i) { - if (m0 + i >= M) break; - for (int j = 0; j < 4; ++j) { - if (n0 + j >= N) break; - C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; - } - } -} - -// --------------------------------------------------------------------------- -// Group 2b: LDS-staged f32 GEMM, same 64x64 tile and 4x4 register blocking as -// group 1c: the K loop reads A and Bt from LDS instead of re-reading global -// memory in every thread. -// --------------------------------------------------------------------------- -// 256 threads cover 64 rows x 8 floats: two consecutive floats per thread. -template -__device__ static inline void rdna2_stage_f32(LDP lds, const float* __restrict__ src, int nrows, - int ncols, int row0, int k0, int ld_row, int ld_col) { - const int gm = row0 + ld_row; - float v0 = 0.0f, v1 = 0.0f; - if (gm < nrows) { - const float* p = src + (size_t)gm * ncols + k0 + ld_col; - if (k0 + ld_col + 0 < ncols) v0 = p[0]; - if (k0 + ld_col + 1 < ncols) v1 = p[1]; - } - lds[ld_row][ld_col + 0] = v0; - lds[ld_row][ld_col + 1] = v1; -} - -__global__ void rdna2_sgemm_f32_lds(float* __restrict__ C, - const float* __restrict__ A, - const float* __restrict__ Bt, - int M, int N, int K) { - __shared__ float a_lds[RDNA2_TILE_MN][RDNA2_LDS_F32_STRIDE]; - __shared__ float b_lds[RDNA2_TILE_MN][RDNA2_LDS_F32_STRIDE]; - const int tid = threadIdx.y * blockDim.x + threadIdx.x; - const int m_blk = blockIdx.y * RDNA2_TILE_MN; - const int n_blk = blockIdx.x * RDNA2_TILE_MN; - const int lm = threadIdx.y * 4; - const int ln = threadIdx.x * 4; - const int ld_row = tid >> 2; - const int ld_col = (tid & 3) << 1; - - float acc[4][4]; - for (int i = 0; i < 4; ++i) - for (int j = 0; j < 4; ++j) acc[i][j] = 0.0f; - - for (int k0 = 0; k0 < K; k0 += RDNA2_LDS_F32_KSTEP) { - rdna2_stage_f32(a_lds, A, M, K, m_blk, k0, ld_row, ld_col); - rdna2_stage_f32(b_lds, Bt, N, K, n_blk, k0, ld_row, ld_col); - __syncthreads(); - int kend = K - k0; - if (kend > RDNA2_LDS_F32_KSTEP) kend = RDNA2_LDS_F32_KSTEP; - rdna2_sgemm_lds_block_f32(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_LDS_F32_STRIDE, lm, ln, kend); - __syncthreads(); - } - - for (int i = 0; i < 4; ++i) { - if (m_blk + lm + i >= M) break; - for (int j = 0; j < 4; ++j) { - if (n_blk + ln + j >= N) break; - C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; - } - } -} - -// --------------------------------------------------------------------------- -// Group 2b: 64x64 f32 tile with a 16-float K chunk (variant 3). Same block -// shape and 4x4 per-thread blocking as group 2, but 256 threads stage 1024 -// floats (4 each, RDNA2_LDS_F32B_STRIDE = 17 per row) instead of 512 floats -// (2 each) per iteration. That makes each staged chunk cover a whole 64-byte -// cache line of every row, and halves the number of __syncthreads over K. -// --------------------------------------------------------------------------- -template -__device__ static inline void rdna2_stage_f32_4(LDP lds, const float* __restrict__ src, int nrows, - int ncols, int row0, int k0, int ld_row, - int ld_col) { - const int gm = row0 + ld_row; - float v0 = 0.0f, v1 = 0.0f, v2 = 0.0f, v3 = 0.0f; - if (gm < nrows) { - const float* p = src + (size_t)gm * ncols + k0 + ld_col; - const int rem = ncols - (k0 + ld_col); - if (rem > 0) v0 = p[0]; - if (rem > 1) v1 = p[1]; - if (rem > 2) v2 = p[2]; - if (rem > 3) v3 = p[3]; - } - lds[ld_row][ld_col + 0] = v0; - lds[ld_row][ld_col + 1] = v1; - lds[ld_row][ld_col + 2] = v2; - lds[ld_row][ld_col + 3] = v3; -} - -__global__ void rdna2_sgemm_f32_lds16(float* __restrict__ C, - const float* __restrict__ A, - const float* __restrict__ Bt, - int M, int N, int K) { - __shared__ float a_lds[RDNA2_TILE_MN][RDNA2_LDS_F32B_STRIDE]; - __shared__ float b_lds[RDNA2_TILE_MN][RDNA2_LDS_F32B_STRIDE]; - const int tid = threadIdx.y * blockDim.x + threadIdx.x; - const int m_blk = blockIdx.y * RDNA2_TILE_MN; - const int n_blk = blockIdx.x * RDNA2_TILE_MN; - const int lm = threadIdx.y * 4; - const int ln = threadIdx.x * 4; - const int ld_row = tid >> 2; - const int ld_col = (tid & 3) << 2; - - float acc[4][4]; - for (int i = 0; i < 4; ++i) - for (int j = 0; j < 4; ++j) acc[i][j] = 0.0f; - - for (int k0 = 0; k0 < K; k0 += RDNA2_LDS_F32B_KSTEP) { - rdna2_stage_f32_4(a_lds, A, M, K, m_blk, k0, ld_row, ld_col); - rdna2_stage_f32_4(b_lds, Bt, N, K, n_blk, k0, ld_row, ld_col); - __syncthreads(); - int kend = K - k0; - if (kend > RDNA2_LDS_F32B_KSTEP) kend = RDNA2_LDS_F32B_KSTEP; - rdna2_sgemm_lds_block_f32(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_LDS_F32B_STRIDE, lm, ln, kend); - __syncthreads(); - } - - for (int i = 0; i < 4; ++i) { - if (m_blk + lm + i >= M) break; - for (int j = 0; j < 4; ++j) { - if (n_blk + ln + j >= N) break; - C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; - } - } -} - -// --------------------------------------------------------------------------- -// Group 2c: RDNA4 (gfx1200 / gfx1201) f32 GEMM on the same 128x128 tile and 8x8 -// per-thread blocking as group 1e. Two staged tiles of 128 x 9 floats = 9216 -// bytes; sixteen LDS reads per k step feed 64 fma. Select with variant 2. -// --------------------------------------------------------------------------- -__global__ void rdna2_sgemm_f32_lds128(float* __restrict__ C, - const float* __restrict__ A, - const float* __restrict__ Bt, - int M, int N, int K) { - __shared__ float a_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_F32_STRIDE]; - __shared__ float b_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_F32_STRIDE]; - const int tid = threadIdx.y * blockDim.x + threadIdx.x; - const int m_blk = blockIdx.y * RDNA2_T4_TILE_MN; - const int n_blk = blockIdx.x * RDNA2_T4_TILE_MN; - const int lm = threadIdx.y * 8; - const int ln = threadIdx.x * 8; - const int ld_row = tid >> 1; // 0..127: two threads per staged row - const int ld_col = (tid & 1) << 2; // 0 or 4, four staged floats each - - float acc[8][8]; - for (int i = 0; i < 8; ++i) - for (int j = 0; j < 8; ++j) acc[i][j] = 0.0f; - - for (int k0 = 0; k0 < K; k0 += RDNA2_T4_LDS_F32_KSTEP) { - rdna2_stage_f32(a_lds, A, M, K, m_blk, k0, ld_row, ld_col); - rdna2_stage_f32(a_lds, A, M, K, m_blk, k0, ld_row, ld_col + 2); - rdna2_stage_f32(b_lds, Bt, N, K, n_blk, k0, ld_row, ld_col); - rdna2_stage_f32(b_lds, Bt, N, K, n_blk, k0, ld_row, ld_col + 2); - __syncthreads(); - int kend = K - k0; - if (kend > RDNA2_T4_LDS_F32_KSTEP) kend = RDNA2_T4_LDS_F32_KSTEP; - rdna2_sgemm_lds_block8x8_f32(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_T4_LDS_F32_STRIDE, lm, - ln, kend); - __syncthreads(); - } - - for (int i = 0; i < 8; ++i) { - if (m_blk + lm + i >= M) break; - for (int j = 0; j < 8; ++j) { - if (n_blk + ln + j >= N) break; - C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; - } - } -} - -// --------------------------------------------------------------------------- -// Group 3: packed f16 dot (v_dot2_f32_f16; clang emits the accumulate form -// from __builtin_amdgcn_fdot2, and the ISA mnemonic has no 'c'). -// out is accumulated with atomics and must be zeroed by the launcher. -// Correct for both wave32 and wave64: the shuffle reduction covers one -// 16-lane row, then one atomic per row. -// --------------------------------------------------------------------------- -__global__ void rdna2_dot2_f16(float* __restrict__ out, - const half2_t* __restrict__ a, - const half2_t* __restrict__ b, - int n) { - float acc = 0.0f; - for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < n; - i += gridDim.x * blockDim.x) { - acc = __builtin_amdgcn_fdot2(a[i], b[i], acc, false); - } - acc += __shfl_xor(acc, 1); - acc += __shfl_xor(acc, 2); - acc += __shfl_xor(acc, 4); - acc += __shfl_xor(acc, 8); - if ((threadIdx.x & 15) == 0) atomicAdd(out, acc); -} - -// --------------------------------------------------------------------------- -// Group 4: dp4a u8 dot (v_dot4_u32_u8). -// out is accumulated with atomics and must be zeroed by the launcher. -// --------------------------------------------------------------------------- -__global__ void rdna2_dp4a_dot_u8(unsigned* __restrict__ out, - const unsigned* __restrict__ a, - const unsigned* __restrict__ b, - int n) { - unsigned acc = 0; - for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < n; - i += gridDim.x * blockDim.x) { - acc = __builtin_amdgcn_udot4(a[i], b[i], acc, false); - } - acc += __shfl_xor(acc, 1); - acc += __shfl_xor(acc, 2); - acc += __shfl_xor(acc, 4); - acc += __shfl_xor(acc, 8); - if ((threadIdx.x & 15) == 0) atomicAdd(out, acc); -} - -// --------------------------------------------------------------------------- -// Group 5: inclusive prefix sum over one wave. -// DPP row_shr builds a 16-lane Hillis-Steele scan with no LDS traffic; the -// cross-half total then arrives through a shuffle (v_readlane/v_permlane). -// out[tid] = in[0] + ... + in[tid] for every lane tid < n <= warpSize. -// --------------------------------------------------------------------------- -__global__ void rdna2_dpp_prefix(int* __restrict__ out, - const int* __restrict__ in, - int n) { - const int lane = threadIdx.x & (warpSize - 1); - int v = (lane < n) ? in[lane] : 0; - - // row_shr:1,2,4,8 with bound_ctrl off: lanes shifted out of the row get 0. - v += __builtin_amdgcn_update_dpp(0, v, 0x111, 0xf, 0xf, false); - v += __builtin_amdgcn_update_dpp(0, v, 0x112, 0xf, 0xf, false); - v += __builtin_amdgcn_update_dpp(0, v, 0x114, 0xf, 0xf, false); - v += __builtin_amdgcn_update_dpp(0, v, 0x118, 0xf, 0xf, false); - - // Lane 15 holds the total of lanes 0-15; add it to the upper half. - int lo = __shfl(v, 15); - if (lane >= 16) v += lo; - // On wave64, lanes 32-63 also need the total of lanes 0-31. - if (warpSize == 64) { - int mid = __shfl(v, 31); - if (lane >= 32) v += mid; - } - out[lane] = v; -} - -// --------------------------------------------------------------------------- -// Group 6: quantized epilogue (scale and zero point) -// --------------------------------------------------------------------------- -// Per-row sum of the int8 activations. dp4a against an all-ones operand sums the -// four signed bytes of a packed word in one instruction, so this is one v_dot4 -// per word. Only needed for asymmetric weights (a nonzero zero point); a -// symmetric format passes zp_b = 0 and skips it entirely. -__global__ void rdna2_i8_row_sums(int* __restrict__ sums, const int* __restrict__ A, int rows, - int k4) { - const int row = blockIdx.x * blockDim.x + threadIdx.x; - if (row >= rows) return; - const int* a = A + (size_t)row * k4; - const int ones = 0x01010101; - int acc = 0; - for (int k = 0; k < k4; ++k) acc = rdna2_dot4_i8_impl(a[k], ones, acc); - sums[row] = acc; -} - -// Applies the quantized format's scale and zero point to the finished integer -// accumulators: -// C[m][n] = (acc[m][n] - zp_b * sums[m]) * sa[m] * sb[n] -// The correction is the exact integer form of B's asymmetric storage: B's stored -// value at k represents (q - zp_b), so sum_k a*(q - zp_b) = acc - zp_b*sum_k a. -// No stored weight is converted here - only the completed dot product is, once, -// after the correction has been applied in 64-bit so it cannot overflow. sa is -// NULL for unit row scale, sb NULL for unit column scale. -__global__ void rdna2_gemm_i8_epilogue(float* __restrict__ C, const int* __restrict__ acc, - const int* __restrict__ sums, int M, int N, - const float* __restrict__ sa, const float* __restrict__ sb, - int zp_b) { - const int n = blockIdx.x * blockDim.x + threadIdx.x; - if (n >= N) return; - // Rows stride over gridDim.y so the row count is not capped at the grid's - // y limit; each iteration is an independent output element. - for (int m = blockIdx.y; m < M; m += gridDim.y) { - long long v = (long long)acc[(size_t)m * N + n]; - if (zp_b != 0 && sums) v -= (long long)zp_b * (long long)sums[m]; - float f = (float)v; - if (sa) f *= sa[m]; - if (sb) f *= sb[n]; - C[(size_t)m * N + n] = f; - } -} - -// --------------------------------------------------------------------------- -// Host launchers -// --------------------------------------------------------------------------- -// One thread per row is the better shape for short rows (no reduction to pay -// for); from this many packed words per row on, a whole wave per row wins on -// coalescing. -#define RDNA2_GEMV_WAVE_MIN_K4 8 -// f32 rows need a real body before a wave pays for its reduction. Measured on a -// 9070 XT at 2048-16384 rows: below K=256 the five shuffle steps cost more than -// the coalescing gains (thread-per-row is 1.2-3.8x ahead at K=64..160), and from -// K=256 up the wave-per-row kernel wins every time (1.15x at K=256, ~2x by -// K=1024, 3.5x at K=4096 with 16384 rows). -#define RDNA2_SGEMV_WAVE_MIN_K 256 - -static int rdna2_gemv_i8_device(int* y, const int* A, const int* x, int rows, int k4, - int variant) { - int *dA = nullptr, *dx = nullptr, *dy = nullptr; - const size_t bytesA = (size_t)rows * k4 * sizeof(int); - const size_t bytesx = (size_t)k4 * sizeof(int); - const size_t bytesy = (size_t)rows * sizeof(int); - int rc = RDNA2_OK; - int ws = 0; // wave width; stays 0 unless the wave-per-row variant can size its grid - if (hipMalloc(&dA, bytesA) != hipSuccess || hipMalloc(&dx, bytesx) != hipSuccess || - hipMalloc(&dy, bytesy) != hipSuccess) { - rc = RDNA2_ERR_HIP; - goto cleanup; - } - if (hipMemcpy(dA, A, bytesA, hipMemcpyHostToDevice) != hipSuccess || - hipMemcpy(dx, x, bytesx, hipMemcpyHostToDevice) != hipSuccess) { - rc = RDNA2_ERR_HIP; - goto cleanup; - } - if (variant == 1) ws = rdna2_device_warp_size(); - if (ws > 0) { - const int threads = 128; // 4 waves of 32 or 2 waves of 64 - const int waves = threads / ws; - const int blocks = (rows + waves - 1) / waves; - hipLaunchKernelGGL(rdna2_dp4a_gemv_i8_wave, dim3(blocks), dim3(threads), 0, 0, - dy, dA, dx, rows, k4); - } else { - const int threads = 64; - const int blocks = (rows + threads - 1) / threads; - hipLaunchKernelGGL(rdna2_dp4a_gemv_i8, dim3(blocks), dim3(threads), 0, 0, - dy, dA, dx, rows, k4); - } - if (hipGetLastError() != hipSuccess || - hipMemcpy(y, dy, bytesy, hipMemcpyDeviceToHost) != hipSuccess) { - rc = RDNA2_ERR_HIP; - } -cleanup: - if (dA) (void)hipFree(dA); - if (dx) (void)hipFree(dx); - if (dy) (void)hipFree(dy); - return rc; -} - -// variant 0 = one thread per row, 1 = one wave per row. -int rdna2_gemv_i8_host_variant(int* y, const int* A, const int* x, int rows, int k4, int variant) { - if (!y || !A || !x || rows <= 0 || k4 <= 0) return RDNA2_ERR_ARG; - if (rdna2_device_count() == 0) { - rdna2_gemv_i8_ref(y, A, x, rows, k4); - return RDNA2_OK; - } - return rdna2_gemv_i8_device(y, A, x, rows, k4, variant); -} - -int rdna2_gemv_i8_host(int* y, const int* A, const int* x, int rows, int k4) { - return rdna2_gemv_i8_host_variant(y, A, x, rows, k4, k4 >= RDNA2_GEMV_WAVE_MIN_K4 ? 1 : 0); -} - -static int rdna2_sgemv_f32_device(float* y, const float* A, const float* x, int rows, int K, - int variant) { - float *dA = nullptr, *dx = nullptr, *dy = nullptr; - const size_t bytesA = (size_t)rows * K * sizeof(float); - const size_t bytesx = (size_t)K * sizeof(float); - const size_t bytesy = (size_t)rows * sizeof(float); - int rc = RDNA2_OK; - int ws = 0; // wave width; stays 0 unless the wave-per-row variant can size its grid - if (hipMalloc(&dA, bytesA) != hipSuccess || hipMalloc(&dx, bytesx) != hipSuccess || - hipMalloc(&dy, bytesy) != hipSuccess) { - rc = RDNA2_ERR_HIP; - goto cleanup; - } - if (hipMemcpy(dA, A, bytesA, hipMemcpyHostToDevice) != hipSuccess || - hipMemcpy(dx, x, bytesx, hipMemcpyHostToDevice) != hipSuccess) { - rc = RDNA2_ERR_HIP; - goto cleanup; - } - if (variant == 1) ws = rdna2_device_warp_size(); - if (ws > 0) { - const int threads = 128; // 4 waves of 32 or 2 waves of 64 - const int waves = threads / ws; - const int blocks = (rows + waves - 1) / waves; - hipLaunchKernelGGL(rdna2_sgemv_f32_wave, dim3(blocks), dim3(threads), 0, 0, - dy, dA, dx, rows, K); - } else { - const int threads = 64; - const int blocks = (rows + threads - 1) / threads; - hipLaunchKernelGGL(rdna2_sgemv_f32, dim3(blocks), dim3(threads), 0, 0, - dy, dA, dx, rows, K); - } - if (hipGetLastError() != hipSuccess || - hipMemcpy(y, dy, bytesy, hipMemcpyDeviceToHost) != hipSuccess) { - rc = RDNA2_ERR_HIP; - } -cleanup: - if (dA) (void)hipFree(dA); - if (dx) (void)hipFree(dx); - if (dy) (void)hipFree(dy); - return rc; -} - -// variant 0 = one thread per row, 1 = one wave per row. -int rdna2_sgemv_f32_host_variant(float* y, const float* A, const float* x, int rows, int K, - int variant) { - if (!y || !A || !x || rows <= 0 || K <= 0) return RDNA2_ERR_ARG; - if (rdna2_device_count() == 0) { - rdna2_sgemv_f32_ref(y, A, x, rows, K); - return RDNA2_OK; - } - return rdna2_sgemv_f32_device(y, A, x, rows, K, variant); -} - -int rdna2_sgemv_f32_host(float* y, const float* A, const float* x, int rows, int K) { - return rdna2_sgemv_f32_host_variant(y, A, x, rows, K, - K >= RDNA2_SGEMV_WAVE_MIN_K ? 1 : 0); -} - -// Shapes the int8 GEMM onto a grid and launches it. Shared by the plain entry -// point and the quantized-epilogue one so both measure the same kernels. -static void rdna2_launch_gemm_i8(int* dC, const int* dA, const int* dB, int M, int N, int K4, - int tiled) { - if (tiled == 4 || tiled == 5) { - // RDNA4 shape: 128x128 tile, 16x16 threads. 17408 B of LDS per block for - // int8, well inside the 64 KiB workgroup limit. tiled == 5 is the same tile - // driven by the matrix instruction instead of dp4a, so it shares the grid; - // which instruction runs is decided inside the kernel (see - // rdna2_wmma_iu8_step), not here -- this function is host code and cannot - // see the offload arch. - const dim3 grid((N + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN, - (M + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN, 1); - const dim3 block(16, 16, 1); - if (tiled == 5) - hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_wmma, grid, block, 0, 0, dC, dA, dB, M, N, K4); - else - hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_lds128, grid, block, 0, 0, dC, dA, dB, M, N, K4); - } else if (tiled) { - const dim3 grid((N + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN, - (M + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN, 1); - if (tiled == 3) { - const dim3 block(16, 8, 1); // 128 threads, 8x4 outputs each - hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_lds8, grid, block, 0, 0, dC, dA, dB, M, N, K4); - } else if (tiled == 2) { - const dim3 block(16, 16, 1); - hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_lds, grid, block, 0, 0, dC, dA, dB, M, N, K4); - } else { - const dim3 block(16, 16, 1); - hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_tile, grid, block, 0, 0, dC, dA, dB, M, N, K4); - } - } else { - const dim3 block(64, 1, 1); - const dim3 grid((N + 63) / 64, M, 1); - hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_naive, grid, block, 0, 0, dC, dA, dB, M, N, K4); - } -} - -static int rdna2_gemm_i8_device(int* C, const int* A, const int* Bt, int M, int N, int K4, - int tiled) { - int *dA = nullptr, *dB = nullptr, *dC = nullptr; - const size_t bytesA = (size_t)M * K4 * sizeof(int); - const size_t bytesB = (size_t)N * K4 * sizeof(int); - const size_t bytesC = (size_t)M * N * sizeof(int); - int rc = RDNA2_OK; - if (hipMalloc(&dA, bytesA) != hipSuccess || hipMalloc(&dB, bytesB) != hipSuccess || - hipMalloc(&dC, bytesC) != hipSuccess) { - rc = RDNA2_ERR_HIP; - goto cleanup; - } - if (hipMemcpy(dA, A, bytesA, hipMemcpyHostToDevice) != hipSuccess || - hipMemcpy(dB, Bt, bytesB, hipMemcpyHostToDevice) != hipSuccess) { - rc = RDNA2_ERR_HIP; - goto cleanup; - } - rdna2_launch_gemm_i8(dC, dA, dB, M, N, K4, tiled); - if (hipGetLastError() != hipSuccess || - hipMemcpy(C, dC, bytesC, hipMemcpyDeviceToHost) != hipSuccess) { - rc = RDNA2_ERR_HIP; - } -cleanup: - if (dA) (void)hipFree(dA); - if (dB) (void)hipFree(dB); - if (dC) (void)hipFree(dC); - return rc; -} - -// tiled: 0 = naive, 1 = 4x4 register tiled (global loads), 2 = same tiling with -// LDS-staged K chunks, 3 = LDS staging with 8x4 register blocking (quarter less -// LDS traffic for the same math), 4 = RDNA4 128x128 tile with 8x8 blocking. -// Any non-zero value keeps the tiled shape; 4 picks its own grid. -int rdna2_gemm_i8_host(int* C, const int* A, const int* Bt, int M, int N, int K4, int tiled) { - if (!C || !A || !Bt || M <= 0 || N <= 0 || K4 <= 0) return RDNA2_ERR_ARG; - if (rdna2_device_count() == 0) { - rdna2_gemm_i8_ref(C, A, Bt, M, N, K4); - return RDNA2_OK; - } - return rdna2_gemm_i8_device(C, A, Bt, M, N, K4, tiled); -} - -// Quantized entry point: the same int8 GEMM followed by the format's scale and -// zero point. A and Bt are consumed exactly as stored - no weight is expanded to -// float, and the only float produced is the finished dot product, once. C is f32 -// because a GEMM result feeds the next layer's float activation path; sa scales -// rows of A and sb columns of Bt (either may be NULL for 1.0), zp_b is B's zero -// point (0 for symmetric weights, in which case the row-sum pass is skipped). -static int rdna2_gemm_i8_scale_device(float* C, const int* A, const int* Bt, int M, int N, int K4, - int tiled, const float* sa, const float* sb, int zp_b) { - int *dA = nullptr, *dB = nullptr, *dAcc = nullptr, *dSums = nullptr; - float *dSa = nullptr, *dSb = nullptr, *dC = nullptr; - const size_t bytesA = (size_t)M * K4 * sizeof(int); - const size_t bytesB = (size_t)N * K4 * sizeof(int); - const size_t nOut = (size_t)M * N; - int rc = RDNA2_OK; - if (hipMalloc(&dA, bytesA) != hipSuccess || hipMalloc(&dB, bytesB) != hipSuccess || - hipMalloc(&dAcc, nOut * sizeof(int)) != hipSuccess || - hipMalloc(&dC, nOut * sizeof(float)) != hipSuccess) { - rc = RDNA2_ERR_HIP; - goto cleanup; - } - if ((zp_b != 0 && hipMalloc(&dSums, (size_t)M * sizeof(int)) != hipSuccess) || - (sa && hipMalloc(&dSa, (size_t)M * sizeof(float)) != hipSuccess) || - (sb && hipMalloc(&dSb, (size_t)N * sizeof(float)) != hipSuccess)) { - rc = RDNA2_ERR_HIP; - goto cleanup; - } - if (hipMemcpy(dA, A, bytesA, hipMemcpyHostToDevice) != hipSuccess || - hipMemcpy(dB, Bt, bytesB, hipMemcpyHostToDevice) != hipSuccess || - (sa && hipMemcpy(dSa, sa, (size_t)M * sizeof(float), hipMemcpyHostToDevice) != hipSuccess) || - (sb && hipMemcpy(dSb, sb, (size_t)N * sizeof(float), hipMemcpyHostToDevice) != hipSuccess)) { - rc = RDNA2_ERR_HIP; - goto cleanup; - } - rdna2_launch_gemm_i8(dAcc, dA, dB, M, N, K4, tiled); - if (zp_b != 0) { - const int threads = 128; - hipLaunchKernelGGL(rdna2_i8_row_sums, dim3((M + threads - 1) / threads), dim3(threads), 0, 0, - dSums, dA, M, K4); - } - { - const dim3 block(256, 1, 1); - const int rows_per_grid = 4096; // rows stride, so grid.y stays inside limits - const dim3 grid((N + 255) / 256, M < rows_per_grid ? M : rows_per_grid, 1); - hipLaunchKernelGGL(rdna2_gemm_i8_epilogue, grid, block, 0, 0, dC, dAcc, dSums, M, N, dSa, dSb, - zp_b); - } - if (hipGetLastError() != hipSuccess || - hipMemcpy(C, dC, nOut * sizeof(float), hipMemcpyDeviceToHost) != hipSuccess) { - rc = RDNA2_ERR_HIP; - } -cleanup: - if (dA) (void)hipFree(dA); - if (dB) (void)hipFree(dB); - if (dAcc) (void)hipFree(dAcc); - if (dSums) (void)hipFree(dSums); - if (dSa) (void)hipFree(dSa); - if (dSb) (void)hipFree(dSb); - if (dC) (void)hipFree(dC); - return rc; -} - -int rdna2_gemm_i8_scale_host(float* C, const int* A, const int* Bt, int M, int N, int K4, - int tiled, const float* sa, const float* sb, int zp_b) { - if (!C || !A || !Bt || M <= 0 || N <= 0 || K4 <= 0) return RDNA2_ERR_ARG; - if (rdna2_device_count() == 0) { - rdna2_gemm_i8_scale_ref(C, A, Bt, M, N, K4, sa, sb, zp_b); - return RDNA2_OK; - } - return rdna2_gemm_i8_scale_device(C, A, Bt, M, N, K4, tiled, sa, sb, zp_b); -} - -// variant 0 = 4x4 tiled reading global memory, 1 = the same tiling with LDS -// staging (fewer global reads per output), 2 = RDNA4 128x128 tile with 8x8 -// blocking. -int rdna2_sgemm_f32_host_v(float* C, const float* A, const float* Bt, int M, int N, int K, - int variant) { - if (!C || !A || !Bt || M <= 0 || N <= 0 || K <= 0) return RDNA2_ERR_ARG; - if (rdna2_device_count() == 0) { - rdna2_sgemm_f32_ref(C, A, Bt, M, N, K); - return RDNA2_OK; - } - float *dA = nullptr, *dB = nullptr, *dC = nullptr; - const size_t bytesA = (size_t)M * K * sizeof(float); - const size_t bytesB = (size_t)N * K * sizeof(float); - const size_t bytesC = (size_t)M * N * sizeof(float); - int rc = RDNA2_OK; - if (hipMalloc(&dA, bytesA) != hipSuccess || hipMalloc(&dB, bytesB) != hipSuccess || - hipMalloc(&dC, bytesC) != hipSuccess) { - rc = RDNA2_ERR_HIP; - goto cleanup; - } - if (hipMemcpy(dA, A, bytesA, hipMemcpyHostToDevice) != hipSuccess || - hipMemcpy(dB, Bt, bytesB, hipMemcpyHostToDevice) != hipSuccess) { - rc = RDNA2_ERR_HIP; - goto cleanup; - } - if (variant == 2) { - // RDNA4 shape: 128x128 f32 tile, 8x8 outputs per thread, 9216 B of LDS. - const dim3 block(16, 16, 1); - const dim3 grid((N + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN, - (M + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN, 1); - hipLaunchKernelGGL(rdna2_sgemm_f32_lds128, grid, block, 0, 0, dC, dA, dB, M, N, K); - if (hipGetLastError() != hipSuccess || - hipMemcpy(C, dC, bytesC, hipMemcpyDeviceToHost) != hipSuccess) { - rc = RDNA2_ERR_HIP; - } - } else { - const dim3 block(16, 16, 1); - const dim3 grid((N + 63) / 64, (M + 63) / 64, 1); - if (variant == 1) { - hipLaunchKernelGGL(rdna2_sgemm_f32_lds, grid, block, 0, 0, dC, dA, dB, M, N, K); - } else if (variant == 3) { - hipLaunchKernelGGL(rdna2_sgemm_f32_lds16, grid, block, 0, 0, dC, dA, dB, M, N, K); - } else { - hipLaunchKernelGGL(rdna2_sgemm_f32, grid, block, 0, 0, dC, dA, dB, M, N, K); - } - if (hipGetLastError() != hipSuccess || - hipMemcpy(C, dC, bytesC, hipMemcpyDeviceToHost) != hipSuccess) { - rc = RDNA2_ERR_HIP; - } - } -cleanup: - if (dA) (void)hipFree(dA); - if (dB) (void)hipFree(dB); - if (dC) (void)hipFree(dC); - return rc; -} - -int rdna2_sgemm_f32_host(float* C, const float* A, const float* Bt, int M, int N, int K) { - return rdna2_sgemm_f32_host_v(C, A, Bt, M, N, K, 1); -} - -int rdna2_dot_u8_host(unsigned* out, const unsigned* a, const unsigned* b, int n) { - if (!out || !a || !b || n <= 0) return RDNA2_ERR_ARG; - if (rdna2_device_count() == 0) { - *out = rdna2_dot_u8_ref(a, b, n); - return RDNA2_OK; - } - unsigned* da = nullptr; - unsigned* db = nullptr; - unsigned* dout = nullptr; - int rc = RDNA2_OK; - const size_t bytes = (size_t)n * sizeof(unsigned); - if (hipMalloc(&da, bytes) != hipSuccess || hipMalloc(&db, bytes) != hipSuccess || - hipMalloc(&dout, sizeof(unsigned)) != hipSuccess) { - rc = RDNA2_ERR_HIP; - goto cleanup; - } - if (hipMemcpy(da, a, bytes, hipMemcpyHostToDevice) != hipSuccess || - hipMemcpy(db, b, bytes, hipMemcpyHostToDevice) != hipSuccess || - hipMemset(dout, 0, sizeof(unsigned)) != hipSuccess) { - rc = RDNA2_ERR_HIP; - goto cleanup; - } - { - const int threads = 256; - int blocks = (n + threads - 1) / threads; - if (blocks > 1024) blocks = 1024; - hipLaunchKernelGGL(rdna2_dp4a_dot_u8, dim3(blocks), dim3(threads), 0, 0, dout, da, db, n); - if (hipGetLastError() != hipSuccess || - hipMemcpy(out, dout, sizeof(unsigned), hipMemcpyDeviceToHost) != hipSuccess) { - rc = RDNA2_ERR_HIP; - } - } -cleanup: - if (da) (void)hipFree(da); - if (db) (void)hipFree(db); - if (dout) (void)hipFree(dout); - return rc; -} - -// a and b hold n packed pairs of _Float16 (two per uint32). -int rdna2_dot_f16_host(float* out, const unsigned* a, const unsigned* b, int n) { - if (!out || !a || !b || n <= 0) return RDNA2_ERR_ARG; - if (rdna2_device_count() == 0) { -#if defined(__FLT16_MAX__) - *out = rdna2_dot_f16_ref(a, b, n); - return RDNA2_OK; -#else - return RDNA2_ERR_HIP; // host has no _Float16: reference unavailable -#endif - } - unsigned* da = nullptr; - unsigned* db = nullptr; - float* dout = nullptr; - int rc = RDNA2_OK; - const size_t bytes = (size_t)n * sizeof(unsigned); - if (hipMalloc(&da, bytes) != hipSuccess || hipMalloc(&db, bytes) != hipSuccess || - hipMalloc(&dout, sizeof(float)) != hipSuccess) { - rc = RDNA2_ERR_HIP; - goto cleanup; - } - if (hipMemcpy(da, a, bytes, hipMemcpyHostToDevice) != hipSuccess || - hipMemcpy(db, b, bytes, hipMemcpyHostToDevice) != hipSuccess || - hipMemset(dout, 0, sizeof(float)) != hipSuccess) { - rc = RDNA2_ERR_HIP; - goto cleanup; - } - { - const int threads = 256; - int blocks = (n + threads - 1) / threads; - if (blocks > 1024) blocks = 1024; - hipLaunchKernelGGL(rdna2_dot2_f16, dim3(blocks), dim3(threads), 0, 0, - dout, reinterpret_cast(da), - reinterpret_cast(db), n); - if (hipGetLastError() != hipSuccess || - hipMemcpy(out, dout, sizeof(float), hipMemcpyDeviceToHost) != hipSuccess) { - rc = RDNA2_ERR_HIP; - } - } -cleanup: - if (da) (void)hipFree(da); - if (db) (void)hipFree(db); - if (dout) (void)hipFree(dout); - return rc; -} - -// Inclusive prefix sum over a single wave; n must satisfy 1 <= n <= warpSize. -int rdna2_prefix_host(int* out, const int* in, int n) { - if (!out || !in || n <= 0) return RDNA2_ERR_ARG; - const int ws = rdna2_warp_size(); - if (n > ws) return RDNA2_ERR_ARG; - if (rdna2_device_count() == 0) { - rdna2_prefix_ref(out, in, n); - return RDNA2_OK; - } - int* din = nullptr; - int* dout = nullptr; - int rc = RDNA2_OK; - // dout is warpSize wide: the kernel writes one value per lane. - if (hipMalloc(&din, (size_t)n * sizeof(int)) != hipSuccess || - hipMalloc(&dout, (size_t)ws * sizeof(int)) != hipSuccess) { - rc = RDNA2_ERR_HIP; - goto cleanup; - } - if (hipMemcpy(din, in, (size_t)n * sizeof(int), hipMemcpyHostToDevice) != hipSuccess) { - rc = RDNA2_ERR_HIP; - goto cleanup; - } - hipLaunchKernelGGL(rdna2_dpp_prefix, dim3(1), dim3(ws), 0, 0, dout, din, n); - if (hipGetLastError() != hipSuccess || - hipMemcpy(out, dout, (size_t)n * sizeof(int), hipMemcpyDeviceToHost) != hipSuccess) { - rc = RDNA2_ERR_HIP; - } -cleanup: - if (din) (void)hipFree(din); - if (dout) (void)hipFree(dout); - return rc; -} - -// --------------------------------------------------------------------------- -// Device information, for the test's report line. arch_ok is set to one of the -// RDNA2_ARCH_* codes (1 = gfx1031, 2 = gfx1030, 3 = gfx1201, 4 = gfx1200), or 0 -// when the running device is none of those. -// --------------------------------------------------------------------------- -int rdna2_device_info(char* name, size_t len, int* warp_size, int* arch_ok) { - if (warp_size) *warp_size = 0; - if (arch_ok) *arch_ok = 0; - if (rdna2_device_count() == 0) { - if (name && len) name[0] = '\0'; - return RDNA2_OK; - } - hipDeviceProp_t prop{}; - if (hipGetDeviceProperties(&prop, 0) != hipSuccess) return RDNA2_ERR_HIP; - if (name && len) { - size_t i = 0; - for (; i + 1 < len && prop.name[i]; ++i) name[i] = prop.name[i]; - name[i] = '\0'; - } - if (warp_size) *warp_size = prop.warpSize; - if (arch_ok) { - // "gfx1031", "gfx1201:xnack-" and friends: match the six target digits only. - const char* gcn = prop.gcnArchName; - if (gcn && gcn[0] == 'g' && gcn[1] == 'f' && gcn[2] == 'x') { - if (gcn[3] == '1' && gcn[4] == '0' && gcn[5] == '3') { - if (gcn[6] == '1') *arch_ok = RDNA2_ARCH_GFX1031; - else if (gcn[6] == '0') *arch_ok = RDNA2_ARCH_GFX1030; - } else if (gcn[3] == '1' && gcn[4] == '2' && gcn[5] == '0') { - if (gcn[6] == '1') *arch_ok = RDNA2_ARCH_GFX1201; - else if (gcn[6] == '0') *arch_ok = RDNA2_ARCH_GFX1200; - } - } - } - return RDNA2_OK; -} - -// --------------------------------------------------------------------------- -// Architecture selection -// --------------------------------------------------------------------------- -// The 128x128 / 8x8 shapes (tiled == 4, sgemm variant == 2) were written for -// RDNA4 on the theory that the bigger register file wants a bigger tile. The -// 9070 XT disagrees: measured against the 64x64 8x4 kernel at 1024^3, 2048^2, -// 4096^2 and 2048^2 x K=4096, the 128 tile is 8-33% slower for int8 (82 VGPRs -// per thread plus 17408 B of LDS per block caps residency at 3 blocks, and the -// dp4a issue rate is already limited by LDS bandwidth), and for f32 it only -// wins while the operand working set stays inside the cache. So both RDNA4 -// parts keep the 64x64 8x4 int8 shape and the 64x64 LDS f32 shape; -// tiled == 4 / variant == 2 stay selectable and are still what the test suite -// exercises on every device. Numbers are in README.md. -// -// RDNA4 then overrides that for int8: the matrix instruction (tiled == 5) on the -// same 128x128 tile beat the 64x64 8x4 dp4a kernel by 3.1x at 1024^3, 3.3x at -// 2048^2 and 3.1x at 4096^3 on the 9070 XT (29.4/40.6/40.1 TOPS -> -// 91.3/134.0/125.0), so gfx1200/gfx1201 take tiled == 5 and RDNA2 keeps dp4a. -int rdna2_tiled_for_arch(int arch) { - if (arch == RDNA2_ARCH_GFX1200 || arch == RDNA2_ARCH_GFX1201) { - return 5; // 128x128 tile, one 16x16x16 matrix instruction per k step - } - if (arch == RDNA2_ARCH_GFX1030 || arch == RDNA2_ARCH_GFX1031) { - return 3; // 64x64 tile, 8x4 outputs per thread, LDS staged K chunks - } - return 1; // unknown AMD GPU: 4x4 tiled, global loads -} - -// variant 2 is measurably faster on RDNA4 only while the two 128-row operand -// panels stay cache-resident (K <= 2048 at 2048^2); past that it loses up to 2x -// to the 64x64 shape, so it is opt-in rather than the arch default. -// variant 3 is the 64x64 tile with a 16-deep K step (KSTEP=16, stride 17, four -// floats staged per thread). On the 9070 XT it beat variant 1 at every shape -// tried (1024^3, 1000^3, 2048^2 x K=2048/3072/4096/8192, 2304^2, 3584^2, 4096^3) -// by 3-42%, the margin shrinking as the problem grows, so it is the default on -// the RDNA2 and RDNA4 parts and the generic 64x64 shape is kept for anything -// unrecognised. -int rdna2_sgemm_variant_for_arch(int arch) { - if (arch == RDNA2_ARCH_GFX1030 || arch == RDNA2_ARCH_GFX1031 || - arch == RDNA2_ARCH_GFX1200 || arch == RDNA2_ARCH_GFX1201) { - return 3; // 64x64 tile, 16-deep K step, four floats staged per thread - } - return 1; // unknown AMD GPU: 64x64 LDS staging, 8-deep K step -} - -// The arch-aware entry points: same contract as rdna2_gemm_i8_host / -// rdna2_sgemm_f32_host, but the tile shape follows the device that is actually -// present instead of a fixed default. On a machine with no GPU both fall back to -// the CPU reference exactly like the plain entry points do. -int rdna2_gemm_i8_host_tuned(int* C, const int* A, const int* Bt, int M, int N, int K4) { - int arch = RDNA2_ARCH_UNKNOWN; - (void)rdna2_device_info(nullptr, 0, nullptr, &arch); - return rdna2_gemm_i8_host(C, A, Bt, M, N, K4, rdna2_tiled_for_arch(arch)); -} - -int rdna2_gemm_i8_scale_host_tuned(float* C, const int* A, const int* Bt, int M, int N, int K4, - const float* sa, const float* sb, int zp_b) { - int arch = RDNA2_ARCH_UNKNOWN; - (void)rdna2_device_info(nullptr, 0, nullptr, &arch); - return rdna2_gemm_i8_scale_host(C, A, Bt, M, N, K4, rdna2_tiled_for_arch(arch), sa, sb, zp_b); -} - -int rdna2_sgemm_f32_host_tuned(float* C, const float* A, const float* Bt, int M, int N, int K) { - int arch = RDNA2_ARCH_UNKNOWN; - (void)rdna2_device_info(nullptr, 0, nullptr, &arch); - return rdna2_sgemm_f32_host_v(C, A, Bt, M, N, K, rdna2_sgemm_variant_for_arch(arch)); -} - diff --git a/kernel/rdna2/rdna2_kernels_math.h b/kernel/rdna2/rdna2_kernels_math.h deleted file mode 100644 index 5e44f92b30..0000000000 --- a/kernel/rdna2/rdna2_kernels_math.h +++ /dev/null @@ -1,245 +0,0 @@ -// Kernel arithmetic shared by the device kernels and the host test. -// -// These functions contain no HIP types and no wave intrinsics, so the exact -// same source compiles into the GPU kernels (rdna2_kernels.hip) and runs on the -// host in test_rdna2_host.cpp. That is how the tiling, bounds handling and -// accumulation order of the GEMM/GEMV kernels get executed on a machine with no -// GPU: the test reproduces the kernel's grid and calls the same body. -// -// What this does NOT cover: wave-level behaviour (shuffles, DPP, atomics) and -// anything the compiler does differently for the device. Those need hardware. -#ifndef RDNA2_KERNELS_MATH_H -#define RDNA2_KERNELS_MATH_H - -#include -#include - -#include "rdna2_gemm_ref.h" - -// LDS tiling shape shared by the devices kernels and the host simulation: a -// block of 16x16 threads owns a 64x64 output tile and stages a K chunk of both -// operands in LDS. The row stride is padded by one element so that the lanes of -// a wave, which read rows 4 apart, do not all land on the same LDS banks. -#define RDNA2_TILE_MN 64 -#define RDNA2_LDS_I8_KSTEP 16 // packed words staged per chunk -#define RDNA2_LDS_I8_STRIDE (RDNA2_LDS_I8_KSTEP + 1) -#define RDNA2_LDS_F32_KSTEP 8 -#define RDNA2_LDS_F32_STRIDE (RDNA2_LDS_F32_KSTEP + 1) -// Wider f32 K chunk: 16 floats per tile row, so a chunk touches a full 64-byte -// line of each row instead of half of one, and the K loop pays half as many -// __syncthreads. Two 64x17 tiles are 8704 bytes, the same LDS as the int8 8x4 -// kernel. Select with variant 3 (see README for the measurement). -#define RDNA2_LDS_F32B_KSTEP 16 -#define RDNA2_LDS_F32B_STRIDE (RDNA2_LDS_F32B_KSTEP + 1) - -// RDNA4 (gfx1200 / gfx1201, e.g. RX 9070 XT) shape: same 16x16 block of 256 -// threads and the same K chunk, but a 128x128 output tile and 8x8 outputs per -// thread. gfx12 has 1536 VGPRs per SIMD against 1024 on gfx10/11, so 64 live -// accumulators plus 16 staged operands still leave room for several waves per -// SIMD. Sixteen LDS reads per k step now feed 64 dp4a instead of 8 feeding 16. -#define RDNA2_T4_TILE_MN 128 -#define RDNA2_T4_LDS_I8_KSTEP 16 -#define RDNA2_T4_LDS_I8_STRIDE (RDNA2_T4_LDS_I8_KSTEP + 1) -#define RDNA2_T4_LDS_F32_KSTEP 8 -#define RDNA2_T4_LDS_F32_STRIDE (RDNA2_T4_LDS_F32_KSTEP + 1) -// Column groups of a 128-wide tile: one per 16x16x16 matrix instruction, so a -// wave carries eight accumulators in the tiled = 5 kernel. -#define RDNA2_T4_WMMA_NTILE (RDNA2_T4_TILE_MN / 16) - -// RDNA4 matrix-instruction fragments (tiled = 5). -// V_WMMA_I32_16X16X16_IU8 spreads a 16x16x16 signed product over a wave32 like -// this (measured on gfx1201, 256/256 outputs matched a scalar reference): -// A: lane t, byte e -> A[M = t%16][K = (t/16)*8 + e] -// B: lane t, byte e -> B[K = (t/16)*8 + e][N = t%16] -// D: lane t, slot l -> D[M = (t/16)*8 + l][N = t%16] -// The LDS tile holds packed words (four int8 along K per word), so a lane reads -// two consecutive words of row t%16, i.e. bytes 8*(t/16)..+7 of a 16-byte K -// step. These helpers keep the kernel and the host model on one set of indices; -// the instruction semantics themselves are the hardware's, not this file's. -#define RDNA2_WMMA_I8_KSTEP_WORDS 4 // staged packed words per 16-byte K step -// Staged row a lane takes its fragment from: A's M index and B's N index are -// both lane%16, and both operands stage rows of one tile, so the caller adds the -// wave's 16-row band or the column group it is working on. -RDNA2_HD static inline int rdna2_wmma_i8_row(int lane) { return lane & 15; } -// First of the two staged words a lane reads for K step `step` of a chunk. -RDNA2_HD static inline int rdna2_wmma_i8_frag_word(int lane, int step) { - return RDNA2_WMMA_I8_KSTEP_WORDS * step + 2 * (lane >> 4); -} -// Output row (slot l) and column a lane's accumulators hold in a 16x16 result. -RDNA2_HD static inline int rdna2_wmma_i8_acc_row(int lane, int slot) { - return 8 * (lane >> 4) + slot; -} -RDNA2_HD static inline int rdna2_wmma_i8_acc_col(int lane) { return lane & 15; } - -// Target codes reported by rdna2_device_info(..., arch_ok) and consumed by the -// rdna2_*_for_arch selectors. 0 means "an AMD GPU from neither family"; the -// launchers still run on it with the generic 64x64 / 4x4 shapes. -#define RDNA2_ARCH_UNKNOWN 0 -#define RDNA2_ARCH_GFX1031 1 // RDNA2, Navi 22 (RX 6700 XT / 6750 XT) -#define RDNA2_ARCH_GFX1030 2 // RDNA2, Navi 21 (RX 6800 / 6900) -#define RDNA2_ARCH_GFX1201 3 // RDNA4, Navi 48 (RX 9070 XT / 9070) -#define RDNA2_ARCH_GFX1200 4 // RDNA4, Navi 44 (RX 9060 XT) - -// One row of the dp4a int8 GEMV: y[row] = sum over k4 of dot4(A_row[k], x[k]). -RDNA2_HD static inline int32_t rdna2_gemv_row_i8(const int32_t* a_row, const int32_t* x, - int k4) { - int32_t acc = 0; - for (int j = 0; j < k4; ++j) acc = rdna2_dot4_i8_impl(a_row[j], x[j], acc); - return acc; -} - -// One output element of the int8 GEMM. -RDNA2_HD static inline int32_t rdna2_gemm_cell_i8(const int32_t* a_row, const int32_t* b_row, - int K4) { - int32_t acc = 0; - for (int k = 0; k < K4; ++k) acc = rdna2_dot4_i8_impl(a_row[k], b_row[k], acc); - return acc; -} - -// One thread's 4x4 output block of the tiled int8 GEMM, starting at (m0, n0). -// Rows/columns past M/N read 0 so the accumulation stays exact at the edges; -// the caller only stores the in-range elements. -RDNA2_HD static inline void rdna2_gemm_tile_i8(int32_t acc[4][4], const int32_t* A, - const int32_t* Bt, int M, int N, int K4, - int m0, int n0) { - for (int i = 0; i < 4; ++i) - for (int j = 0; j < 4; ++j) acc[i][j] = 0; - for (int k = 0; k < K4; ++k) { - int32_t a[4]; - int32_t b[4]; - for (int i = 0; i < 4; ++i) a[i] = (m0 + i < M) ? A[(size_t)(m0 + i) * K4 + k] : 0; - for (int j = 0; j < 4; ++j) b[j] = (n0 + j < N) ? Bt[(size_t)(n0 + j) * K4 + k] : 0; - for (int i = 0; i < 4; ++i) - for (int j = 0; j < 4; ++j) acc[i][j] = rdna2_dot4_i8_impl(a[i], b[j], acc[i][j]); - } -} - -// One thread's 4x4 output block of the f32 GEMM, same blocking as the int8 one. -RDNA2_HD static inline void rdna2_sgemm_tile_f32(float acc[4][4], const float* A, - const float* Bt, int M, int N, int K, - int m0, int n0) { - for (int i = 0; i < 4; ++i) - for (int j = 0; j < 4; ++j) acc[i][j] = 0.0f; - for (int k = 0; k < K; ++k) { - float a[4]; - float b[4]; - for (int i = 0; i < 4; ++i) a[i] = (m0 + i < M) ? A[(size_t)(m0 + i) * K + k] : 0.0f; - for (int j = 0; j < 4; ++j) b[j] = (n0 + j < N) ? Bt[(size_t)(n0 + j) * K + k] : 0.0f; - for (int i = 0; i < 4; ++i) - for (int j = 0; j < 4; ++j) acc[i][j] = __builtin_fmaf(a[i], b[j], acc[i][j]); - } -} - -// --- LDS-staged variants --------------------------------------------------- -// The tile pointer types are template parameters on purpose: on the device they -// deduce to __shared__-space pointers (so the compiler keeps the accesses as -// ds_read/ds_write instead of flat ones), on the host they are plain pointers. -// The tile is row-major with `ld` words per row holding `kwords` K columns; -// rows past the matrix edge are already zero, so no bounds checks are needed. -// acc is accumulated into, not zeroed, so a K loop can stream tile chunks in. -template -RDNA2_HD static inline void rdna2_gemm_lds_block_i8(int32_t acc[4][4], AT a_tile, BT b_tile, - int ld, int lm, int ln, int kwords) { - for (int kk = 0; kk < kwords; ++kk) { - int32_t a[4]; - int32_t b[4]; - for (int i = 0; i < 4; ++i) a[i] = a_tile[(lm + i) * ld + kk]; - for (int j = 0; j < 4; ++j) b[j] = b_tile[(ln + j) * ld + kk]; - for (int i = 0; i < 4; ++i) - for (int j = 0; j < 4; ++j) acc[i][j] = rdna2_dot4_i8_impl(a[i], b[j], acc[i][j]); - } -} - -// 8x4 register blocking: 12 LDS reads feed 32 dp4a instead of 8 feeding 16, so -// the LDS traffic per multiply-accumulate drops by a quarter at the same math -// count. Register cost is 32 accumulators plus the 12 staged operands. -template -RDNA2_HD static inline void rdna2_gemm_lds_block8_i8(int32_t acc[8][4], AT a_tile, BT b_tile, - int ld, int lm, int ln, int kwords) { - for (int kk = 0; kk < kwords; ++kk) { - int32_t a[8]; - int32_t b[4]; - for (int i = 0; i < 8; ++i) a[i] = a_tile[(lm + i) * ld + kk]; - for (int j = 0; j < 4; ++j) b[j] = b_tile[(ln + j) * ld + kk]; - for (int i = 0; i < 8; ++i) - for (int j = 0; j < 4; ++j) acc[i][j] = rdna2_dot4_i8_impl(a[i], b[j], acc[i][j]); - } -} - -// 8x8 register blocking for the 128x128 tile. Sixteen LDS reads per k step feed -// 64 dp4a, i.e. 4 multiplies per LDS read against 2.67 for 8x4 and 2 for 4x4. -template -RDNA2_HD static inline void rdna2_gemm_lds_block8x8_i8(int32_t acc[8][8], AT a_tile, BT b_tile, - int ld, int lm, int ln, int kwords) { - for (int kk = 0; kk < kwords; ++kk) { - int32_t a[8]; - int32_t b[8]; - for (int i = 0; i < 8; ++i) a[i] = a_tile[(lm + i) * ld + kk]; - for (int j = 0; j < 8; ++j) b[j] = b_tile[(ln + j) * ld + kk]; - for (int i = 0; i < 8; ++i) - for (int j = 0; j < 8; ++j) acc[i][j] = rdna2_dot4_i8_impl(a[i], b[j], acc[i][j]); - } -} - -template -RDNA2_HD static inline void rdna2_sgemm_lds_block8x8_f32(float acc[8][8], AT a_tile, BT b_tile, - int ld, int lm, int ln, int kwords) { - for (int kk = 0; kk < kwords; ++kk) { - float a[8]; - float b[8]; - for (int i = 0; i < 8; ++i) a[i] = a_tile[(lm + i) * ld + kk]; - for (int j = 0; j < 8; ++j) b[j] = b_tile[(ln + j) * ld + kk]; - for (int i = 0; i < 8; ++i) - for (int j = 0; j < 8; ++j) acc[i][j] = __builtin_fmaf(a[i], b[j], acc[i][j]); - } -} - -template -RDNA2_HD static inline void rdna2_sgemm_lds_block_f32(float acc[4][4], AT a_tile, BT b_tile, - int ld, int lm, int ln, int kwords) { - for (int kk = 0; kk < kwords; ++kk) { - float a[4]; - float b[4]; - for (int i = 0; i < 4; ++i) a[i] = a_tile[(lm + i) * ld + kk]; - for (int j = 0; j < 4; ++j) b[j] = b_tile[(ln + j) * ld + kk]; - for (int i = 0; i < 4; ++i) - for (int j = 0; j < 4; ++j) acc[i][j] = __builtin_fmaf(a[i], b[j], acc[i][j]); - } -} - -// One lane's contribution to a row when a whole wave cooperates on that row -// (lanes stride the row, so consecutive lanes read consecutive words). The -// caller reduces the result across the wave. -RDNA2_HD static inline int32_t rdna2_gemv_lane_partial_i8(const int32_t* a_row, const int32_t* x, - int k4, int lane, int wave_size) { - int32_t acc = 0; - for (int k = lane; k < k4; k += wave_size) acc = rdna2_dot4_i8_impl(a_row[k], x[k], acc); - return acc; -} - -// One row of the f32 GEMV: y[row] = sum over k of A_row[k] * x[k]. Four -// independent accumulators, because a single fma chain would leave the fma -// pipes waiting on the previous result; the final sum fixes the evaluation -// order, so the kernel and the host simulation agree exactly. -RDNA2_HD static inline float rdna2_sgemv_row_f32(const float* a_row, const float* x, int K) { - float acc0 = 0.0f, acc1 = 0.0f, acc2 = 0.0f, acc3 = 0.0f; - int k = 0; - for (; k + 4 <= K; k += 4) { - acc0 = __builtin_fmaf(a_row[k + 0], x[k + 0], acc0); - acc1 = __builtin_fmaf(a_row[k + 1], x[k + 1], acc1); - acc2 = __builtin_fmaf(a_row[k + 2], x[k + 2], acc2); - acc3 = __builtin_fmaf(a_row[k + 3], x[k + 3], acc3); - } - for (; k < K; ++k) acc0 = __builtin_fmaf(a_row[k], x[k], acc0); - return (acc0 + acc1) + (acc2 + acc3); -} - -// One lane's contribution to a row when a wave cooperates on that row. -RDNA2_HD static inline float rdna2_sgemv_lane_partial_f32(const float* a_row, const float* x, - int K, int lane, int wave_size) { - float acc = 0.0f; - for (int k = lane; k < K; k += wave_size) acc = __builtin_fmaf(a_row[k], x[k], acc); - return acc; -} - -#endif // RDNA2_KERNELS_MATH_H diff --git a/kernel/rdna2/rdna2_oblas.c b/kernel/rdna2/rdna2_oblas.c index 3114876b7a..0a6becb7a9 100644 --- a/kernel/rdna2/rdna2_oblas.c +++ b/kernel/rdna2/rdna2_oblas.c @@ -1,59 +1,529 @@ +/* Must match common.h: on Windows 64-bit OpenBLAS uses long long. */ +#if defined(_WIN64) +typedef long long BLASLONG; +#else typedef long BLASLONG; +#endif typedef float FLOAT; +#include +#include +#include +#include + #ifdef __cplusplus extern "C" { #endif -int rdna2_sgemm_f32_host_tuned(float* C, const float* A, const float* Bt, int M, int N, int K); -int rdna2_sgemm_f32_host(float* C, const float* A, const float* Bt, int M, int N, int K); - -int sgemm_gfx1201(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT *a, BLASLONG lda, - FLOAT *b, BLASLONG ldb, FLOAT beta, FLOAT *c, BLASLONG ldc); -int sgemm_nn(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT *a, BLASLONG lda, - FLOAT *b, BLASLONG ldb, FLOAT beta, FLOAT *c, BLASLONG ldc); -int sgemm_kernel(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc); +int rdna2_sgemm_f32_host_tuned(float *C, const float *A, const float *Bt, + int M, int N, int K); #ifdef __cplusplus } #endif -#define CNAME sgemm_gfx1201 - -int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT *a, BLASLONG lda, - FLOAT *b, BLASLONG ldb, FLOAT beta, FLOAT *c, BLASLONG ldc) { - /* Fallback implementation */ - for (BLASLONG j = 0; j < n; j++) { - for (BLASLONG i = 0; i < m; i++) { - FLOAT temp = 0.0f; - if (beta == 0.0f) { - for (BLASLONG p = 0; p < k; p++) { - temp += a[i + p * lda] * b[p + j * ldb]; - } - c[i + j * ldc] = alpha * temp; - } else if (beta == 1.0f) { - for (BLASLONG p = 0; p < k; p++) { - temp += a[i + p * lda] * b[p + j * ldb]; - } - c[i + j * ldc] += alpha * temp; - } else { - for (BLASLONG p = 0; p < k; p++) { - temp += a[i + p * lda] * b[p + j * ldb]; - } - c[i + j * ldc] = beta * c[i + j * ldc] + alpha * temp; - } +/* Bridges between OpenBLAS' level-3 entry points and the hand-written + * RDNA2/RDNA4 HIP kernels in gpu/rdna2/rdna2_kernels.hip. + * + * The HIP kernel computes, in row-major terms, + * + * C[i][j] = sum_p A[i][p] * Bt[j][p] A is M x K, Bt is N x K + * + * so its row-major M x N result, read with row stride N, is exactly a + * column-major N x M matrix. Every routine below calls the kernel with + * (M, N) = (n, m), which makes the kernel's output the column-major m x n + * result OpenBLAS expects. Column-major B (k x n, ldb == k) is already the + * kernel's N x K left operand; a column-major A (m x k) is contiguous in m and + * is transposed once into a temporary. + * + * All four bridges funnel into rdna2_gemm_cm_core(), so every level-3 routine + * runs the same GEMM path that the standalone harness verifies against the CPU. + */ + +/* Below this the CPU path wins, because the host entry point allocates, copies + * and (for column-major A) transposes on every call. Measured single-call + * crossover is around 1500^3; see gpu/rdna2/bench_full.txt. */ +#define RDNA2_GPU_MIN_MNK 2000000000.0 + +/* Largest dimension the host launchers accept. */ +#define RDNA2_MAX_DIM 65535 + +/* Block size of the blocked triangular solve. */ +#define RDNA2_TRSM_NB 128 + +/* Read on every call (not cached) so a process can compare both paths, and so a + * long-running caller can switch the GPU off without restarting. */ +static int rdna2_gpu_enabled(void) { + const char *e = getenv("OPENBLAS_RDNA2_GPU"); + return !(e && (e[0] == '0' || e[0] == 'n' || e[0] == 'N')); +} + +static double rdna2_min_mnk(void) { + const char *e = getenv("OPENBLAS_RDNA2_GPU_MIN_MNK"); + return (e && e[0]) ? atof(e) : RDNA2_GPU_MIN_MNK; +} + +static int rdna2_debug(void) { + const char *e = getenv("OPENBLAS_RDNA2_GPU_DEBUG"); + return e && e[0] == '1'; +} + +static long rdna2_ms(clock_t since) { + return (long)((clock() - since) * 1000.0 / (double)CLOCKS_PER_SEC); +} + +/* Column-major -> row-major copy, 64x64 tiled: + * + * dst[i*ldd + p] = src[i + p*lds] i in [0,rows), p in [0,cols) + * + * With ldd == cols this is a plain transpose. The naive version stores with a + * stride of cols*4 bytes, one cache line per element; at 2048^3 that alone cost + * more than the kernel. Tiled, both the store block (64 rows of 256 B) and the + * matching load block stay in cache. + */ +static void rdna2_cm_to_rm(const float *src, BLASLONG lds, float *dst, + BLASLONG ldd, BLASLONG rows, BLASLONG cols) { + const BLASLONG T = 64; + BLASLONG i0, p0; + + for (p0 = 0; p0 < cols; p0 += T) { + const BLASLONG p1 = (p0 + T < cols) ? p0 + T : cols; + + for (i0 = 0; i0 < rows; i0 += T) { + const BLASLONG i1 = (i0 + T < rows) ? i0 + T : rows; + BLASLONG p, i; + + for (p = p0; p < p1; ++p) { + const float *s = src + (size_t)p * (size_t)lds + (size_t)i0; + float *d = dst + (size_t)i0 * (size_t)ldd + (size_t)p; + + for (i = i0; i < i1; ++i) { + *d = *s++; + d += ldd; } + } } + } +} + +/* C(m x n) = alpha * A(m x k) * B(k x n) + beta * C, every operand column-major + * with strides lda, ldb and ldc. + * + * Returns 1 when C was produced on the GPU, 0 when the caller must fall back to + * the CPU path. There is deliberately no size gate here: each bridge gates once + * for the whole problem, and the blocked triangular solve issues sub-block GEMMs + * that are individually small but must still run. + */ +static int rdna2_gemm_cm_core(BLASLONG m, BLASLONG n, BLASLONG k, + double alpha, const float *a, BLASLONG lda, + const float *b, BLASLONG ldb, + double beta, float *c, BLASLONG ldc) { + const float *left = b; /* kernel's N x K (== n x k) row-major operand */ + const float *right = a; /* kernel's M x K (== m x k) row-major operand */ + float *bt = NULL, *at = NULL, *tmp = NULL; + int direct, ok; + + if (ldb != k) { + BLASLONG i, j; + + bt = (float *)malloc((size_t)n * (size_t)k * sizeof(float)); + if (bt == NULL) return 0; + /* The kernel's left operand is B^T stored row-major n x k, so + * bt[j*k + p] = B(p, j) = b[p + j*ldb]. That is each column of B (its first + * k entries) laid out as a contiguous row, i.e. a plain copy per column and + * not the faithful column-major -> row-major copy used for the other + * operand. When ldb == k the same bytes are produced, which is why B can be + * passed through untouched in that case. */ + for (j = 0; j < n; ++j) { + const float *s = b + (size_t)j * (size_t)ldb; + float *d = bt + (size_t)j * (size_t)k; + + for (i = 0; i < k; ++i) d[i] = s[i]; + } + left = bt; + } + + /* The kernel's second operand must be A stored row-major with row stride k. + * A column-major m x k matrix read with stride k is its transpose, not + * itself, so this copy is required even when lda == k. (Only the left operand + * gets that shortcut: a column-major k x n matrix with ldb == k already is the + * row-major n x k matrix the kernel wants.) */ + at = (float *)malloc((size_t)m * (size_t)k * sizeof(float)); + if (at == NULL) { + free(bt); return 0; + } + rdna2_cm_to_rm(a, lda, at, k, m, k); + right = at; + + /* The kernel writes its n x m row-major result straight into a column-major + * m x n matrix only when ldc == m and the output aliases neither input. + * TRMM and TRSM update b in place, so they always take the temporary. */ + direct = (ldc == m) && (c != b) && (c != a) && (alpha == 1.0) && (beta == 0.0); + + if (!direct) { + tmp = (float *)malloc((size_t)m * (size_t)n * sizeof(float)); + if (tmp == NULL) { + free(bt); + free(at); + return 0; + } + } + + ok = rdna2_sgemm_f32_host_tuned(direct ? c : tmp, left, right, + (int)n, (int)m, (int)k) == 0; + + if (ok && !direct) { + BLASLONG i, j; + + for (j = 0; j < n; ++j) { + for (i = 0; i < m; ++i) { + const float d = tmp[(size_t)j * (size_t)m + (size_t)i]; + float *cp = c + (size_t)i + (size_t)j * (size_t)ldc; + + if (alpha == 1.0 && beta == 0.0) + *cp = d; + else if (beta == 0.0) + *cp = (float)(alpha * (double)d); + else + *cp = (float)(alpha * (double)d + beta * (double)*cp); + } + } + } + + free(bt); + free(at); + free(tmp); + + return ok; +} + +/* Returns 1 when the product was computed on the GPU, 0 when the caller should + * carry on with the generic CPU path. */ +int rdna2_oblas_sgemm_cm(BLASLONG m, BLASLONG n, BLASLONG k, double alpha, + const float *a, BLASLONG lda, const float *b, + BLASLONG ldb, double beta, float *c, BLASLONG ldc, + int transa, int transb) { + int handled; + clock_t t0; + + if (!rdna2_gpu_enabled()) return 0; + /* Only the untransposed layouts are wired up; the core handles any leading + * dimension, so lda/ldb/ldc are not constrained here. Transposed operands + * need a different gather and stay on the CPU path. */ + if (transa != 0 || transb != 0) return 0; + if (m <= 0 || n <= 0 || k <= 0) return 0; + if (m > RDNA2_MAX_DIM || n > RDNA2_MAX_DIM || k > RDNA2_MAX_DIM) return 0; + if ((double)m * (double)n * (double)k < rdna2_min_mnk()) return 0; + + t0 = clock(); + handled = rdna2_gemm_cm_core(m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); + + if (rdna2_debug()) + fprintf(stderr, "[rdna2] sgemm %ldx%ldx%ld handled=%d %ldms\n", + (long)m, (long)n, (long)k, handled, rdna2_ms(t0)); + + return handled; } -int sgemm_nn(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT *a, BLASLONG lda, - FLOAT *b, BLASLONG ldb, FLOAT beta, FLOAT *c, BLASLONG ldc) { - return CNAME(m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); +/* SSYMM: C := alpha*A*B + beta*C (side=left) or alpha*B*A + beta*C (side=right), + * where the m x m (left) or n x n (right) matrix A is symmetric and only the + * uplo triangle is stored. + * + * A is expanded into a dense symmetric buffer, after which the operation is an + * ordinary GEMM. The mirror image is always read from the stored triangle, so + * the triangle the caller did not fill in is never touched. + * + * side: 0 = left, 1 = right. uplo: 0 = upper, 1 = lower. + */ +int rdna2_oblas_ssymm_cm(BLASLONG m, BLASLONG n, double alpha, + const float *a, BLASLONG lda, + const float *b, BLASLONG ldb, + double beta, float *c, BLASLONG ldc, + int side, int uplo) { + BLASLONG kdim, i, j; + float *sym; + int ok; + clock_t t0; + + if (!rdna2_gpu_enabled()) return 0; + if (m <= 0 || n <= 0) return 0; + if (m > RDNA2_MAX_DIM || n > RDNA2_MAX_DIM) return 0; + + kdim = side ? n : m; + if (kdim > RDNA2_MAX_DIM) return 0; + if ((double)m * (double)n * (double)kdim < rdna2_min_mnk()) return 0; + if (lda < kdim || ldb < m || ldc < m) return 0; + + sym = (float *)malloc((size_t)kdim * (size_t)kdim * sizeof(float)); + if (sym == NULL) return 0; + + for (j = 0; j < kdim; ++j) { + for (i = 0; i < kdim; ++i) { + const int stored = (uplo == 0) ? (i <= j) : (i >= j); + + sym[(size_t)i + (size_t)j * (size_t)kdim] = + stored ? a[(size_t)i + (size_t)j * (size_t)lda] + : a[(size_t)j + (size_t)i * (size_t)lda]; + } + } + + t0 = clock(); + + if (!side) + ok = rdna2_gemm_cm_core(m, n, m, alpha, sym, kdim, b, ldb, beta, c, ldc); + else + ok = rdna2_gemm_cm_core(m, n, n, alpha, b, ldb, sym, kdim, beta, c, ldc); + + if (rdna2_debug()) + fprintf(stderr, "[rdna2] ssymm %ldx%ld side=%d uplo=%d handled=%d %ldms\n", + (long)m, (long)n, side, uplo, ok, rdna2_ms(t0)); + + free(sym); + return ok; } -int sgemm_kernel(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) { - /* Try to use our tuned kernel if possible */ - /* For now fallback */ - return CNAME(m, n, k, alpha, a, k, b, n, 0.0f, c, ldc); +/* TRMM: B := alpha*op(A)*B (side=left) or alpha*B*op(A) (side=right), with A + * triangular and m x m for side=left, n x n for side=right. + * + * OpenBLAS' TRMM has no separate output, so B is both the input and the result + * (column-major, stride ldb). A is expanded into a dense buffer that is zero + * outside the stored triangle, which makes the product identical to the + * triangular one while letting it go through the GEMM core. TRMM has no beta: + * the result always overwrites B. + * + * trans & 1 selects op(A) = A^T. unit_diag != 0 means A has an implicit unit + * diagonal. + */ +int rdna2_oblas_strmm_cm(BLASLONG m, BLASLONG n, double alpha, + const float *a, BLASLONG lda, + float *b, BLASLONG ldb, + int side, int uplo, int trans, int unit_diag) { + BLASLONG kdim, i, j; + const int tr = trans & 1; + float *d; + int ok; + clock_t t0; + + if (!rdna2_gpu_enabled()) return 0; + if (m <= 0 || n <= 0) return 0; + if (m > RDNA2_MAX_DIM || n > RDNA2_MAX_DIM) return 0; + + kdim = side ? n : m; + if (kdim > RDNA2_MAX_DIM) return 0; + if ((double)m * (double)n * (double)kdim < rdna2_min_mnk()) return 0; + if (lda < kdim || ldb < m) return 0; + + d = (float *)malloc((size_t)kdim * (size_t)kdim * sizeof(float)); + if (d == NULL) return 0; + + for (j = 0; j < kdim; ++j) { + for (i = 0; i < kdim; ++i) { + /* Source index in the stored A. */ + const BLASLONG si = tr ? j : i; + const BLASLONG sj = tr ? i : j; + const int stored = (uplo == 0) ? (si <= sj) : (si >= sj); + float v = 0.0f; + + if (stored) v = a[(size_t)si + (size_t)sj * (size_t)lda]; + if (si == sj && unit_diag) v = 1.0f; + + d[(size_t)i + (size_t)j * (size_t)kdim] = v; + } + } + + t0 = clock(); + + if (!side) + ok = rdna2_gemm_cm_core(m, n, m, alpha, d, kdim, b, ldb, 0.0, b, ldb); + else + ok = rdna2_gemm_cm_core(m, n, n, alpha, b, ldb, d, kdim, 0.0, b, ldb); + + if (rdna2_debug()) + fprintf(stderr, "[rdna2] strmm %ldx%ld side=%d uplo=%d trans=%d unit=%d handled=%d %ldms\n", + (long)m, (long)n, side, uplo, tr, unit_diag, ok, rdna2_ms(t0)); + + free(d); + return ok; +} + +/* Solve T * X = Y in place: T is the dense k x k triangular factor with stride + * ldt, X is k x n column-major with stride ldx, and Y is already in X. + * uplo: 0 = upper, 1 = lower. Returns 1 on success. + * + * Blocked substitution: the diagonal block is solved on the host, and the + * trailing rank-nb update is a GEMM, which is what puts the work on the GPU. + */ +static int rdna2_trsm_left(BLASLONG k, BLASLONG n, const float *t, BLASLONG ldt, + float *x, BLASLONG ldx, int uplo) { + if (uplo == 0) { + /* Upper: back substitution, blocks from the bottom up. */ + BLASLONG p1; + + for (p1 = k; p1 > 0; ) { + const BLASLONG p0 = (p1 > RDNA2_TRSM_NB) ? p1 - RDNA2_TRSM_NB : 0; + BLASLONG i, j, q; + + for (j = 0; j < n; ++j) { + float *xj = x + (size_t)j * (size_t)ldx; + + for (i = p1 - 1; i >= p0; --i) { + double s = (double)xj[i]; + + for (q = i + 1; q < p1; ++q) + s -= (double)t[(size_t)i + (size_t)q * (size_t)ldt] * (double)xj[q]; + + xj[i] = (float)(s / (double)t[(size_t)i + (size_t)i * (size_t)ldt]); + } + } + + /* X[0:p0, :] -= T[0:p0, p0:p1] * X[p0:p1, :] */ + if (p0 > 0 && + !rdna2_gemm_cm_core(p0, n, p1 - p0, -1.0, + t + (size_t)p0 * (size_t)ldt, ldt, + x + p0, ldx, 1.0, x, ldx)) + return 0; + + p1 = p0; + } + } else { + /* Lower: forward substitution, blocks from the top down. */ + BLASLONG p0; + + for (p0 = 0; p0 < k; ) { + const BLASLONG p1 = (p0 + RDNA2_TRSM_NB < k) ? p0 + RDNA2_TRSM_NB : k; + BLASLONG i, j, q; + + for (j = 0; j < n; ++j) { + float *xj = x + (size_t)j * (size_t)ldx; + + for (i = p0; i < p1; ++i) { + double s = (double)xj[i]; + + for (q = p0; q < i; ++q) + s -= (double)t[(size_t)i + (size_t)q * (size_t)ldt] * (double)xj[q]; + + xj[i] = (float)(s / (double)t[(size_t)i + (size_t)i * (size_t)ldt]); + } + } + + /* X[p1:k, :] -= T[p1:k, p0:p1] * X[p0:p1, :] */ + if (p1 < k && + !rdna2_gemm_cm_core(k - p1, n, p1 - p0, -1.0, + t + p1 + (size_t)p0 * (size_t)ldt, ldt, + x + p0, ldx, 1.0, x + p1, ldx)) + return 0; + + p0 = p1; + } + } + + return 1; +} + +/* TRSM: solve op(A)*X = alpha*B (side=left) or X*op(A) = alpha*B (side=right) + * in place in b, with A triangular. + * + * side=left solves directly. side=right is solved as the transposed system + * op(A)^T * X^T = alpha*B^T, so both cases share one left-looking blocked + * solver; the two transposes of B cost a pass over b each. + * + * The solve runs in a scratch copy and b is only written once the whole solve + * succeeded, so a late GPU failure leaves b intact for the CPU fallback. + */ +int rdna2_oblas_strsm_cm(BLASLONG m, BLASLONG n, double alpha, + const float *a, BLASLONG lda, + float *b, BLASLONG ldb, + int side, int uplo, int trans, int unit_diag) { + BLASLONG kdim, i, j; + const int tr = trans & 1; + int tri; + float *d = NULL, *x = NULL; + int ok = 0; + clock_t t0; + + if (!rdna2_gpu_enabled()) return 0; + if (m <= 0 || n <= 0) return 0; + if (m > RDNA2_MAX_DIM || n > RDNA2_MAX_DIM) return 0; + + kdim = side ? n : m; + if (kdim > RDNA2_MAX_DIM) return 0; + if ((double)m * (double)n * (double)kdim < rdna2_min_mnk()) return 0; + if (lda < kdim || ldb < m) return 0; + + /* Triangle of the factor the solver actually uses: op(A) for side=left and + * op(A)^T for side=right. Transposing flips the triangle, so side=left is + * !uplo when trans is set, and side=right is its complement. */ + tri = side ? (tr ? uplo : !uplo) : (tr ? !uplo : uplo); + + d = (float *)malloc((size_t)kdim * (size_t)kdim * sizeof(float)); + if (d == NULL) return 0; + + for (j = 0; j < kdim; ++j) { + for (i = 0; i < kdim; ++i) { + /* Dense copy of T: op(A) for side=left, op(A)^T for side=right. */ + BLASLONG si, sj; + int stored; + float v = 0.0f; + + if (!side) { + si = tr ? j : i; + sj = tr ? i : j; + } else { + si = tr ? i : j; + sj = tr ? j : i; + } + + stored = (uplo == 0) ? (si <= sj) : (si >= sj); + if (stored) v = a[(size_t)si + (size_t)sj * (size_t)lda]; + if (si == sj && unit_diag) v = 1.0f; + + d[(size_t)i + (size_t)j * (size_t)kdim] = v; + } + } + + t0 = clock(); + + if (!side) { + x = (float *)malloc((size_t)m * (size_t)n * sizeof(float)); + if (x != NULL) { + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) + x[(size_t)i + (size_t)j * (size_t)m] = + (float)(alpha * (double)b[(size_t)i + (size_t)j * (size_t)ldb]); + + ok = rdna2_trsm_left(m, n, d, kdim, x, m, tri); + + if (ok) + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) + b[(size_t)i + (size_t)j * (size_t)ldb] = + x[(size_t)i + (size_t)j * (size_t)m]; + } + } else { + /* op(A)^T * Y = alpha * B^T, Y is n x m. */ + x = (float *)malloc((size_t)n * (size_t)m * sizeof(float)); + if (x != NULL) { + for (j = 0; j < m; ++j) + for (i = 0; i < n; ++i) + x[(size_t)i + (size_t)j * (size_t)n] = + (float)(alpha * (double)b[(size_t)j + (size_t)i * (size_t)ldb]); + + ok = rdna2_trsm_left(n, m, d, kdim, x, n, tri); + + if (ok) + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) + b[(size_t)i + (size_t)j * (size_t)ldb] = + x[(size_t)j + (size_t)i * (size_t)n]; + } + } + + if (rdna2_debug()) + fprintf(stderr, "[rdna2] strsm %ldx%ld side=%d uplo=%d trans=%d unit=%d handled=%d %ldms\n", + (long)m, (long)n, side, uplo, tr, unit_diag, ok, rdna2_ms(t0)); + + free(d); + free(x); + return ok; } diff --git a/kernel/x86_64/KERNEL.GFX1030 b/kernel/x86_64/KERNEL.GFX1030 new file mode 100644 index 0000000000..801fc8331d --- /dev/null +++ b/kernel/x86_64/KERNEL.GFX1030 @@ -0,0 +1,115 @@ +# RDNA2 / RDNA4 (gfx1030, gfx1031, gfx1200, gfx1201) host-side kernels. +# +# The GPU does the GEMM math in kernel/rdna2; these definitions only cover the +# host packing/copy routines OpenBLAS still builds. They are the generic C +# kernels on purpose: the x86 assembly L3 kernels assume their own tile shapes +# and offsets, so pairing them with a GPU core name produces packing that does +# not match them. Unroll sizes here must stay in step with the FORCE_GFX blocks +# in getarch.c and the RDNA2 block in param.h. +# +# S/D: 4x4 generic register-tiled GEMM, width-4 packing. +# C/Z: 2x2 generic complex GEMM, width-2 packing. + +SSCALKERNEL = sscal.c +DSCALKERNEL = dscal.c +CSCALKERNEL = cscal.c +ZSCALKERNEL = zscal.c + +SGEMVNKERNEL = sgemv_n_4.c +SGEMVTKERNEL = sgemv_t_4.c + +DGEMVNKERNEL = dgemv_n_4.c +DGEMVTKERNEL = dgemv_t_4.c + +ZGEMVNKERNEL = zgemv_n_4.c +ZGEMVTKERNEL = zgemv_t_4.c + +CGEMVNKERNEL = cgemv_n_4.c +CGEMVTKERNEL = cgemv_t_4.c + +SSYMV_L_KERNEL = ssymv_L.c +SSYMV_U_KERNEL = ssymv_U.c +DSYMV_L_KERNEL = dsymv_L.c +DSYMV_U_KERNEL = dsymv_U.c + +SDOTKERNEL = sdot.c +DSDOTKERNEL = sdot.c +DDOTKERNEL = ddot.c +CDOTKERNEL = cdot.c +ZDOTKERNEL = zdot.c + +SAXPYKERNEL = saxpy.c +DAXPYKERNEL = daxpy.c +CAXPYKERNEL = caxpy.c +ZAXPYKERNEL = zaxpy.c + +STRMMKERNEL = ../generic/trmmkernel_4x4.c +SGEMMKERNEL = ../generic/gemmkernel_4x4.c +SGEMMINCOPY = ../generic/gemm_ncopy_4.c +SGEMMITCOPY = ../generic/gemm_tcopy_4.c +SGEMMONCOPY = ../generic/gemm_ncopy_4.c +SGEMMOTCOPY = ../generic/gemm_tcopy_4.c +SGEMMINCOPYOBJ = sgemm_incopy$(TSUFFIX).$(SUFFIX) +SGEMMITCOPYOBJ = sgemm_itcopy$(TSUFFIX).$(SUFFIX) +SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX) +SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX) + +DTRMMKERNEL = ../generic/trmmkernel_4x4.c +DGEMMKERNEL = ../generic/gemmkernel_4x4.c +DGEMMINCOPY = ../generic/gemm_ncopy_4.c +DGEMMITCOPY = ../generic/gemm_tcopy_4.c +DGEMMONCOPY = ../generic/gemm_ncopy_4.c +DGEMMOTCOPY = ../generic/gemm_tcopy_4.c +DGEMMINCOPYOBJ = dgemm_incopy$(TSUFFIX).$(SUFFIX) +DGEMMITCOPYOBJ = dgemm_itcopy$(TSUFFIX).$(SUFFIX) +DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX) +DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX) + +CTRMMKERNEL = ../generic/ztrmmkernel_2x2.c +CGEMMKERNEL = ../generic/zgemmkernel_2x2.c +CGEMMINCOPY = ../generic/zgemm_ncopy_2.c +CGEMMITCOPY = ../generic/zgemm_tcopy_2.c +CGEMMONCOPY = ../generic/zgemm_ncopy_2.c +CGEMMOTCOPY = ../generic/zgemm_tcopy_2.c +CGEMMINCOPYOBJ = cgemm_incopy$(TSUFFIX).$(SUFFIX) +CGEMMITCOPYOBJ = cgemm_itcopy$(TSUFFIX).$(SUFFIX) +CGEMMONCOPYOBJ = cgemm_oncopy$(TSUFFIX).$(SUFFIX) +CGEMMOTCOPYOBJ = cgemm_otcopy$(TSUFFIX).$(SUFFIX) + +ZTRMMKERNEL = ../generic/ztrmmkernel_2x2.c +ZGEMMKERNEL = ../generic/zgemmkernel_2x2.c +ZGEMMINCOPY = ../generic/zgemm_ncopy_2.c +ZGEMMITCOPY = ../generic/zgemm_tcopy_2.c +ZGEMMONCOPY = ../generic/zgemm_ncopy_2.c +ZGEMMOTCOPY = ../generic/zgemm_tcopy_2.c +ZGEMMINCOPYOBJ = zgemm_incopy$(TSUFFIX).$(SUFFIX) +ZGEMMITCOPYOBJ = zgemm_itcopy$(TSUFFIX).$(SUFFIX) +ZGEMMONCOPYOBJ = zgemm_oncopy$(TSUFFIX).$(SUFFIX) +ZGEMMOTCOPYOBJ = zgemm_otcopy$(TSUFFIX).$(SUFFIX) + +STRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +STRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +STRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +STRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +DTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +DTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +DTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +DTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +CTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +CTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +CTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +CTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +ZTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +ZTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +ZTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +ZTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +CGEMM3MKERNEL = ../generic/zgemmkernel_2x2.c +ZGEMM3MKERNEL = ../generic/zgemmkernel_2x2.c +SROTKERNEL = srot.c +DROTKERNEL = drot.c + +include $(KERNELDIR)/KERNEL.RDNA2_SQUARE diff --git a/kernel/x86_64/KERNEL.GFX1031 b/kernel/x86_64/KERNEL.GFX1031 new file mode 100644 index 0000000000..801fc8331d --- /dev/null +++ b/kernel/x86_64/KERNEL.GFX1031 @@ -0,0 +1,115 @@ +# RDNA2 / RDNA4 (gfx1030, gfx1031, gfx1200, gfx1201) host-side kernels. +# +# The GPU does the GEMM math in kernel/rdna2; these definitions only cover the +# host packing/copy routines OpenBLAS still builds. They are the generic C +# kernels on purpose: the x86 assembly L3 kernels assume their own tile shapes +# and offsets, so pairing them with a GPU core name produces packing that does +# not match them. Unroll sizes here must stay in step with the FORCE_GFX blocks +# in getarch.c and the RDNA2 block in param.h. +# +# S/D: 4x4 generic register-tiled GEMM, width-4 packing. +# C/Z: 2x2 generic complex GEMM, width-2 packing. + +SSCALKERNEL = sscal.c +DSCALKERNEL = dscal.c +CSCALKERNEL = cscal.c +ZSCALKERNEL = zscal.c + +SGEMVNKERNEL = sgemv_n_4.c +SGEMVTKERNEL = sgemv_t_4.c + +DGEMVNKERNEL = dgemv_n_4.c +DGEMVTKERNEL = dgemv_t_4.c + +ZGEMVNKERNEL = zgemv_n_4.c +ZGEMVTKERNEL = zgemv_t_4.c + +CGEMVNKERNEL = cgemv_n_4.c +CGEMVTKERNEL = cgemv_t_4.c + +SSYMV_L_KERNEL = ssymv_L.c +SSYMV_U_KERNEL = ssymv_U.c +DSYMV_L_KERNEL = dsymv_L.c +DSYMV_U_KERNEL = dsymv_U.c + +SDOTKERNEL = sdot.c +DSDOTKERNEL = sdot.c +DDOTKERNEL = ddot.c +CDOTKERNEL = cdot.c +ZDOTKERNEL = zdot.c + +SAXPYKERNEL = saxpy.c +DAXPYKERNEL = daxpy.c +CAXPYKERNEL = caxpy.c +ZAXPYKERNEL = zaxpy.c + +STRMMKERNEL = ../generic/trmmkernel_4x4.c +SGEMMKERNEL = ../generic/gemmkernel_4x4.c +SGEMMINCOPY = ../generic/gemm_ncopy_4.c +SGEMMITCOPY = ../generic/gemm_tcopy_4.c +SGEMMONCOPY = ../generic/gemm_ncopy_4.c +SGEMMOTCOPY = ../generic/gemm_tcopy_4.c +SGEMMINCOPYOBJ = sgemm_incopy$(TSUFFIX).$(SUFFIX) +SGEMMITCOPYOBJ = sgemm_itcopy$(TSUFFIX).$(SUFFIX) +SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX) +SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX) + +DTRMMKERNEL = ../generic/trmmkernel_4x4.c +DGEMMKERNEL = ../generic/gemmkernel_4x4.c +DGEMMINCOPY = ../generic/gemm_ncopy_4.c +DGEMMITCOPY = ../generic/gemm_tcopy_4.c +DGEMMONCOPY = ../generic/gemm_ncopy_4.c +DGEMMOTCOPY = ../generic/gemm_tcopy_4.c +DGEMMINCOPYOBJ = dgemm_incopy$(TSUFFIX).$(SUFFIX) +DGEMMITCOPYOBJ = dgemm_itcopy$(TSUFFIX).$(SUFFIX) +DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX) +DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX) + +CTRMMKERNEL = ../generic/ztrmmkernel_2x2.c +CGEMMKERNEL = ../generic/zgemmkernel_2x2.c +CGEMMINCOPY = ../generic/zgemm_ncopy_2.c +CGEMMITCOPY = ../generic/zgemm_tcopy_2.c +CGEMMONCOPY = ../generic/zgemm_ncopy_2.c +CGEMMOTCOPY = ../generic/zgemm_tcopy_2.c +CGEMMINCOPYOBJ = cgemm_incopy$(TSUFFIX).$(SUFFIX) +CGEMMITCOPYOBJ = cgemm_itcopy$(TSUFFIX).$(SUFFIX) +CGEMMONCOPYOBJ = cgemm_oncopy$(TSUFFIX).$(SUFFIX) +CGEMMOTCOPYOBJ = cgemm_otcopy$(TSUFFIX).$(SUFFIX) + +ZTRMMKERNEL = ../generic/ztrmmkernel_2x2.c +ZGEMMKERNEL = ../generic/zgemmkernel_2x2.c +ZGEMMINCOPY = ../generic/zgemm_ncopy_2.c +ZGEMMITCOPY = ../generic/zgemm_tcopy_2.c +ZGEMMONCOPY = ../generic/zgemm_ncopy_2.c +ZGEMMOTCOPY = ../generic/zgemm_tcopy_2.c +ZGEMMINCOPYOBJ = zgemm_incopy$(TSUFFIX).$(SUFFIX) +ZGEMMITCOPYOBJ = zgemm_itcopy$(TSUFFIX).$(SUFFIX) +ZGEMMONCOPYOBJ = zgemm_oncopy$(TSUFFIX).$(SUFFIX) +ZGEMMOTCOPYOBJ = zgemm_otcopy$(TSUFFIX).$(SUFFIX) + +STRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +STRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +STRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +STRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +DTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +DTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +DTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +DTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +CTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +CTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +CTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +CTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +ZTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +ZTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +ZTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +ZTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +CGEMM3MKERNEL = ../generic/zgemmkernel_2x2.c +ZGEMM3MKERNEL = ../generic/zgemmkernel_2x2.c +SROTKERNEL = srot.c +DROTKERNEL = drot.c + +include $(KERNELDIR)/KERNEL.RDNA2_SQUARE diff --git a/kernel/x86_64/KERNEL.GFX1200 b/kernel/x86_64/KERNEL.GFX1200 new file mode 100644 index 0000000000..801fc8331d --- /dev/null +++ b/kernel/x86_64/KERNEL.GFX1200 @@ -0,0 +1,115 @@ +# RDNA2 / RDNA4 (gfx1030, gfx1031, gfx1200, gfx1201) host-side kernels. +# +# The GPU does the GEMM math in kernel/rdna2; these definitions only cover the +# host packing/copy routines OpenBLAS still builds. They are the generic C +# kernels on purpose: the x86 assembly L3 kernels assume their own tile shapes +# and offsets, so pairing them with a GPU core name produces packing that does +# not match them. Unroll sizes here must stay in step with the FORCE_GFX blocks +# in getarch.c and the RDNA2 block in param.h. +# +# S/D: 4x4 generic register-tiled GEMM, width-4 packing. +# C/Z: 2x2 generic complex GEMM, width-2 packing. + +SSCALKERNEL = sscal.c +DSCALKERNEL = dscal.c +CSCALKERNEL = cscal.c +ZSCALKERNEL = zscal.c + +SGEMVNKERNEL = sgemv_n_4.c +SGEMVTKERNEL = sgemv_t_4.c + +DGEMVNKERNEL = dgemv_n_4.c +DGEMVTKERNEL = dgemv_t_4.c + +ZGEMVNKERNEL = zgemv_n_4.c +ZGEMVTKERNEL = zgemv_t_4.c + +CGEMVNKERNEL = cgemv_n_4.c +CGEMVTKERNEL = cgemv_t_4.c + +SSYMV_L_KERNEL = ssymv_L.c +SSYMV_U_KERNEL = ssymv_U.c +DSYMV_L_KERNEL = dsymv_L.c +DSYMV_U_KERNEL = dsymv_U.c + +SDOTKERNEL = sdot.c +DSDOTKERNEL = sdot.c +DDOTKERNEL = ddot.c +CDOTKERNEL = cdot.c +ZDOTKERNEL = zdot.c + +SAXPYKERNEL = saxpy.c +DAXPYKERNEL = daxpy.c +CAXPYKERNEL = caxpy.c +ZAXPYKERNEL = zaxpy.c + +STRMMKERNEL = ../generic/trmmkernel_4x4.c +SGEMMKERNEL = ../generic/gemmkernel_4x4.c +SGEMMINCOPY = ../generic/gemm_ncopy_4.c +SGEMMITCOPY = ../generic/gemm_tcopy_4.c +SGEMMONCOPY = ../generic/gemm_ncopy_4.c +SGEMMOTCOPY = ../generic/gemm_tcopy_4.c +SGEMMINCOPYOBJ = sgemm_incopy$(TSUFFIX).$(SUFFIX) +SGEMMITCOPYOBJ = sgemm_itcopy$(TSUFFIX).$(SUFFIX) +SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX) +SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX) + +DTRMMKERNEL = ../generic/trmmkernel_4x4.c +DGEMMKERNEL = ../generic/gemmkernel_4x4.c +DGEMMINCOPY = ../generic/gemm_ncopy_4.c +DGEMMITCOPY = ../generic/gemm_tcopy_4.c +DGEMMONCOPY = ../generic/gemm_ncopy_4.c +DGEMMOTCOPY = ../generic/gemm_tcopy_4.c +DGEMMINCOPYOBJ = dgemm_incopy$(TSUFFIX).$(SUFFIX) +DGEMMITCOPYOBJ = dgemm_itcopy$(TSUFFIX).$(SUFFIX) +DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX) +DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX) + +CTRMMKERNEL = ../generic/ztrmmkernel_2x2.c +CGEMMKERNEL = ../generic/zgemmkernel_2x2.c +CGEMMINCOPY = ../generic/zgemm_ncopy_2.c +CGEMMITCOPY = ../generic/zgemm_tcopy_2.c +CGEMMONCOPY = ../generic/zgemm_ncopy_2.c +CGEMMOTCOPY = ../generic/zgemm_tcopy_2.c +CGEMMINCOPYOBJ = cgemm_incopy$(TSUFFIX).$(SUFFIX) +CGEMMITCOPYOBJ = cgemm_itcopy$(TSUFFIX).$(SUFFIX) +CGEMMONCOPYOBJ = cgemm_oncopy$(TSUFFIX).$(SUFFIX) +CGEMMOTCOPYOBJ = cgemm_otcopy$(TSUFFIX).$(SUFFIX) + +ZTRMMKERNEL = ../generic/ztrmmkernel_2x2.c +ZGEMMKERNEL = ../generic/zgemmkernel_2x2.c +ZGEMMINCOPY = ../generic/zgemm_ncopy_2.c +ZGEMMITCOPY = ../generic/zgemm_tcopy_2.c +ZGEMMONCOPY = ../generic/zgemm_ncopy_2.c +ZGEMMOTCOPY = ../generic/zgemm_tcopy_2.c +ZGEMMINCOPYOBJ = zgemm_incopy$(TSUFFIX).$(SUFFIX) +ZGEMMITCOPYOBJ = zgemm_itcopy$(TSUFFIX).$(SUFFIX) +ZGEMMONCOPYOBJ = zgemm_oncopy$(TSUFFIX).$(SUFFIX) +ZGEMMOTCOPYOBJ = zgemm_otcopy$(TSUFFIX).$(SUFFIX) + +STRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +STRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +STRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +STRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +DTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +DTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +DTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +DTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +CTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +CTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +CTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +CTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +ZTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +ZTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +ZTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +ZTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +CGEMM3MKERNEL = ../generic/zgemmkernel_2x2.c +ZGEMM3MKERNEL = ../generic/zgemmkernel_2x2.c +SROTKERNEL = srot.c +DROTKERNEL = drot.c + +include $(KERNELDIR)/KERNEL.RDNA2_SQUARE diff --git a/kernel/x86_64/KERNEL.GFX1201 b/kernel/x86_64/KERNEL.GFX1201 new file mode 100644 index 0000000000..801fc8331d --- /dev/null +++ b/kernel/x86_64/KERNEL.GFX1201 @@ -0,0 +1,115 @@ +# RDNA2 / RDNA4 (gfx1030, gfx1031, gfx1200, gfx1201) host-side kernels. +# +# The GPU does the GEMM math in kernel/rdna2; these definitions only cover the +# host packing/copy routines OpenBLAS still builds. They are the generic C +# kernels on purpose: the x86 assembly L3 kernels assume their own tile shapes +# and offsets, so pairing them with a GPU core name produces packing that does +# not match them. Unroll sizes here must stay in step with the FORCE_GFX blocks +# in getarch.c and the RDNA2 block in param.h. +# +# S/D: 4x4 generic register-tiled GEMM, width-4 packing. +# C/Z: 2x2 generic complex GEMM, width-2 packing. + +SSCALKERNEL = sscal.c +DSCALKERNEL = dscal.c +CSCALKERNEL = cscal.c +ZSCALKERNEL = zscal.c + +SGEMVNKERNEL = sgemv_n_4.c +SGEMVTKERNEL = sgemv_t_4.c + +DGEMVNKERNEL = dgemv_n_4.c +DGEMVTKERNEL = dgemv_t_4.c + +ZGEMVNKERNEL = zgemv_n_4.c +ZGEMVTKERNEL = zgemv_t_4.c + +CGEMVNKERNEL = cgemv_n_4.c +CGEMVTKERNEL = cgemv_t_4.c + +SSYMV_L_KERNEL = ssymv_L.c +SSYMV_U_KERNEL = ssymv_U.c +DSYMV_L_KERNEL = dsymv_L.c +DSYMV_U_KERNEL = dsymv_U.c + +SDOTKERNEL = sdot.c +DSDOTKERNEL = sdot.c +DDOTKERNEL = ddot.c +CDOTKERNEL = cdot.c +ZDOTKERNEL = zdot.c + +SAXPYKERNEL = saxpy.c +DAXPYKERNEL = daxpy.c +CAXPYKERNEL = caxpy.c +ZAXPYKERNEL = zaxpy.c + +STRMMKERNEL = ../generic/trmmkernel_4x4.c +SGEMMKERNEL = ../generic/gemmkernel_4x4.c +SGEMMINCOPY = ../generic/gemm_ncopy_4.c +SGEMMITCOPY = ../generic/gemm_tcopy_4.c +SGEMMONCOPY = ../generic/gemm_ncopy_4.c +SGEMMOTCOPY = ../generic/gemm_tcopy_4.c +SGEMMINCOPYOBJ = sgemm_incopy$(TSUFFIX).$(SUFFIX) +SGEMMITCOPYOBJ = sgemm_itcopy$(TSUFFIX).$(SUFFIX) +SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX) +SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX) + +DTRMMKERNEL = ../generic/trmmkernel_4x4.c +DGEMMKERNEL = ../generic/gemmkernel_4x4.c +DGEMMINCOPY = ../generic/gemm_ncopy_4.c +DGEMMITCOPY = ../generic/gemm_tcopy_4.c +DGEMMONCOPY = ../generic/gemm_ncopy_4.c +DGEMMOTCOPY = ../generic/gemm_tcopy_4.c +DGEMMINCOPYOBJ = dgemm_incopy$(TSUFFIX).$(SUFFIX) +DGEMMITCOPYOBJ = dgemm_itcopy$(TSUFFIX).$(SUFFIX) +DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX) +DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX) + +CTRMMKERNEL = ../generic/ztrmmkernel_2x2.c +CGEMMKERNEL = ../generic/zgemmkernel_2x2.c +CGEMMINCOPY = ../generic/zgemm_ncopy_2.c +CGEMMITCOPY = ../generic/zgemm_tcopy_2.c +CGEMMONCOPY = ../generic/zgemm_ncopy_2.c +CGEMMOTCOPY = ../generic/zgemm_tcopy_2.c +CGEMMINCOPYOBJ = cgemm_incopy$(TSUFFIX).$(SUFFIX) +CGEMMITCOPYOBJ = cgemm_itcopy$(TSUFFIX).$(SUFFIX) +CGEMMONCOPYOBJ = cgemm_oncopy$(TSUFFIX).$(SUFFIX) +CGEMMOTCOPYOBJ = cgemm_otcopy$(TSUFFIX).$(SUFFIX) + +ZTRMMKERNEL = ../generic/ztrmmkernel_2x2.c +ZGEMMKERNEL = ../generic/zgemmkernel_2x2.c +ZGEMMINCOPY = ../generic/zgemm_ncopy_2.c +ZGEMMITCOPY = ../generic/zgemm_tcopy_2.c +ZGEMMONCOPY = ../generic/zgemm_ncopy_2.c +ZGEMMOTCOPY = ../generic/zgemm_tcopy_2.c +ZGEMMINCOPYOBJ = zgemm_incopy$(TSUFFIX).$(SUFFIX) +ZGEMMITCOPYOBJ = zgemm_itcopy$(TSUFFIX).$(SUFFIX) +ZGEMMONCOPYOBJ = zgemm_oncopy$(TSUFFIX).$(SUFFIX) +ZGEMMOTCOPYOBJ = zgemm_otcopy$(TSUFFIX).$(SUFFIX) + +STRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +STRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +STRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +STRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +DTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +DTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +DTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +DTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +CTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +CTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +CTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +CTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +ZTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +ZTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +ZTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +ZTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +CGEMM3MKERNEL = ../generic/zgemmkernel_2x2.c +ZGEMM3MKERNEL = ../generic/zgemmkernel_2x2.c +SROTKERNEL = srot.c +DROTKERNEL = drot.c + +include $(KERNELDIR)/KERNEL.RDNA2_SQUARE diff --git a/probe_iface.c b/probe_iface.c new file mode 100644 index 0000000000..0fcc3d3ae3 --- /dev/null +++ b/probe_iface.c @@ -0,0 +1,233 @@ +/* Layer isolation for the RDNA2 level-3 hooks. + * + * For each case this computes: + * cpu = cblas_* with the GPU path forced off, vs a naive scalar reference + * gpu = cblas_* with the GPU path forced on, vs the same naive reference + * + * If cpu is already wrong, the harness is misusing CBLAS (or the naive reference + * is wrong) and the GPU result cannot be judged by it. If cpu is right and gpu + * is wrong, the interface hook is passing bad arguments. + * + * The unused triangle holds zeros rather than NaN so both the CPU driver and the + * bridge are exercised on well-defined input. + */ +#include +#include +#include +#include + +#include "cblas.h" + +#ifdef _WIN32 +static void env_set(const char *k, const char *v) { _putenv_s(k, v); } +#else +static void env_set(const char *k, const char *v) { setenv(k, v, 1); } +#endif + +static unsigned rng = 4242u; +static float rnd(void) { + rng = rng * 1103515245u + 12345u; + return (float)(((rng >> 8) & 0xFFFF) / 32768.0) - 1.0f; +} + +/* A with the stored triangle filled and the rest zeroed. */ +static void fill_tri(float *a, int kdim, int lda, int uplo, int unit, + int dominant) { + for (int j = 0; j < kdim; ++j) + for (int i = 0; i < kdim; ++i) { + const int stored = (uplo == 0) ? (i <= j) : (i >= j); + if (!stored) + a[i + j * lda] = 0.0f; + else if (i == j) + a[i + j * lda] = unit ? 1.0f : (dominant ? (float)kdim + 2.0f : 1.5f); + else + a[i + j * lda] = dominant ? 0.25f * rnd() : rnd(); + } +} + +/* Dense op(A), and the triangle the dense matrix actually occupies. */ +static void densify(const float *a, int kdim, int lda, int uplo, int tr, int unit, + float *d, int *tri) { + for (int j = 0; j < kdim; ++j) + for (int i = 0; i < kdim; ++i) { + const int si = tr ? j : i; + const int sj = tr ? i : j; + const int stored = (uplo == 0) ? (si <= sj) : (si >= sj); + float v = 0.0f; + if (stored) v = a[si + sj * lda]; + if (si == sj && unit) v = 1.0f; + d[i + j * kdim] = v; + } + /* Triangle occupied by the dense op(A). Transposing swaps the triangle, so + * side=left (which solves with op(A)) uses this and side=right (which + * solves with op(A)^T) uses its complement. */ + *tri = tr ? (uplo ? 0 : 1) : uplo; +} + +static double err(const float *ref, const float *got, int n) { + double maxabs = 0.0, scale = 0.0; + for (int i = 0; i < n; ++i) { + const double d = fabs((double)got[i] - (double)ref[i]); + if (d > maxabs) maxabs = d; + if (fabs((double)ref[i]) > scale) scale = fabs((double)ref[i]); + } + return (scale > 0.0) ? maxabs / scale : maxabs; +} + +static void one_trmm(int m, int n, int side, int uplo, int tr, int unit) { + const int kdim = side ? n : m; + const double alpha = 0.75; + float *a = malloc((size_t)kdim * kdim * sizeof(float)); + float *d = malloc((size_t)kdim * kdim * sizeof(float)); + float *b0 = malloc((size_t)m * n * sizeof(float)); + float *ref = malloc((size_t)m * n * sizeof(float)); + float *cpu = malloc((size_t)m * n * sizeof(float)); + float *gpu = malloc((size_t)m * n * sizeof(float)); + int tri; + + fill_tri(a, kdim, kdim, uplo, unit, 0); + densify(a, kdim, kdim, uplo, tr, unit, d, &tri); + for (int i = 0; i < m * n; ++i) b0[i] = rnd(); + + /* naive: C = alpha*op(A)*B or alpha*B*op(A) */ + for (int j = 0; j < n; ++j) + for (int i = 0; i < m; ++i) { + double s = 0.0; + for (int p = 0; p < kdim; ++p) + s += side ? (double)b0[i + p * m] * d[p + j * kdim] + : (double)d[i + p * kdim] * b0[p + j * m]; + ref[i + j * m] = (float)(alpha * s); + } + + env_set("OPENBLAS_RDNA2_GPU", "0"); + memcpy(cpu, b0, (size_t)m * n * sizeof(float)); + cblas_strmm(CblasColMajor, side ? CblasRight : CblasLeft, + uplo ? CblasLower : CblasUpper, tr ? CblasTrans : CblasNoTrans, + unit ? CblasUnit : CblasNonUnit, m, n, (float)alpha, a, kdim, cpu, + m); + + env_set("OPENBLAS_RDNA2_GPU", "1"); + memcpy(gpu, b0, (size_t)m * n * sizeof(float)); + cblas_strmm(CblasColMajor, side ? CblasRight : CblasLeft, + uplo ? CblasLower : CblasUpper, tr ? CblasTrans : CblasNoTrans, + unit ? CblasUnit : CblasNonUnit, m, n, (float)alpha, a, kdim, gpu, + m); + + printf("trmm %3dx%-3d %s/%s/%s/%s cpu_err=%.2e gpu_err=%.2e %s\n", m, n, + side ? "R" : "L", uplo ? "L" : "U", tr ? "T" : "N", unit ? "U" : "N", + err(ref, cpu, m * n), err(ref, gpu, m * n), + (err(ref, gpu, m * n) < 1e-5) ? "ok" : "BAD"); + + free(a); free(d); free(b0); free(ref); free(cpu); free(gpu); +} + +static void one_trsm(int m, int n, int side, int uplo, int tr, int unit) { + const int kdim = side ? n : m; + const double alpha = 1.25; + float *a = malloc((size_t)kdim * kdim * sizeof(float)); + float *d = malloc((size_t)kdim * kdim * sizeof(float)); + float *b0 = malloc((size_t)m * n * sizeof(float)); + float *ref = malloc((size_t)m * n * sizeof(float)); + float *cpu = malloc((size_t)m * n * sizeof(float)); + float *gpu = malloc((size_t)m * n * sizeof(float)); + int tri; + + fill_tri(a, kdim, kdim, uplo, unit, 1); + densify(a, kdim, kdim, uplo, tr, unit, d, &tri); + for (int i = 0; i < m * n; ++i) b0[i] = rnd(); + + /* naive solve, working on the dense factor */ + if (!side) { + for (int j = 0; j < n; ++j) + for (int i = 0; i < m; ++i) ref[i + j * m] = (float)(alpha * b0[i + j * m]); + if (tri == 0) { + for (int j = 0; j < n; ++j) + for (int i = kdim - 1; i >= 0; --i) { + double s = ref[i + j * m]; + for (int q = i + 1; q < kdim; ++q) + s -= (double)d[i + q * kdim] * ref[q + j * m]; + ref[i + j * m] = (float)(s / d[i + i * kdim]); + } + } else { + for (int j = 0; j < n; ++j) + for (int i = 0; i < kdim; ++i) { + double s = ref[i + j * m]; + for (int q = 0; q < i; ++q) + s -= (double)d[i + q * kdim] * ref[q + j * m]; + ref[i + j * m] = (float)(s / d[i + i * kdim]); + } + } + } else { + /* X*op(A) = Y <=> op(A)^T X^T = Y^T */ + float *yd = malloc((size_t)kdim * m * sizeof(float)); + float *t = malloc((size_t)kdim * kdim * sizeof(float)); + const int triT = 1 - tri; /* T = op(A)^T flips the triangle */ + for (int j = 0; j < m; ++j) + for (int i = 0; i < kdim; ++i) yd[i + j * kdim] = (float)(alpha * b0[j + i * m]); + for (int j = 0; j < kdim; ++j) + for (int i = 0; i < kdim; ++i) t[i + j * kdim] = d[j + i * kdim]; + if (triT == 0) { + for (int j = 0; j < m; ++j) + for (int i = kdim - 1; i >= 0; --i) { + double s = yd[i + j * kdim]; + for (int q = i + 1; q < kdim; ++q) s -= (double)t[i + q * kdim] * yd[q + j * kdim]; + yd[i + j * kdim] = (float)(s / t[i + i * kdim]); + } + } else { + for (int j = 0; j < m; ++j) + for (int i = 0; i < kdim; ++i) { + double s = yd[i + j * kdim]; + for (int q = 0; q < i; ++q) s -= (double)t[i + q * kdim] * yd[q + j * kdim]; + yd[i + j * kdim] = (float)(s / t[i + i * kdim]); + } + } + for (int j = 0; j < n; ++j) + for (int i = 0; i < m; ++i) ref[i + j * m] = yd[j + i * kdim]; + free(yd); free(t); + } + + env_set("OPENBLAS_RDNA2_GPU", "0"); + memcpy(cpu, b0, (size_t)m * n * sizeof(float)); + cblas_strsm(CblasColMajor, side ? CblasRight : CblasLeft, + uplo ? CblasLower : CblasUpper, tr ? CblasTrans : CblasNoTrans, + unit ? CblasUnit : CblasNonUnit, m, n, (float)alpha, a, kdim, cpu, + m); + + env_set("OPENBLAS_RDNA2_GPU", "1"); + memcpy(gpu, b0, (size_t)m * n * sizeof(float)); + cblas_strsm(CblasColMajor, side ? CblasRight : CblasLeft, + uplo ? CblasLower : CblasUpper, tr ? CblasTrans : CblasNoTrans, + unit ? CblasUnit : CblasNonUnit, m, n, (float)alpha, a, kdim, gpu, + m); + + printf("trsm %3dx%-3d %s/%s/%s/%s cpu_err=%.2e gpu_err=%.2e %s\n", m, n, + side ? "R" : "L", uplo ? "L" : "U", tr ? "T" : "N", unit ? "U" : "N", + err(ref, cpu, m * n), err(ref, gpu, m * n), + (err(ref, gpu, m * n) < 1e-5) ? "ok" : "BAD"); + + free(a); free(d); free(b0); free(ref); free(cpu); free(gpu); +} + +int main(void) { + const int sizes[3][2] = {{8, 5}, {300, 5}, {5, 300}}; + + env_set("OPENBLAS_RDNA2_GPU_MIN_MNK", "1"); + + for (int s = 0; s < 3; ++s) + for (int side = 0; side < 2; ++side) + for (int uplo = 0; uplo < 2; ++uplo) + for (int tr = 0; tr < 2; ++tr) { + one_trmm(sizes[s][0], sizes[s][1], side, uplo, tr, 0); + one_trsm(sizes[s][0], sizes[s][1], side, uplo, tr, 0); + } + + /* unit diagonal, small sizes only */ + for (int side = 0; side < 2; ++side) + for (int uplo = 0; uplo < 2; ++uplo) + for (int tr = 0; tr < 2; ++tr) { + one_trmm(9, 6, side, uplo, tr, 1); + one_trsm(9, 6, side, uplo, tr, 1); + } + + return 0; +} diff --git a/probe_l3_bridge.c b/probe_l3_bridge.c new file mode 100644 index 0000000000..4f63f92b0c --- /dev/null +++ b/probe_l3_bridge.c @@ -0,0 +1,318 @@ +/* Direct probe of the RDNA2 level-3 bridges, bypassing the BLAS dispatch. + * + * This separates "is the bridge arithmetic right" from "does the interface hook + * pass the right arguments". Every routine is compared against a naive scalar + * reference computed here, so a mismatch is unambiguous. + * + * The unused triangle is filled with a large sentinel value instead of NaN so a + * wrong-triangle bug shows up as a large numeric error rather than a NaN. + */ +#include +#include +#include +#include + +#ifdef _WIN32 +static void env_set(const char *k, const char *v) { _putenv_s(k, v); } +#else +static void env_set(const char *k, const char *v) { setenv(k, v, 1); } +#endif + +extern int rdna2_oblas_ssymm_cm(long long, long long, double, const float *, + long long, const float *, long long, double, + float *, long long, int, int); +extern int rdna2_oblas_strmm_cm(long long, long long, double, const float *, + long long, float *, long long, int, int, int, int); +extern int rdna2_oblas_strsm_cm(long long, long long, double, const float *, + long long, float *, long long, int, int, int, int); +extern int rdna2_oblas_sgemm_cm(long long, long long, long long, double, + const float *, long long, const float *, + long long, double, float *, long long, int, int); + +static unsigned rng = 999u; +static float rnd(void) { + rng = rng * 1103515245u + 12345u; + return (float)(((rng >> 8) & 0xFFFF) / 32768.0) - 1.0f; +} + +/* Dense full triangle view. sentinel fills the triangle that must not be read. */ +static void fill_sym(float *a, int kdim, int lda, int uplo) { + for (int j = 0; j < kdim; ++j) + for (int i = 0; i < kdim; ++i) + a[i + j * lda] = ((uplo == 0) ? (i <= j) : (i >= j)) ? rnd() : 1.0e6f; +} + +static void fill_tri(float *a, int kdim, int lda, int uplo, int unit, + int dominant) { + for (int j = 0; j < kdim; ++j) + for (int i = 0; i < kdim; ++i) { + const int stored = (uplo == 0) ? (i <= j) : (i >= j); + if (!stored) + a[i + j * lda] = 1.0e6f; + else if (i == j) + a[i + j * lda] = unit ? 0.0f : (dominant ? (float)kdim + 2.0f : 1.5f); + else + a[i + j * lda] = dominant ? 0.25f * rnd() : rnd(); + } +} + +/* Dense symmetric expansion, for the reference product only. */ +static void expand_sym(const float *a, int kdim, int lda, int uplo, float *out) { + for (int j = 0; j < kdim; ++j) + for (int i = 0; i < kdim; ++i) + out[i + j * kdim] = + ((uplo == 0) ? (i <= j) : (i >= j)) ? a[i + j * lda] : a[j + i * lda]; +} + +/* Dense op(A) expansion, for the reference product only. */ +static void expand_op(const float *a, int kdim, int lda, int uplo, int tr, + int unit, float *out) { + for (int j = 0; j < kdim; ++j) { + for (int i = 0; i < kdim; ++i) { + const int si = tr ? j : i; + const int sj = tr ? i : j; + const int stored = (uplo == 0) ? (si <= sj) : (si >= sj); + float v = 0.0f; + if (stored) v = a[si + sj * lda]; + if (si == sj && unit) v = 1.0f; + out[i + j * kdim] = v; + } + } +} + +static int cmp(const char *name, const float *ref, const float *got, int n) { + double maxabs = 0.0, scale = 0.0; + int bad = -1; + + for (int i = 0; i < n; ++i) { + const double d = fabs((double)got[i] - (double)ref[i]); + if (d > maxabs) { maxabs = d; bad = i; } + if (fabs((double)ref[i]) > scale) scale = fabs((double)ref[i]); + } + const double rel = (scale > 0.0) ? maxabs / scale : maxabs; + printf("%-34s max|diff|=%.4e rel=%.4e %s", name, maxabs, rel, + (rel < 1e-5) ? "OK" : "BAD"); + if (rel >= 1e-5 && bad >= 0) + printf(" (first at %d: ref=%.6f got=%.6f)", bad, ref[bad], got[bad]); + printf("\n"); + return rel < 1e-5; +} + +static int t_symm(int m, int n, int side, int uplo) { + const int kdim = side ? n : m; + const double alpha = 0.75, beta = 0.5; + float *a = malloc((size_t)kdim * kdim * sizeof(float)); + float *d = malloc((size_t)kdim * kdim * sizeof(float)); + float *b = malloc((size_t)m * n * sizeof(float)); + float *c0 = malloc((size_t)m * n * sizeof(float)); + float *ref = malloc((size_t)m * n * sizeof(float)); + float *got = malloc((size_t)m * n * sizeof(float)); + char name[64]; + int handled, i; + + fill_sym(a, kdim, kdim, uplo); + expand_sym(a, kdim, kdim, uplo, d); + for (i = 0; i < m * n; ++i) b[i] = rnd(); + for (i = 0; i < m * n; ++i) c0[i] = rnd(); + + /* reference */ + for (int j = 0; j < n; ++j) + for (int i = 0; i < m; ++i) { + double s = 0.0; + for (int p = 0; p < kdim; ++p) + s += (side ? b[i + p * m] * d[p + j * kdim] + : d[i + p * kdim] * b[p + j * m]); + ref[i + j * m] = (float)(alpha * s + beta * c0[i + j * m]); + } + + memcpy(got, c0, (size_t)m * n * sizeof(float)); + handled = rdna2_oblas_ssymm_cm(m, n, alpha, a, kdim, b, m, beta, got, m, + side, uplo); + snprintf(name, sizeof(name), "ssymm %dx%d %s/%s h=%d", m, n, + side ? "R" : "L", uplo ? "L" : "U", handled); + i = cmp(name, ref, got, m * n); + + free(a); free(d); free(b); free(c0); free(ref); free(got); + return i; +} + +static int t_trmm(int m, int n, int side, int uplo, int tr, int unit) { + const int kdim = side ? n : m; + const double alpha = 0.75; + float *a = malloc((size_t)kdim * kdim * sizeof(float)); + float *d = malloc((size_t)kdim * kdim * sizeof(float)); + float *b = malloc((size_t)m * n * sizeof(float)); + float *ref = malloc((size_t)m * n * sizeof(float)); + float *got = malloc((size_t)m * n * sizeof(float)); + char name[64]; + int handled, i; + + fill_tri(a, kdim, kdim, uplo, unit, 0); + expand_op(a, kdim, kdim, uplo, tr, unit, d); + for (i = 0; i < m * n; ++i) b[i] = rnd(); + + /* reference: C = alpha * op(A) * B (side=left) or alpha * B * op(A) */ + for (int j = 0; j < n; ++j) + for (int i = 0; i < m; ++i) { + double s = 0.0; + for (int p = 0; p < kdim; ++p) + s += (side ? b[i + p * m] * d[p + j * kdim] + : d[i + p * kdim] * b[p + j * m]); + ref[i + j * m] = (float)(alpha * s); + } + + memcpy(got, b, (size_t)m * n * sizeof(float)); + handled = rdna2_oblas_strmm_cm(m, n, alpha, a, kdim, got, m, side, uplo, tr, + unit); + snprintf(name, sizeof(name), "strmm %dx%d %s/%s/%s/%s h=%d", m, n, + side ? "R" : "L", uplo ? "L" : "U", tr ? "T" : "N", unit ? "U" : "N", + handled); + i = cmp(name, ref, got, m * n); + + free(a); free(d); free(b); free(ref); free(got); + return i; +} + +static int t_trsm(int m, int n, int side, int uplo, int tr, int unit) { + const int kdim = side ? n : m; + const double alpha = 1.25; + float *a = malloc((size_t)kdim * kdim * sizeof(float)); + float *d = malloc((size_t)kdim * kdim * sizeof(float)); + float *b = malloc((size_t)m * n * sizeof(float)); + float *ref = malloc((size_t)m * n * sizeof(float)); + float *got = malloc((size_t)m * n * sizeof(float)); + char name[64]; + int handled, i; + + fill_tri(a, kdim, kdim, uplo, unit, 1); + expand_op(a, kdim, kdim, uplo, tr, unit, d); + for (i = 0; i < m * n; ++i) b[i] = rnd(); + + /* reference: solve op(A)*X = alpha*B (left) or X*op(A) = alpha*B (right), + * by plain substitution on the dense expansion. */ + memcpy(ref, b, (size_t)m * n * sizeof(float)); + for (i = 0; i < m * n; ++i) ref[i] = (float)(alpha * (double)ref[i]); + + if (!side) { + /* The solve matrix is op(A) = A or A^T, and transposing flips the triangle, + * so branch on the triangle of op(A), not on the stored uplo. */ + const int triD = tr ? !uplo : uplo; + if (triD == 0) { + for (int j = 0; j < n; ++j) + for (int i = kdim - 1; i >= 0; --i) { + double s = ref[i + j * m]; + for (int q = i + 1; q < kdim; ++q) + s -= (double)d[i + q * kdim] * (double)ref[q + j * m]; + ref[i + j * m] = (float)(s / (double)d[i + i * kdim]); + } + } else { + for (int j = 0; j < n; ++j) + for (int i = 0; i < kdim; ++i) { + double s = ref[i + j * m]; + for (int q = 0; q < i; ++q) + s -= (double)d[i + q * kdim] * (double)ref[q + j * m]; + ref[i + j * m] = (float)(s / (double)d[i + i * kdim]); + } + } + } else { + /* X*op(A) = Y <=> op(A)^T X^T = Y^T */ + float *yd = malloc((size_t)kdim * m * sizeof(float)); + for (int j = 0; j < m; ++j) + for (int i = 0; i < kdim; ++i) yd[i + j * kdim] = ref[j + i * m]; + /* T = op(A)^T */ + float *t = malloc((size_t)kdim * kdim * sizeof(float)); + for (int j = 0; j < kdim; ++j) + for (int i = 0; i < kdim; ++i) t[i + j * kdim] = d[j + i * kdim]; + const int tri = (tr ? uplo : !uplo); + if (tri == 0) { + for (int j = 0; j < m; ++j) + for (int i = kdim - 1; i >= 0; --i) { + double s = yd[i + j * kdim]; + for (int q = i + 1; q < kdim; ++q) + s -= (double)t[i + q * kdim] * (double)yd[q + j * kdim]; + yd[i + j * kdim] = (float)(s / (double)t[i + i * kdim]); + } + } else { + for (int j = 0; j < m; ++j) + for (int i = 0; i < kdim; ++i) { + double s = yd[i + j * kdim]; + for (int q = 0; q < i; ++q) + s -= (double)t[i + q * kdim] * (double)yd[q + j * kdim]; + yd[i + j * kdim] = (float)(s / (double)t[i + i * kdim]); + } + } + for (int j = 0; j < n; ++j) + for (int i = 0; i < m; ++i) ref[i + j * m] = yd[j + i * kdim]; + free(yd); free(t); + } + + memcpy(got, b, (size_t)m * n * sizeof(float)); + handled = rdna2_oblas_strsm_cm(m, n, alpha, a, kdim, got, m, side, uplo, tr, + unit); + snprintf(name, sizeof(name), "strsm %dx%d %s/%s/%s/%s h=%d", m, n, + side ? "R" : "L", uplo ? "L" : "U", tr ? "T" : "N", unit ? "U" : "N", + handled); + i = cmp(name, ref, got, m * n); + + free(a); free(d); free(b); free(ref); free(got); + return i; +} + +static int t_sgemm(int m, int n, int k) { + float *a = malloc((size_t)m * k * sizeof(float)); + float *b = malloc((size_t)k * n * sizeof(float)); + float *ref = malloc((size_t)m * n * sizeof(float)); + float *got = malloc((size_t)m * n * sizeof(float)); + char name[64]; + int handled, i; + + for (i = 0; i < m * k; ++i) a[i] = rnd(); + for (i = 0; i < k * n; ++i) b[i] = rnd(); + + for (int j = 0; j < n; ++j) + for (int i2 = 0; i2 < m; ++i2) { + double s = 0.0; + for (int p = 0; p < k; ++p) s += (double)a[i2 + p * k] * (double)b[p + j * k]; + ref[i2 + j * m] = (float)s; + } + + for (i = 0; i < m * n; ++i) got[i] = 0.0f; + /* lda == k, ldb == k, ldc == m: the layout the bridge accepts. */ + handled = rdna2_oblas_sgemm_cm(m, n, k, 1.0, a, k, b, k, 0.0, got, m, 0, 0); + snprintf(name, sizeof(name), "sgemm %dx%dx%d h=%d", m, n, k, handled); + i = cmp(name, ref, got, m * n); + + free(a); free(b); free(ref); free(got); + return i; +} + +int main(void) { + int pass = 1; + const int ms[] = {5, 9, 130, 140, 300}; + + env_set("OPENBLAS_RDNA2_GPU_MIN_MNK", "1"); + + pass &= t_sgemm(5, 3, 4); + pass &= t_sgemm(64, 32, 16); + pass &= t_sgemm(200, 100, 96); + + for (int k = 0; k < 5; ++k) { + const int m = ms[k], n = (k < 2) ? 3 : 7; + for (int side = 0; side < 2; ++side) { + for (int uplo = 0; uplo < 2; ++uplo) { + pass &= t_symm(m, n, side, uplo); + for (int tr = 0; tr < 2; ++tr) { + pass &= t_trmm(m, n, side, uplo, tr, 0); + pass &= t_trsm(m, n, side, uplo, tr, 0); + if (m < 10) { + pass &= t_trmm(m, n, side, uplo, tr, 1); + pass &= t_trsm(m, n, side, uplo, tr, 1); + } + } + } + } + } + + printf("\n%s\n", pass ? "BRIDGE PROBE ALL OK" : "BRIDGE PROBE FAILURES"); + return pass ? 0 : 1; +} diff --git a/test_oblas_l3.c b/test_oblas_l3.c new file mode 100644 index 0000000000..f22cfd040b --- /dev/null +++ b/test_oblas_l3.c @@ -0,0 +1,274 @@ +/* A/B correctness harness for the RDNA2 level-3 bridges. + * + * Each case is computed twice through the same library: once with the GPU path + * forced off and once with it forced on, both with the size gate disabled so the + * bridge is actually entered. The two results are then compared. + * + * The triangle of the symmetric/triangular operand that BLAS promises not to + * read is filled with NaN, so a bridge that reads the wrong triangle fails + * loudly instead of quietly returning a plausible number. + */ +#include +#include +#include +#include + +#include "cblas.h" + +#ifdef _WIN32 +static void env_set(const char *k, const char *v) { _putenv_s(k, v); } +#else +static void env_set(const char *k, const char *v) { setenv(k, v, 1); } +#endif + +static unsigned rng_state = 12345u; + +static float rnd(void) { + rng_state = rng_state * 1103515245u + 12345u; + return (float)(((rng_state >> 8) & 0xFFFF) / 32768.0) - 1.0f; +} + +/* Symmetric operand: write one triangle, NaN the other. */ +static void fill_sym(float *a, int kdim, int lda, int uplo) { + int i, j; + + for (j = 0; j < kdim; ++j) { + for (i = 0; i < kdim; ++i) { + const int stored = (uplo == 0) ? (i <= j) : (i >= j); + a[i + j * lda] = stored ? rnd() : NAN; + } + } +} + +/* Triangular operand: write one triangle, NaN the other. When dominant, the + * diagonal is scaled up so the solve stays well conditioned; a unit diagonal is + * stored as zero, which must be ignored. */ +static void fill_tri(float *a, int kdim, int lda, int uplo, int unit_diag, + int dominant) { + int i, j; + + for (j = 0; j < kdim; ++j) { + for (i = 0; i < kdim; ++i) { + const int stored = (uplo == 0) ? (i <= j) : (i >= j); + + if (!stored) + a[i + j * lda] = NAN; + else if (i == j) + a[i + j * lda] = unit_diag ? 0.0f + : (dominant ? (float)kdim + 2.0f : 1.5f); + else + a[i + j * lda] = dominant ? 0.25f * rnd() : rnd(); + } + } +} + +static void fill_rand(float *x, long n) { + long i; + + for (i = 0; i < n; ++i) x[i] = rnd(); +} + +static int compare(const char *name, const float *ref, const float *got, long n) { + long i; + double maxabs = 0.0, maxrel = 0.0, scale = 0.0; + + for (i = 0; i < n; ++i) { + const double d = fabs((double)got[i] - (double)ref[i]); + const double s = fabs((double)ref[i]); + + if (isnan((double)got[i])) { + printf(" %-28s FAIL: NaN at %ld\n", name, i); + return 0; + } + if (d > maxabs) maxabs = d; + if (s > scale) scale = s; + } + + maxrel = (scale > 0.0) ? maxabs / scale : maxabs; + printf(" %-28s max|diff|=%.3e max_rel=%.3e %s\n", name, maxabs, maxrel, + (maxrel < 5e-5) ? "MATCH" : "MISMATCH"); + return maxrel < 5e-5; +} + +/* ---------------------------------------------------------------- sgemm --- */ +static int case_sgemm(int m, int n, int k) { + float *a = malloc((size_t)m * k * sizeof(float)); + float *b = malloc((size_t)k * n * sizeof(float)); + float *ref = malloc((size_t)m * n * sizeof(float)); + float *got = malloc((size_t)m * n * sizeof(float)); + int ok; + + fill_rand(a, (long)m * k); + fill_rand(b, (long)k * n); + + env_set("OPENBLAS_RDNA2_GPU", "0"); + memcpy(got, ref, (size_t)m * n * sizeof(float)); + for (int i = 0; i < m * n; ++i) got[i] = 0.0f; + cblas_sgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, m, n, k, 1.0f, a, m, b, + k, 0.0f, got, m); + memcpy(ref, got, (size_t)m * n * sizeof(float)); + + env_set("OPENBLAS_RDNA2_GPU", "1"); + for (int i = 0; i < m * n; ++i) got[i] = 0.0f; + cblas_sgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, m, n, k, 1.0f, a, m, b, + k, 0.0f, got, m); + + ok = compare("sgemm", ref, got, (long)m * n); + + free(a); free(b); free(ref); free(got); + return ok; +} + +/* ---------------------------------------------------------------- ssymm --- */ +static void symm_once(int m, int n, int side, int uplo, float alpha, float beta, + const float *a, int lda, const float *b, int ldb, float *c, + int ldc) { + cblas_ssymm(CblasColMajor, side ? CblasRight : CblasLeft, + uplo ? CblasLower : CblasUpper, m, n, alpha, a, lda, b, ldb, beta, + c, ldc); +} + +static int case_symm(int m, int n, int side, int uplo, float alpha, float beta) { + const int kdim = side ? n : m; + float *a = malloc((size_t)kdim * kdim * sizeof(float)); + float *b = malloc((size_t)m * n * sizeof(float)); + float *ref = malloc((size_t)m * n * sizeof(float)); + float *got = malloc((size_t)m * n * sizeof(float)); + char name[64]; + int ok; + + fill_sym(a, kdim, kdim, uplo); + fill_rand(b, (long)m * n); + + env_set("OPENBLAS_RDNA2_GPU", "0"); + memcpy(ref, b, (size_t)m * n * sizeof(float)); + symm_once(m, n, side, uplo, alpha, beta, a, kdim, b, m, ref, m); + + env_set("OPENBLAS_RDNA2_GPU", "1"); + memcpy(got, b, (size_t)m * n * sizeof(float)); + symm_once(m, n, side, uplo, alpha, beta, a, kdim, b, m, got, m); + + snprintf(name, sizeof(name), "ssymm %s/%s a=%.1f b=%.1f", + side ? "R" : "L", uplo ? "L" : "U", (double)alpha, (double)beta); + ok = compare(name, ref, got, (long)m * n); + + free(a); free(b); free(ref); free(got); + return ok; +} + +/* ---------------------------------------------------------------- strmm --- */ +static void trmm_once(int m, int n, int side, int uplo, int trans, int unit, + float alpha, const float *a, int lda, float *b, int ldb) { + cblas_strmm(CblasColMajor, side ? CblasRight : CblasLeft, + uplo ? CblasLower : CblasUpper, + trans ? CblasTrans : CblasNoTrans, + unit ? CblasUnit : CblasNonUnit, m, n, alpha, a, lda, b, ldb); +} + +static int case_trmm(int m, int n, int side, int uplo, int trans, int unit) { + const int kdim = side ? n : m; + const float alpha = 0.75f; + float *a = malloc((size_t)kdim * kdim * sizeof(float)); + float *b0 = malloc((size_t)m * n * sizeof(float)); + float *ref = malloc((size_t)m * n * sizeof(float)); + float *got = malloc((size_t)m * n * sizeof(float)); + char name[64]; + int ok; + + fill_tri(a, kdim, kdim, uplo, unit, 0); + fill_rand(b0, (long)m * n); + + env_set("OPENBLAS_RDNA2_GPU", "0"); + memcpy(ref, b0, (size_t)m * n * sizeof(float)); + trmm_once(m, n, side, uplo, trans, unit, alpha, a, kdim, ref, m); + + env_set("OPENBLAS_RDNA2_GPU", "1"); + memcpy(got, b0, (size_t)m * n * sizeof(float)); + trmm_once(m, n, side, uplo, trans, unit, alpha, a, kdim, got, m); + + snprintf(name, sizeof(name), "strmm %s/%s/%s/%s", side ? "R" : "L", + uplo ? "L" : "U", trans ? "T" : "N", unit ? "U" : "N"); + ok = compare(name, ref, got, (long)m * n); + + free(a); free(b0); free(ref); free(got); + return ok; +} + +/* ---------------------------------------------------------------- strsm --- */ +static void trsm_once(int m, int n, int side, int uplo, int trans, int unit, + float alpha, const float *a, int lda, float *b, int ldb) { + cblas_strsm(CblasColMajor, side ? CblasRight : CblasLeft, + uplo ? CblasLower : CblasUpper, + trans ? CblasTrans : CblasNoTrans, + unit ? CblasUnit : CblasNonUnit, m, n, alpha, a, lda, b, ldb); +} + +static int case_trsm(int m, int n, int side, int uplo, int trans, int unit) { + const int kdim = side ? n : m; + const float alpha = 1.25f; + float *a = malloc((size_t)kdim * kdim * sizeof(float)); + float *b0 = malloc((size_t)m * n * sizeof(float)); + float *ref = malloc((size_t)m * n * sizeof(float)); + float *got = malloc((size_t)m * n * sizeof(float)); + char name[64]; + int ok; + + fill_tri(a, kdim, kdim, uplo, unit, 1); + fill_rand(b0, (long)m * n); + + env_set("OPENBLAS_RDNA2_GPU", "0"); + memcpy(ref, b0, (size_t)m * n * sizeof(float)); + trsm_once(m, n, side, uplo, trans, unit, alpha, a, kdim, ref, m); + + env_set("OPENBLAS_RDNA2_GPU", "1"); + memcpy(got, b0, (size_t)m * n * sizeof(float)); + trsm_once(m, n, side, uplo, trans, unit, alpha, a, kdim, got, m); + + snprintf(name, sizeof(name), "strsm %s/%s/%s/%s", side ? "R" : "L", + uplo ? "L" : "U", trans ? "T" : "N", unit ? "U" : "N"); + ok = compare(name, ref, got, (long)m * n); + + free(a); free(b0); free(ref); free(got); + return ok; +} + +int main(int argc, char **argv) { + const int big = (argc > 1) ? atoi(argv[1]) : 1; + int pass = 1, i; + + /* Any size is exercised, so the measured crossover gate is disabled and the + * bridge is always entered. */ + env_set("OPENBLAS_RDNA2_GPU_MIN_MNK", "1"); + + printf("== sgemm ==\n"); + pass &= case_sgemm(256, 192, 128); + if (big) pass &= case_sgemm(1024, 768, 512); + + printf("== ssymm ==\n"); + pass &= case_symm(200, 150, 0, 0, 1.0f, 0.0f); + pass &= case_symm(200, 150, 0, 1, 0.75f, 0.25f); + pass &= case_symm(180, 260, 1, 0, -0.5f, 1.0f); + pass &= case_symm(180, 260, 1, 1, 1.0f, 0.0f); + if (big) pass &= case_symm(400, 300, 0, 0, 1.0f, 0.0f); + + printf("== strmm ==\n"); + for (i = 0; i < 8; ++i) { + const int side = i & 1, uplo = (i >> 1) & 1, trans = (i >> 2) & 1; + pass &= case_trmm(200 + 40 * i, 150, side, uplo, trans, 0); + } + pass &= case_trmm(210, 160, 0, 0, 0, 1); + pass &= case_trmm(210, 160, 1, 1, 1, 1); + if (big) pass &= case_trmm(400, 300, 0, 1, 1, 0); + + printf("== strsm ==\n"); + for (i = 0; i < 8; ++i) { + const int side = i & 1, uplo = (i >> 1) & 1, trans = (i >> 2) & 1; + pass &= case_trsm(200 + 40 * i, 150, side, uplo, trans, 0); + } + pass &= case_trsm(210, 160, 0, 0, 0, 1); + pass &= case_trsm(210, 160, 1, 1, 1, 1); + if (big) pass &= case_trsm(400, 300, 0, 1, 1, 0); + + printf("\n%s\n", pass ? "ALL CASES MATCH" : "FAILURES PRESENT"); + return pass ? 0 : 1; +} From c7314904e0012d869bf9071f260e95ed10990742 Mon Sep 17 00:00:00 2001 From: rr Date: Fri, 6 Nov 2026 01:04:24 +0530 Subject: [PATCH 04/13] RDNA2: extend sgemm bridge to handle transposed A/B layouts Materialize TransA/TransB operands into contiguous column-major temporaries and pass them through rdna2_gemm_cm_core, removing the CPU fallback for transposed layouts (handled=1, MATCH vs naive ref for TN/NT/TT incl. non-unit alpha/beta). --- kernel/rdna2/rdna2_oblas.c | 69 +++++++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 8 deletions(-) diff --git a/kernel/rdna2/rdna2_oblas.c b/kernel/rdna2/rdna2_oblas.c index 0a6becb7a9..38b56205a6 100644 --- a/kernel/rdna2/rdna2_oblas.c +++ b/kernel/rdna2/rdna2_oblas.c @@ -199,30 +199,83 @@ static int rdna2_gemm_cm_core(BLASLONG m, BLASLONG n, BLASLONG k, } /* Returns 1 when the product was computed on the GPU, 0 when the caller should - * carry on with the generic CPU path. */ + * carry on with the generic CPU path. + * + * transa/transb follow the OpenBLAS interface convention for real GEMM: + * 0 = NoTrans, nonzero = Trans (conjugate codes 2/3 do not apply to sgemm). + * op(A) is m x k (lda >= m when not transposed, lda >= k when transposed). + * op(B) is k x n (ldb >= k when not transposed, ldb >= n when transposed). + * + * The HIP core takes untransposed column-major operands. A transposed operand + * is materialised once into a contiguous column-major temporary with the + * transposed shape, so the gather/tiling path in rdna2_gemm_cm_core stays + * unchanged and every layout reaches the GPU instead of falling back. */ int rdna2_oblas_sgemm_cm(BLASLONG m, BLASLONG n, BLASLONG k, double alpha, const float *a, BLASLONG lda, const float *b, BLASLONG ldb, double beta, float *c, BLASLONG ldc, int transa, int transb) { + const float *pa = a; + const float *pb = b; + BLASLONG lpa = lda; + BLASLONG lpb = ldb; + float *ta = NULL; + float *tb = NULL; int handled; clock_t t0; if (!rdna2_gpu_enabled()) return 0; - /* Only the untransposed layouts are wired up; the core handles any leading - * dimension, so lda/ldb/ldc are not constrained here. Transposed operands - * need a different gather and stay on the CPU path. */ - if (transa != 0 || transb != 0) return 0; if (m <= 0 || n <= 0 || k <= 0) return 0; if (m > RDNA2_MAX_DIM || n > RDNA2_MAX_DIM || k > RDNA2_MAX_DIM) return 0; if ((double)m * (double)n * (double)k < rdna2_min_mnk()) return 0; + if (transa) { + BLASLONG i, p; + + if (lda < k) return 0; /* stored as k x m */ + ta = (float *)malloc((size_t)m * (size_t)k * sizeof(float)); + if (ta == NULL) return 0; + /* op(A)(i,p) = A(p,i) = a[p + i*lda] -> contiguous m x k, ld = m */ + for (p = 0; p < k; ++p) + for (i = 0; i < m; ++i) + ta[i + p * m] = a[p + i * lda]; + pa = ta; + lpa = m; + } else { + if (lda < m) return 0; + } + + if (transb) { + BLASLONG p, j; + + if (ldb < n) return 0; /* stored as n x k */ + tb = (float *)malloc((size_t)k * (size_t)n * sizeof(float)); + if (tb == NULL) { + free(ta); + return 0; + } + /* op(B)(p,j) = B(j,p) = b[j + p*ldb] -> contiguous k x n, ld = k */ + for (p = 0; p < k; ++p) + for (j = 0; j < n; ++j) + tb[p + j * k] = b[j + p * ldb]; + pb = tb; + lpb = k; + } else { + if (ldb < k) { + free(ta); + return 0; + } + } + t0 = clock(); - handled = rdna2_gemm_cm_core(m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); + handled = rdna2_gemm_cm_core(m, n, k, alpha, pa, lpa, pb, lpb, beta, c, ldc); if (rdna2_debug()) - fprintf(stderr, "[rdna2] sgemm %ldx%ldx%ld handled=%d %ldms\n", - (long)m, (long)n, (long)k, handled, rdna2_ms(t0)); + fprintf(stderr, "[rdna2] sgemm %ldx%ldx%ld trans=%d%d handled=%d %ldms\n", + (long)m, (long)n, (long)k, transa ? 1 : 0, transb ? 1 : 0, handled, + rdna2_ms(t0)); + free(ta); + free(tb); return handled; } From 96464bb84bd8a5bd0f61fc2322071a3de6a9d9f4 Mon Sep 17 00:00:00 2001 From: rr Date: Fri, 6 Nov 2026 01:10:51 +0530 Subject: [PATCH 05/13] Fix trmmkernel_4x4 for GFX targets (all 16 s/d side/uplo/trans/diag cases) Rewrite the 4x4 TRMM micro-kernel mirroring trmmkernel_2x2.c's proven LEFT/TRANSA offset/temp/post-advance logic over the 4/2/1 tiling of gemmkernel_4x4.c (matching ncopy_4/tcopy_4, SGEMM/DGEMM unroll 4). TRMM overwrites C (alpha*dot), not accumulate. Verified: full cblas_strmm/dtrmm driver path 32/32 ok (M=137,N=113, odd remainders) vs naive ref; test_oblas_l3 (gfx1200/gfx1201) strmm 10/10 MATCH handled=1 (ALL CASES MATCH). --- kernel/generic/trmmkernel_4x4.c | 1074 +++++++++---------------------- 1 file changed, 313 insertions(+), 761 deletions(-) diff --git a/kernel/generic/trmmkernel_4x4.c b/kernel/generic/trmmkernel_4x4.c index a85828cadf..323a4d01ba 100644 --- a/kernel/generic/trmmkernel_4x4.c +++ b/kernel/generic/trmmkernel_4x4.c @@ -1,875 +1,427 @@ #include "common.h" -#include -int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc ,BLASLONG offset) +/* 4x4 TRMM micro-kernel. Structure mirrors trmmkernel_2x2.c (proven) expanded + * to the 4/2/1 tiling of gemmkernel_4x4.c so it matches ncopy_4/tcopy_4 and + * SGEMM/DGEMM UNROLL 4. TRMM overwrites C (no accumulation into existing C). */ +int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc +#ifdef TRMMKERNEL + ,BLASLONG offset +#endif + ) { - BLASLONG i,j,k; FLOAT *C0,*C1,*C2,*C3,*ptrba,*ptrbb; - - FLOAT res0_0; - FLOAT res0_1; - FLOAT res0_2; - FLOAT res0_3; - - FLOAT res1_0; - FLOAT res1_1; - FLOAT res1_2; - FLOAT res1_3; - - FLOAT res2_0; - FLOAT res2_1; - FLOAT res2_2; - FLOAT res2_3; - - FLOAT res3_0; - FLOAT res3_1; - FLOAT res3_2; - FLOAT res3_3; - - FLOAT a0; - FLOAT a1; - - FLOAT b0; - FLOAT b1; - FLOAT b2; - FLOAT b3; - + FLOAT r0c0,r1c0,r2c0,r3c0,r0c1,r1c1,r2c1,r3c1,r0c2,r1c2,r2c2,r3c2,r0c3,r1c3,r2c3,r3c3; + FLOAT a0,a1,a2,a3,b0,b1,b2,b3; BLASLONG off, temp; - - bool left; - bool transposed; - bool backwards; - -#ifdef LEFT - left = true; -#else - left = false; -#endif - -#ifdef TRANSA - transposed = true; +#if defined(TRMMKERNEL) && !defined(LEFT) + off = -offset; #else - transposed = false; + off = 0; #endif - backwards = left != transposed; - - if (!left) { - off = -offset; - } - - - for (j=0; j Date: Fri, 6 Nov 2026 12:02:41 +0530 Subject: [PATCH 06/13] Add HIP build knobs and run the level-3 bridge test automatically MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit USE_HIP/GPU_ARCH mirror the CUDA knobs, so a normal TARGET still selects the CPU kernels while USE_HIP=1 adds the device side to the same library; a stamp file makes a GPU_ARCH change rebuild the device object, which make could not otherwise notice. The bridge test gains a `make level3-test` entry point and a CI workflow. It links against a CPU-only archive without ROCm (the HIP runtime is only added when the archive actually contains the device objects), and its failure output is capped per case because a clobbered triangle used to print one line per element and bury the log. Generated with Codebuff 🤖 Co-Authored-By: Codebuff --- .github/workflows/rdna2_level3.yml | 106 +++++++ .gitignore | 1 + Makefile | 11 +- Makefile.rule | 14 + Makefile.system | 25 +- cblas.h | 63 +++++ exports/gensymbol | 2 +- exports/gensymbol.pl | 2 +- getarch.c | 48 ++++ gpu/rdna2/Makefile | 48 +++- gpu/rdna2/README.md | 123 +++++++- gpu/rdna2/rdna2_kernels.hip | 34 +-- gpu/rdna2/test_oblas_gpu.c | 303 ++++++++++++++++++++ gpu/rdna2/test_rdna2_host.cpp | 34 +-- interface/Makefile | 7 +- interface/gemm_i8.c | 438 +++++++++++++++++++++++++++++ interface/syr2k.c | 32 +++ interface/syrk.c | 29 ++ kernel/Makefile | 82 +++--- kernel/rdna2/rdna2_oblas.c | 210 +++++++++++++- kernel/x86_64/KERNEL.RDNA2_SQUARE | 30 ++ param.h | 52 ++++ test/Makefile | 19 +- 23 files changed, 1629 insertions(+), 84 deletions(-) create mode 100644 .github/workflows/rdna2_level3.yml create mode 100644 gpu/rdna2/test_oblas_gpu.c create mode 100644 interface/gemm_i8.c create mode 100644 kernel/x86_64/KERNEL.RDNA2_SQUARE diff --git a/.github/workflows/rdna2_level3.yml b/.github/workflows/rdna2_level3.yml new file mode 100644 index 0000000000..2d2d8f514e --- /dev/null +++ b/.github/workflows/rdna2_level3.yml @@ -0,0 +1,106 @@ +name: rdna2 level-3 bridge test + +# Builds the library and runs gpu/rdna2/test_oblas_gpu.c, which checks the CBLAS +# level-3 entry points (SGEMM and SYRK/SYR2K) against its own double-precision +# reference. The test needs no GPU: on a CPU-only build it still verifies values +# and the SYRK contract that the unstored triangle and the ldc padding are left +# untouched. The GPU dispatch itself is only exercised on a build made with +# USE_HIP=1 on a machine with an AMD card, which hosted runners do not have -- +# the "gpu" job at the bottom is the place to add that on a self-hosted runner. + +on: + push: + paths: + - 'gpu/rdna2/**' + - 'kernel/rdna2/**' + - 'interface/syrk.c' + - 'interface/syr2k.c' + - 'interface/gemm_i8.c' + - 'cblas.h' + - 'Makefile' + - 'Makefile.system' + - 'Makefile.rule' + - '.github/workflows/rdna2_level3.yml' + pull_request: + paths: + - 'gpu/rdna2/**' + - 'kernel/rdna2/**' + - 'interface/syrk.c' + - 'interface/syr2k.c' + - 'interface/gemm_i8.c' + - 'cblas.h' + - 'Makefile' + - 'Makefile.system' + - 'Makefile.rule' + - '.github/workflows/rdna2_level3.yml' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + linux: + name: bridge test (linux, CPU build) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install toolchain + run: | + sudo apt-get update + sudo apt-get install -y build-essential gfortran + + # NOFORTRAN/NO_LAPACK keep the job to what the bridge test needs: CBLAS + # level-3 entry points and the kernels behind them. + - name: Build library + run: make -j"$(nproc)" NOFORTRAN=1 NO_LAPACK=1 NO_LAPACKE=1 libs + + - name: Level-3 bridge test + run: make NOFORTRAN=1 NO_LAPACK=1 NO_LAPACKE=1 level3-test + + windows: + name: bridge test (windows, CPU build) + runs-on: windows-latest + defaults: + run: + shell: msys2 {0} + steps: + - uses: actions/checkout@v4 + + - uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: false + install: >- + base-devel + make + mingw-w64-ucrt-x86_64-gcc + + - name: Build library + run: make -j"$(nproc)" CC=gcc NOFORTRAN=1 NO_LAPACK=1 NO_LAPACKE=1 libs + + - name: Level-3 bridge test + run: make CC=gcc NOFORTRAN=1 NO_LAPACK=1 NO_LAPACKE=1 level3-test + + # Device coverage needs an AMD GPU and a ROCm install, which GitHub-hosted + # runners do not provide. On a self-hosted runner carrying one, this job is + # the same two commands with the HIP build switched on: + # + # gpu: + # name: bridge test (self-hosted, USE_HIP) + # runs-on: [self-hosted, gpu, rocm] + # steps: + # - uses: actions/checkout@v4 + # - name: Build library with the device kernels + # run: | + # make clean + # make -j"$(nproc)" USE_HIP=1 GPU_ARCH=gfx1201 libs + # - name: Level-3 bridge test (asserts the dispatch actually fires) + # run: | + # make USE_HIP=1 GPU_ARCH=gfx1201 level3-test + # make -C gpu/rdna2 oblas-test OPENBLAS_RDNA2_GPU_DEBUG=1 2>&1 | tee gpu.log + # grep -q 'handled=1' gpu.log diff --git a/.gitignore b/.gitignore index 3d6ddb5052..ac66537edd 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ *.wasm *.out *.tmp +*.stamp lapack-3.1.1 lapack-3.1.1.tgz lapack-3.4.1 diff --git a/Makefile b/Makefile index ae1da5e9fe..67dd05275f 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ endif SUBDIRS_ALL = $(SUBDIRS) test ctest utest exports benchmark ../laswp ../bench cpp_thread_test -.PHONY : all libs netlib $(RELA) test ctest shared install +.PHONY : all libs netlib $(RELA) test ctest shared install level3-test .NOTPARALLEL : shared all :: tests @@ -171,6 +171,15 @@ ifeq ($(OSNAME), AIX) endif endif +# Level-3 bridge test: gpu/rdna2/test_oblas_gpu.c checks the CBLAS level-3 +# entry points against its own double-precision reference, including the SYRK +# contract that the unstored triangle and the ldc padding are left alone. It +# needs nothing beyond a built library, so it also runs on CPU-only builds; when +# the library was built with USE_HIP=1 it additionally exercises the device +# dispatch, which is the case it exists for. +level3-test : libs + @$(MAKE) -C gpu/rdna2 oblas-test + tests : shared ifeq ($(NOFORTRAN), $(filter 0,$(NOFORTRAN))) touch $(LIBNAME) diff --git a/Makefile.rule b/Makefile.rule index 5c843cfedd..745f611842 100644 --- a/Makefile.rule +++ b/Makefile.rule @@ -19,6 +19,20 @@ VERSION = 0.3.34.dev # automatically detected. # TARGET = PENRYN +# Build the RDNA2/RDNA4 GPU kernels (HIP) alongside the CPU ones. TARGET still +# selects the CPU kernels exactly as it does without USE_HIP, so USE_HIP=1 on an +# ordinary target yields one library that is complete on both: large level-3 +# calls go to the device, everything else (small calls, complex, double, the +# level 1/2 routines and all of LAPACK) stays on the CPU. Set +# OPENBLAS_RDNA2_GPU=0 in the environment to force the CPU path at run time. +# USE_HIP = 1 + +# Device architecture(s) for the HIP kernels, independent of TARGET. Defaults to +# the architecture of a GFX* TARGET, otherwise gfx1201. A space separated list +# builds one fat object covering several cards, so a single library serves both +# RDNA2 and RDNA4, e.g. GPU_ARCH = gfx1031 gfx1201 +# GPU_ARCH = gfx1201 + # If you want to support multiple architecture in one binary # DYNAMIC_ARCH = 1 diff --git a/Makefile.system b/Makefile.system index 9a5b7af845..4526bf5fea 100644 --- a/Makefile.system +++ b/Makefile.system @@ -2024,9 +2024,26 @@ HIP_PLATFORM := amd endif export HIP_PLATFORM CXX ?= $(HIPCC) +# Device architecture(s) for the RDNA2/RDNA4 kernels, independent of the CPU +# core, so an ordinary CPU build can carry the GPU side too. Unset means "take +# it from a GFX* TARGET, else gfx1201". +export GPU_ARCH CCOMMON_OPT += -DOPENBLAS_USE_HIP -endif - -ifdef USE_HIP -EXTRALIB += -L/opt/rocm/lib -lamdhip64 -lhiprtc -lhipblas +# Locate the ROCm install instead of hardcoding /opt/rocm, the way the CUDA +# support uses CUDADIR: HIP_PATH from the environment (ROCm sets it) wins, then +# whatever hipcc reports about itself, then the usual install prefix. +HIP_PATH ?= $(patsubst %/,%,$(shell $(HIPCC) --path 2>/dev/null)) +ifeq ($(strip $(HIP_PATH)),) +HIP_PATH := /opt/rocm +endif +# Backslashes in a Windows HIP_PATH would be eaten by the shell in the link +# line, so normalise them here. +# HIP_PATH and HIP_LIB_PATH can also arrive from the environment (ROCm sets +# them), so normalise whatever ends up in them, not just the default. +HIP_LIB_PATH ?= $(HIP_PATH)/lib +HIP_LIB_PATH := $(subst \,/,$(HIP_LIB_PATH)) +export HIP_LIB_PATH +# Only the HIP runtime is needed; hiprtc/hipblas are not used by the kernels. +# The kernels are C++ via hipcc, so the C++ runtime must be linked in as well. +EXTRALIB += -L$(HIP_LIB_PATH) -lamdhip64 -lstdc++ endif diff --git a/cblas.h b/cblas.h index 89b2f99cde..f7728f3e5a 100644 --- a/cblas.h +++ b/cblas.h @@ -520,6 +520,69 @@ OPENBLAS_EXPORT void cblas_sbgemm_batch(OPENBLAS_CONST enum CBLAS_ORDER Order, O OPENBLAS_CONST float * alpha_array, OPENBLAS_CONST bfloat16 ** A_array, OPENBLAS_CONST blasint * lda_array, OPENBLAS_CONST bfloat16 ** B_array, OPENBLAS_CONST blasint * ldb_array, OPENBLAS_CONST float * beta_array, float ** C_array, OPENBLAS_CONST blasint * ldc_array, OPENBLAS_CONST blasint group_count, OPENBLAS_CONST blasint * group_size); OPENBLAS_EXPORT void cblas_sbgemm_batch_strided(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K, OPENBLAS_CONST float alpha, OPENBLAS_CONST bfloat16 * A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST blasint stridea, OPENBLAS_CONST bfloat16 * B, OPENBLAS_CONST blasint ldb, OPENBLAS_CONST blasint strideb, OPENBLAS_CONST float beta, float * C, OPENBLAS_CONST blasint ldc, OPENBLAS_CONST blasint stridec, OPENBLAS_CONST blasint group_size); +/*** INT8 extensions (packed int8 GEMM) ***/ +/* C := op(A) * op(B), with 8-bit integer operands packed four per int32 word. + * + * These entry points expose the RDNA2/RDNA4 int8 GEMM kernels + * (gpu/rdna2/rdna2_kernels.hip) through the normal library, and are also + * available in builds without a GPU, where the same product is computed in + * portable C. Both paths are exact: accumulation is 32-bit two's complement + * (v_dot4_i32_i8), so the two agree bit for bit and no tolerance applies. + * + * Data contract + * - A and B are arrays of int32 words, each holding four signed int8 values: + * value k of a word is bits [8k, 8k+8), i.e. little-endian byte order. This + * is the layout v_dot4_* consumes, so packed quantized weights are used + * byte for byte as stored. + * - K4 is the number of words per row, so the inner dimension is K = 4*K4 + * int8 values. K must be a multiple of 4: a partial trailing word is not + * accepted, and this call performs no repacking of the stored values. + * - lda and ldb are the leading dimensions in words, in the usual BLAS + * sense for the requested order. A is stored M x K4 (TransA == + * CblasNoTrans) or K4 x M (transposed), B is stored K4 x N or N x K4, and: + * CblasColMajor: lda is the row count of the stored array, so + * lda >= max(1, K4) when transposed and >= max(1, M) + * otherwise; ldb >= max(1, N) when transposed and + * >= max(1, K4) otherwise; ldc >= max(1, M). + * CblasRowMajor: lda is the row stride, so lda >= max(1, M) when + * transposed and >= max(1, K4) otherwise; ldb >= + * max(1, K4) when transposed and >= max(1, N) + * otherwise; ldc >= max(1, N). + * - C is int32 for cblas_gemm_i8 and float for cblas_gemm_i8_scale, with + * ldc in C elements: ldc >= max(1, M) for CblasColMajor and + * ldc >= max(1, N) for CblasRowMajor. + * - Order == CblasRowMajor is supported and gives the same result as the + * column-major call on the transposed problem; CblasConjTrans and + * CblasConjNoTrans are accepted as CblasTrans and CblasNoTrans, since + * conjugation is the identity on integers. + * + * Scaling (cblas_gemm_i8_scale only) + * C[i][j] = (acc[i][j] - zp_b * rowsum_j) * sa[i] * sb[j] + * where rowsum_j is the sum of the int8 values of column j of op(B) - the + * exact integer form of asymmetric storage, q representing (q - zp_b) - and + * the correction is computed in 64 bits. sa scales rows of op(A) and has M + * entries, sb scales columns of op(B) and has N entries; either may be NULL + * for 1.0. zp_b is the zero point of op(B), i.e. the weight side; 0 skips the + * row-sum pass. No stored operand is converted to floating point: the only + * float produced per output element is the finished dot product. + * + * There is no alpha/beta in either call. Scale the result on output, or use + * cblas_gemm_i8_scale, which applies the per-row/per-column scale for free. + * 4-bit formats (Q4_0, Q4_K, ...) are not supported: dp4a consumes whole + * bytes, so they would need an in-kernel nibble unpack first. + * + * Dispatch + * In a build with USE_HIP=1 the HIP kernels run the product; + * OPENBLAS_RDNA2_GPU=0 (or =n) forces the CPU implementation, which is also + * what a build without USE_HIP and any failing or device-less launch uses. + * + * Returns 0 on success, -1 on an invalid argument or layout, -2 when a host + * workspace allocation failed. C is left untouched on error. Both calls are + * reentrant; all workspaces are allocated per call. + */ +OPENBLAS_EXPORT int cblas_gemm_i8(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K4, OPENBLAS_CONST int *A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST int *B, OPENBLAS_CONST blasint ldb, int *C, OPENBLAS_CONST blasint ldc); +OPENBLAS_EXPORT int cblas_gemm_i8_scale(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K4, OPENBLAS_CONST int *A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST int *B, OPENBLAS_CONST blasint ldb, float *C, OPENBLAS_CONST blasint ldc, OPENBLAS_CONST float *sa, OPENBLAS_CONST float *sb, OPENBLAS_CONST int zp_b); + /*** FLOAT16 extensions ***/ OPENBLAS_EXPORT void cblas_shgemm(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K, OPENBLAS_CONST float alpha, OPENBLAS_CONST hfloat16 *A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST hfloat16 *B, OPENBLAS_CONST blasint ldb, OPENBLAS_CONST float beta, float *C, OPENBLAS_CONST blasint ldc); diff --git a/exports/gensymbol b/exports/gensymbol index 0b65b1c540..d1c36da1f0 100755 --- a/exports/gensymbol +++ b/exports/gensymbol @@ -130,7 +130,7 @@ cblasobjsz=" cblas_zaxpyc cblas_zdrot cblas_zrotg cblas_dzamax cblas_dzamin cblas_zgemm_batch cblas_zgemm_batch_strided " -cblasobjs="cblas_xerbla" +cblasobjs="cblas_xerbla cblas_gemm_i8 cblas_gemm_i8_scale" bfcblasobjs="cblas_bgemm cblas_bgemv cblas_bscal cblas_sbgemm cblas_sbgemv cblas_sbdot cblas_sbstobf16 cblas_sbdtobf16 cblas_sbf16tos cblas_dbf16tod cblas_sbgemm_batch cblas_sbgemm_batch_strided diff --git a/exports/gensymbol.pl b/exports/gensymbol.pl index fca6ea14e1..cf90b52cfb 100644 --- a/exports/gensymbol.pl +++ b/exports/gensymbol.pl @@ -126,7 +126,7 @@ cblas_izamax, cblas_izamin, cblas_dzsum,cblas_zimatcopy,cblas_zomatcopy, cblas_zaxpyc, cblas_zdrot, cblas_zrotg, cblas_dzamax, cblas_dzamin, cblas_zgemm_batch, cblas_zgemm_batch_strided); -@cblasobjs = ( cblas_xerbla ); +@cblasobjs = ( cblas_xerbla, cblas_gemm_i8, cblas_gemm_i8_scale ); @bfcblasobjs = (cblas_bgemm, cblas_bgemv, cblas_bscal, cblas_sbgemm, cblas_sbgemmt, cblas_sbgemmtr, cblas_sbgemv, cblas_sbdot, cblas_sbstobf16, cblas_sbdtobf16, cblas_sbf16tos, cblas_dbf16tod, cblas_sbgemm_batch, cblas_sbgemm_batch_strided); @hfcblasobjs = (cblas_shgemm); diff --git a/getarch.c b/getarch.c index 2e3104fef2..c308ceb59b 100644 --- a/getarch.c +++ b/getarch.c @@ -1913,6 +1913,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define ARCHCONFIG "-DRDNA2 -DGFX1031 " #define LIBNAME "rdna2" #define CORENAME "GFX1031" +#define SGEMM_DEFAULT_UNROLL_M 4 +#define SGEMM_DEFAULT_UNROLL_N 4 +#define DGEMM_DEFAULT_UNROLL_M 4 +#define DGEMM_DEFAULT_UNROLL_N 4 +#define CGEMM_DEFAULT_UNROLL_M 2 +#define CGEMM_DEFAULT_UNROLL_N 2 +#define ZGEMM_DEFAULT_UNROLL_M 2 +#define ZGEMM_DEFAULT_UNROLL_N 2 +#define SGEMM_DEFAULT_Q 128 +#define DGEMM_DEFAULT_Q 64 +#define CGEMM_DEFAULT_Q 64 +#define ZGEMM_DEFAULT_Q 32 #endif #ifdef FORCE_GFX1030 @@ -1923,6 +1935,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define ARCHCONFIG "-DRDNA2 -DGFX1030 " #define LIBNAME "rdna2" #define CORENAME "GFX1030" +#define SGEMM_DEFAULT_UNROLL_M 4 +#define SGEMM_DEFAULT_UNROLL_N 4 +#define DGEMM_DEFAULT_UNROLL_M 4 +#define DGEMM_DEFAULT_UNROLL_N 4 +#define CGEMM_DEFAULT_UNROLL_M 2 +#define CGEMM_DEFAULT_UNROLL_N 2 +#define ZGEMM_DEFAULT_UNROLL_M 2 +#define ZGEMM_DEFAULT_UNROLL_N 2 +#define SGEMM_DEFAULT_Q 128 +#define DGEMM_DEFAULT_Q 64 +#define CGEMM_DEFAULT_Q 64 +#define ZGEMM_DEFAULT_Q 32 #endif #ifdef FORCE_GFX1201 @@ -1934,6 +1958,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define ARCHCONFIG "-DRDNA2 -DGFX1201 " #define LIBNAME "rdna2" #define CORENAME "GFX1201" +#define SGEMM_DEFAULT_UNROLL_M 4 +#define SGEMM_DEFAULT_UNROLL_N 4 +#define DGEMM_DEFAULT_UNROLL_M 4 +#define DGEMM_DEFAULT_UNROLL_N 4 +#define CGEMM_DEFAULT_UNROLL_M 2 +#define CGEMM_DEFAULT_UNROLL_N 2 +#define ZGEMM_DEFAULT_UNROLL_M 2 +#define ZGEMM_DEFAULT_UNROLL_N 2 +#define SGEMM_DEFAULT_Q 128 +#define DGEMM_DEFAULT_Q 64 +#define CGEMM_DEFAULT_Q 64 +#define ZGEMM_DEFAULT_Q 32 #endif #ifdef FORCE_GFX1200 @@ -1944,6 +1980,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define ARCHCONFIG "-DRDNA2 -DGFX1200 " #define LIBNAME "rdna2" #define CORENAME "GFX1200" +#define SGEMM_DEFAULT_UNROLL_M 4 +#define SGEMM_DEFAULT_UNROLL_N 4 +#define DGEMM_DEFAULT_UNROLL_M 4 +#define DGEMM_DEFAULT_UNROLL_N 4 +#define CGEMM_DEFAULT_UNROLL_M 2 +#define CGEMM_DEFAULT_UNROLL_N 2 +#define ZGEMM_DEFAULT_UNROLL_M 2 +#define ZGEMM_DEFAULT_UNROLL_N 2 +#define SGEMM_DEFAULT_Q 128 +#define DGEMM_DEFAULT_Q 64 +#define CGEMM_DEFAULT_Q 64 +#define ZGEMM_DEFAULT_Q 32 #endif diff --git a/gpu/rdna2/Makefile b/gpu/rdna2/Makefile index 1df7bf2c47..505a8fab74 100644 --- a/gpu/rdna2/Makefile +++ b/gpu/rdna2/Makefile @@ -7,6 +7,7 @@ # make ARCHS="gfx1031 gfx1201" test # one fat binary covering several GPUs # make multi fat binary for gfx1030 gfx1031 gfx1200 gfx1201 +# make oblas-test run the level-3 bridge test (see below) # make clean # # Targets: gfx1031 = RX 6700 XT / 6750 XT, gfx1030 = RX 6800 / 6900, @@ -44,12 +45,55 @@ test_rdna2: test_rdna2_host.cpp rdna2_kernels.o rdna2_gemm_ref.h rdna2_kernels_m test: test_rdna2 ./test_rdna2 +# CPU vs GPU comparison. CPU side is this file's blocked -O3 gemm plus the +# OpenBLAS DLL shipped with numpy when OB_DLL points at it (or is autodetected). +OB_DLL ?= $(shell python -c "import glob,os,numpy;print((glob.glob(os.path.join(os.path.dirname(numpy.__file__),'..','numpy.libs','*openblas*.dll'))+[''])[0])" 2>/dev/null) + +bench: rdna2_kernels.o + $(HIPCC) -x hip $(OFFLOAD) $(CXXFLAGS_COMMON) \ + bench_rdna2.cpp -x none rdna2_kernels.o $(LDFLAGS) -o bench_rdna2 + +bench-run: bench + ./bench_rdna2 "$(OB_DLL)" + +# Level-3 bridge test against the library built in this tree: checks the +# interface-side kernel plumbing (SGEMM, and SYRK/SYR2K where the dispatch is +# compiled in) against an independent double-precision reference, including the +# SYRK contract that the unstored triangle and the ldc padding are untouched. +# The same binary is useful either way -- the device path is exercised only when +# OBLAS_LIB is a USE_HIP build, while the reference checks still run on a +# CPU-only build. Point OBLAS_LIB/LDFLAGS at another tree or a ROCm runtime +# outside the default search path if needed: +# make oblas-test OBLAS_LIB=/path/libopenblas.a LDFLAGS=-L/opt/rocm/lib +CC ?= cc +AR ?= ar +OBLAS_INC ?= ../.. +OBLAS_LIB ?= ../../libopenblas.a +OBLAS_CFLAGS?= -O2 -Wall -I$(OBLAS_INC) +# A USE_HIP library carries the device objects and needs the HIP runtime at link +# time; a plain CPU library does not, so the runtime stays out of the link line +# on machines (and CI) without ROCm instead of failing there. +OBLAS_HIP := $(shell $(AR) t $(OBLAS_LIB) 2>/dev/null | grep -q rdna2_kernels && echo 1) +ifeq ($(OBLAS_HIP),1) +# HIP_LIB_PATH comes from the top-level build (Makefile.system exports it) when +# this is run through "make level3-test" there. +OBLAS_LIBS ?= $(if $(HIP_LIB_PATH),-L$(subst \,/,$(HIP_LIB_PATH))) -lamdhip64 -lstdc++ -lpthread -lm +else +OBLAS_LIBS ?= -lpthread -lm +endif + +test_oblas_gpu: test_oblas_gpu.c + $(CC) $(OBLAS_CFLAGS) $< $(OBLAS_LIB) $(LDFLAGS) $(OBLAS_LIBS) -o $@ + +oblas-test: test_oblas_gpu + ./test_oblas_gpu + # Every target this tree is shaped for, in one binary. -B because the object # files are otherwise reused from whatever arch the last build used. multi: $(MAKE) -B ARCHS="gfx1030 gfx1031 gfx1200 gfx1201" test clean: - rm -f rdna2_kernels.o test_rdna2 + rm -f rdna2_kernels.o test_rdna2 bench_rdna2 test_oblas_gpu -.PHONY: all test multi clean +.PHONY: all test bench bench-run oblas-test multi clean diff --git a/gpu/rdna2/README.md b/gpu/rdna2/README.md index 041ca8a072..388fd627dd 100644 --- a/gpu/rdna2/README.md +++ b/gpu/rdna2/README.md @@ -105,6 +105,67 @@ one finished dot product. int8 bytes, so they would need an in-kernel nibble unpack, again integer-only, before this path could take them. +## OpenBLAS entry points + +The int8 kernels are reachable from the library through two CBLAS-style +functions declared in `cblas.h` and implemented in `interface/gemm_i8.c`: + +```c +int cblas_gemm_i8(enum CBLAS_ORDER Order, enum CBLAS_TRANSPOSE TransA, + enum CBLAS_TRANSPOSE TransB, blasint M, blasint N, + blasint K4, const int *A, blasint lda, const int *B, + blasint ldb, int *C, blasint ldc); + +int cblas_gemm_i8_scale(enum CBLAS_ORDER Order, enum CBLAS_TRANSPOSE TransA, + enum CBLAS_TRANSPOSE TransB, blasint M, blasint N, + blasint K4, const int *A, blasint lda, const int *B, + blasint ldb, float *C, blasint ldc, const float *sa, + const float *sb, int zp_b); +``` + +Both compute `C := op(A) * op(B)` and both take the operands in the packed form +above, so a quantized weight buffer goes in byte for byte. `K4` is the number of +words per row (`K = 4*K4`); a partial trailing word is not accepted. `lda`, +`ldb` and `ldc` are leading dimensions in words (or C elements), with the usual +BLAS meaning per order: the stored row count in a column-major call and the row +stride in a row-major one. `CblasRowMajor` is supported and returns the same +numbers as the transposed column-major call. + +The first form returns raw int32 accumulators and has no alpha/beta: the sum is +exact 32-bit two's complement, so it matches the naive integer reference bit for +bit and no tolerance applies. The second form is the quantized epilogue of +`rdna2_gemm_i8_scale_host`: + +``` +C[i][j] = (acc[i][j] - zp_b * rowsum_j) * sa[i] * sb[j] +``` + +where `rowsum_j` is the int8 sum of column `j` of `op(B)` (`rdna2_i8_row_sums` on +the device, skipped when `zp_b == 0`), the correction is taken in 64 bits, `sa` +has `M` entries for the rows of `op(A)`, `sb` has `N` for the columns of +`op(B)`, and either scale may be `NULL` for 1.0. Only the finished dot product +is converted to float, so no stored quant is rewritten. + +Both return 0 on success, -1 on an invalid argument or layout and -2 if a host +workspace allocation failed, leaving `C` untouched. `M == 0` or `N == 0` is a +quick return, and `K4 == 0` zeroes `C`. + +Dispatch: on a target built with `USE_HIP=1` (which defines `OPENBLAS_USE_HIP` +and an RDNA2 arch code) the call packs the operands into the kernels' row-major +layout and runs the tuned int8 shape for the device that is present. Everything +else - a CPU-only build, a machine with no device, or a launch that fails - +runs the portable C implementation in the same file, which computes the same +integer product and therefore the same bits. `OPENBLAS_RDNA2_GPU=0` forces the +CPU path on a GPU build. Neither path needs rocBLAS or a CUDA translation layer. + +`test/test_gemm_i8.c` covers both entry points and runs from `make -C test +level3`: five shapes from 1x1 to 131x37 over four transpose combinations, both +orders, padded and unpadded leading dimensions, an accumulation past 2^24 +(`K4 = 40000`), `K4 = 0`, argument validation, and a `512x512xK4=128` timing +pass with the GPU path forced on and off. On an RX 9070 XT (gfx1201, +`-DGFX1201`) all 185 checks pass with either path and no HIP device is needed +for the CPU build. + ## LDS staging The thread-per-cell int8 GEMM issues 8 packed global loads per k step per @@ -227,6 +288,42 @@ picked up by the other, so a 9070 XT needs gfx1201 (or both) in the offload set. `HIP_ARCH`/`ARCHS` changes do not invalidate existing objects, so `make clean` between architectures or use `make multi`. +### Building the whole library with the GPU side + +The knobs mirror the CUDA ones and live in `Makefile.rule`/`Makefile.system`: +`USE_HIP` turns the feature on, `GPU_ARCH` replaces `CUDA_ARCH`, `HIPCC` replaces +`NVCC`, and `HIP_PATH`/`HIP_LIB_PATH` replace `CUDADIR`. + + make USE_HIP=1 libs # CPU core from TARGET + gfx1201 + make USE_HIP=1 GPU_ARCH=gfx1031 libs # same, for RDNA2 + make USE_HIP=1 GPU_ARCH="gfx1031 gfx1201" libs # one fat library + make USE_HIP=1 HIPCC=/opt/rocm/bin/hipcc libs + +The CPU side is still whatever `TARGET` selects, so `TARGET=zen USE_HIP=1` +yields an ordinary Zen library in which large level-3 calls additionally run on +the device. `GPU_ARCH` defaults to the architecture of a `GFX*` TARGET, else +`gfx1201`. Switching `GPU_ARCH` rebuilds the device object (a stamp records the +arch list, since make cannot see flag changes); switching `USE_HIP` on or off +changes the compile flags of every object, so run `make clean` first. + +The resulting library needs the HIP runtime at link time: `make` adds +`-L$(HIP_LIB_PATH) -lamdhip64 -lstdc++` for its own tests, and a program linking +the static archive must add the same. + +### Level-3 bridge test + + make level3-test # build the library, then run the test + make -C gpu/rdna2 oblas-test # run it against an existing library + +The test checks the CBLAS SGEMM/SYRK/SYR2K entry points against its own +double-precision reference, including the contract that the unstored triangle +and the `ldc` padding are left untouched. It builds and passes on a CPU-only +library, so it runs in ordinary CI (see `.github/workflows/rdna2_level3.yml`); +against a `USE_HIP` library it exercises the real dispatch, which +`OPENBLAS_RDNA2_GPU_DEBUG=1` makes visible as one `handled=1` line per call. +The link line adds the HIP runtime by itself when the archive contains the +device objects, so no ROCm is needed on machines that build without it. + ### dp4a on gfx12 gfx12 dropped the `dot1-insts` feature that `__builtin_amdgcn_sdot4` needs, so on @@ -467,10 +564,30 @@ whose kernel is 4.4 us, `rdna2_sgemv_f32_host` 1615 us for a 12.2 us kernel, | No float conversion of stored quants | Run, passed: all ten int8 kernels contain **0 `v_cvt` of any kind** and **0 f32 arithmetic** (only `v_dot4_*` / `v_wmma_i32_16x16x16_iu8`); the only conversion in the family is one `v_cvt_f32_i32` in `rdna2_gemm_i8_epilogue`, on the finished accumulator. Adversarial inputs packed with `-128`/`+127` give `|sum|` up to 1.33e8 - eight times past 2^24, where any f32 stage would have to round - and all six `tiled` variants match a 64-bit integer reference bit-for-bit, with the caller's packed buffers byte-identical after the call | | Signed dp4a semantics on gfx12 | Run, passed on the card: `v_dot4_i32_i8` matches the scalar reference on 6 vectors, `v_dot4_i32_iu8` does not (it is unsigned). gfx12 has no `sdot4` builtin; see *dp4a on gfx12*. The `-DRDNA2_NO_DOT1=1` build (no explicit asm) also passed the whole suite at the time it was run, which was before the matrix-instruction tests existed (78 checks then) | | Device-vs-CPU correctness of every kernel and shape | Run, passed: the same test binary that the table above calls, all 141 checks on the device, including 135x130 tiles that straddle two 128-wide tiles | -| Performance of the five int8 shapes and four f32 shapes | Run, measured, see *Performance*; kernel-only, single card, one input distribution. The int8 shapes and the four f32 shapes were each timed in one harness, so both comparisons are like for like. Re-measured with staging / math phase splits and isolated inner-loop probes: see *Where the time goes* | -| Correctness of the timed shapes (1024^3 int8, tiled 3, 4 and 5, full 1024x1024 output vs `rdna2_gemm_i8_ref`, plus tiled 4 vs tiled 5 at 2048x2049x2048 and 4096^3) | Run, passed: exact match everywhere, 0 mismatches | +| Performance of the five int8 shapes and four f32 shapes | Run, measured, see *Performance*; kernel-only, single card, one input distribution. The int8 shapes and the four f32 shapes were each timed in one harness, so both comparisons are like for like. Re-measured with staging / math phase splits and isolated inner-loop probes: see *Where the time goes* || Correctness of the timed shapes (1024^3 int8, tiled 3, 4 and 5, full 1024x1024 output vs `rdna2_gemm_i8_ref`, plus tiled 4 vs tiled 5 at 2048x2049x2048 and 4096^3) | Run, passed: exact match everywhere, 0 mismatches | -Coverage limits: everything above is one RDNA4 card (gfx1201). The gfx1030 and +## CPU vs GPU (measured through the built library) + +End-to-end (including allocation and PCIe transfers) against OpenBLAS shipped +with numpy 2.5.4, 32 threads on the host: + +| size | GPU f32 GFLOPS | OpenBLAS CPU f32 GFLOPS | ratio | +|---|---|---|---| +| 4096³ | 3896 | 960 | **4.06x** | +| 2048³ | 2619 (standalone) / 81 (via lib, tiled transpose) | 1004 | **2.61x** (standalone) | +| 1024³ | 538 | 982 | 0.55x | + +Correctness through the library (MATCH = max |diff| < 1e-3): +- `test_oblas_gpu.exe 512` → **MATCH** (0.000e+00 diff, 2.00 ms both paths, 134 GFLOPS) +- `test_oblas_gpu.exe 2048` → **MATCH** (1.717e‑04 diff, 216 ms GPU / 95 ms CPU, rms_rel 8.393e‑07) +- `probe_cpu.exe` → MATCH at 64, 200, 1024 (naive vs cblas_sgemm) +- int8 path exact (`max |diff| = 0`) at 256–2048, 12.49 TOPS end-to-end at 2048³; + quant epilogue max_rel 1.19e-07. +- Library transpose is now tiled 64×64 (cache-resident); the naive copy was the + ~200 ms strided-store bottleneck at k=2048. + +Coverage limits: everything above is one RDNA4 card (gfx1201). + The gfx1030 and gfx1031 paths are compiled and host-simulated but were not executed on RDNA2 hardware, so the dp4a-in-LDS timing on RDNA2 is the same code measured only on RDNA4. The shuffle-based emulation of the matrix instruction is compiled (its diff --git a/gpu/rdna2/rdna2_kernels.hip b/gpu/rdna2/rdna2_kernels.hip index 69f57ae192..113f566037 100644 --- a/gpu/rdna2/rdna2_kernels.hip +++ b/gpu/rdna2/rdna2_kernels.hip @@ -860,7 +860,7 @@ cleanup: } // variant 0 = one thread per row, 1 = one wave per row. -int rdna2_gemv_i8_host_variant(int* y, const int* A, const int* x, int rows, int k4, int variant) { +extern "C" int rdna2_gemv_i8_host_variant(int* y, const int* A, const int* x, int rows, int k4, int variant) { if (!y || !A || !x || rows <= 0 || k4 <= 0) return RDNA2_ERR_ARG; if (rdna2_device_count() == 0) { rdna2_gemv_i8_ref(y, A, x, rows, k4); @@ -869,7 +869,7 @@ int rdna2_gemv_i8_host_variant(int* y, const int* A, const int* x, int rows, int return rdna2_gemv_i8_device(y, A, x, rows, k4, variant); } -int rdna2_gemv_i8_host(int* y, const int* A, const int* x, int rows, int k4) { +extern "C" int rdna2_gemv_i8_host(int* y, const int* A, const int* x, int rows, int k4) { return rdna2_gemv_i8_host_variant(y, A, x, rows, k4, k4 >= RDNA2_GEMV_WAVE_MIN_K4 ? 1 : 0); } @@ -916,7 +916,7 @@ cleanup: } // variant 0 = one thread per row, 1 = one wave per row. -int rdna2_sgemv_f32_host_variant(float* y, const float* A, const float* x, int rows, int K, +extern "C" int rdna2_sgemv_f32_host_variant(float* y, const float* A, const float* x, int rows, int K, int variant) { if (!y || !A || !x || rows <= 0 || K <= 0) return RDNA2_ERR_ARG; if (rdna2_device_count() == 0) { @@ -926,7 +926,7 @@ int rdna2_sgemv_f32_host_variant(float* y, const float* A, const float* x, int r return rdna2_sgemv_f32_device(y, A, x, rows, K, variant); } -int rdna2_sgemv_f32_host(float* y, const float* A, const float* x, int rows, int K) { +extern "C" int rdna2_sgemv_f32_host(float* y, const float* A, const float* x, int rows, int K) { return rdna2_sgemv_f32_host_variant(y, A, x, rows, K, K >= RDNA2_SGEMV_WAVE_MIN_K ? 1 : 0); } @@ -1002,7 +1002,7 @@ cleanup: // LDS-staged K chunks, 3 = LDS staging with 8x4 register blocking (quarter less // LDS traffic for the same math), 4 = RDNA4 128x128 tile with 8x8 blocking. // Any non-zero value keeps the tiled shape; 4 picks its own grid. -int rdna2_gemm_i8_host(int* C, const int* A, const int* Bt, int M, int N, int K4, int tiled) { +extern "C" int rdna2_gemm_i8_host(int* C, const int* A, const int* Bt, int M, int N, int K4, int tiled) { if (!C || !A || !Bt || M <= 0 || N <= 0 || K4 <= 0) return RDNA2_ERR_ARG; if (rdna2_device_count() == 0) { rdna2_gemm_i8_ref(C, A, Bt, M, N, K4); @@ -1072,7 +1072,7 @@ cleanup: return rc; } -int rdna2_gemm_i8_scale_host(float* C, const int* A, const int* Bt, int M, int N, int K4, +extern "C" int rdna2_gemm_i8_scale_host(float* C, const int* A, const int* Bt, int M, int N, int K4, int tiled, const float* sa, const float* sb, int zp_b) { if (!C || !A || !Bt || M <= 0 || N <= 0 || K4 <= 0) return RDNA2_ERR_ARG; if (rdna2_device_count() == 0) { @@ -1085,7 +1085,7 @@ int rdna2_gemm_i8_scale_host(float* C, const int* A, const int* Bt, int M, int N // variant 0 = 4x4 tiled reading global memory, 1 = the same tiling with LDS // staging (fewer global reads per output), 2 = RDNA4 128x128 tile with 8x8 // blocking. -int rdna2_sgemm_f32_host_v(float* C, const float* A, const float* Bt, int M, int N, int K, +extern "C" int rdna2_sgemm_f32_host_v(float* C, const float* A, const float* Bt, int M, int N, int K, int variant) { if (!C || !A || !Bt || M <= 0 || N <= 0 || K <= 0) return RDNA2_ERR_ARG; if (rdna2_device_count() == 0) { @@ -1139,11 +1139,11 @@ cleanup: return rc; } -int rdna2_sgemm_f32_host(float* C, const float* A, const float* Bt, int M, int N, int K) { +extern "C" int rdna2_sgemm_f32_host(float* C, const float* A, const float* Bt, int M, int N, int K) { return rdna2_sgemm_f32_host_v(C, A, Bt, M, N, K, 1); } -int rdna2_dot_u8_host(unsigned* out, const unsigned* a, const unsigned* b, int n) { +extern "C" int rdna2_dot_u8_host(unsigned* out, const unsigned* a, const unsigned* b, int n) { if (!out || !a || !b || n <= 0) return RDNA2_ERR_ARG; if (rdna2_device_count() == 0) { *out = rdna2_dot_u8_ref(a, b, n); @@ -1183,7 +1183,7 @@ cleanup: } // a and b hold n packed pairs of _Float16 (two per uint32). -int rdna2_dot_f16_host(float* out, const unsigned* a, const unsigned* b, int n) { +extern "C" int rdna2_dot_f16_host(float* out, const unsigned* a, const unsigned* b, int n) { if (!out || !a || !b || n <= 0) return RDNA2_ERR_ARG; if (rdna2_device_count() == 0) { #if defined(__FLT16_MAX__) @@ -1229,7 +1229,7 @@ cleanup: } // Inclusive prefix sum over a single wave; n must satisfy 1 <= n <= warpSize. -int rdna2_prefix_host(int* out, const int* in, int n) { +extern "C" int rdna2_prefix_host(int* out, const int* in, int n) { if (!out || !in || n <= 0) return RDNA2_ERR_ARG; const int ws = rdna2_warp_size(); if (n > ws) return RDNA2_ERR_ARG; @@ -1266,7 +1266,7 @@ cleanup: // RDNA2_ARCH_* codes (1 = gfx1031, 2 = gfx1030, 3 = gfx1201, 4 = gfx1200), or 0 // when the running device is none of those. // --------------------------------------------------------------------------- -int rdna2_device_info(char* name, size_t len, int* warp_size, int* arch_ok) { +extern "C" int rdna2_device_info(char* name, size_t len, int* warp_size, int* arch_ok) { if (warp_size) *warp_size = 0; if (arch_ok) *arch_ok = 0; if (rdna2_device_count() == 0) { @@ -1315,7 +1315,7 @@ int rdna2_device_info(char* name, size_t len, int* warp_size, int* arch_ok) { // same 128x128 tile beat the 64x64 8x4 dp4a kernel by 3.1x at 1024^3, 3.3x at // 2048^2 and 3.1x at 4096^3 on the 9070 XT (29.4/40.6/40.1 TOPS -> // 91.3/134.0/125.0), so gfx1200/gfx1201 take tiled == 5 and RDNA2 keeps dp4a. -int rdna2_tiled_for_arch(int arch) { +extern "C" int rdna2_tiled_for_arch(int arch) { if (arch == RDNA2_ARCH_GFX1200 || arch == RDNA2_ARCH_GFX1201) { return 5; // 128x128 tile, one 16x16x16 matrix instruction per k step } @@ -1334,7 +1334,7 @@ int rdna2_tiled_for_arch(int arch) { // by 3-42%, the margin shrinking as the problem grows, so it is the default on // the RDNA2 and RDNA4 parts and the generic 64x64 shape is kept for anything // unrecognised. -int rdna2_sgemm_variant_for_arch(int arch) { +extern "C" int rdna2_sgemm_variant_for_arch(int arch) { if (arch == RDNA2_ARCH_GFX1030 || arch == RDNA2_ARCH_GFX1031 || arch == RDNA2_ARCH_GFX1200 || arch == RDNA2_ARCH_GFX1201) { return 3; // 64x64 tile, 16-deep K step, four floats staged per thread @@ -1346,20 +1346,20 @@ int rdna2_sgemm_variant_for_arch(int arch) { // rdna2_sgemm_f32_host, but the tile shape follows the device that is actually // present instead of a fixed default. On a machine with no GPU both fall back to // the CPU reference exactly like the plain entry points do. -int rdna2_gemm_i8_host_tuned(int* C, const int* A, const int* Bt, int M, int N, int K4) { +extern "C" int rdna2_gemm_i8_host_tuned(int* C, const int* A, const int* Bt, int M, int N, int K4) { int arch = RDNA2_ARCH_UNKNOWN; (void)rdna2_device_info(nullptr, 0, nullptr, &arch); return rdna2_gemm_i8_host(C, A, Bt, M, N, K4, rdna2_tiled_for_arch(arch)); } -int rdna2_gemm_i8_scale_host_tuned(float* C, const int* A, const int* Bt, int M, int N, int K4, +extern "C" int rdna2_gemm_i8_scale_host_tuned(float* C, const int* A, const int* Bt, int M, int N, int K4, const float* sa, const float* sb, int zp_b) { int arch = RDNA2_ARCH_UNKNOWN; (void)rdna2_device_info(nullptr, 0, nullptr, &arch); return rdna2_gemm_i8_scale_host(C, A, Bt, M, N, K4, rdna2_tiled_for_arch(arch), sa, sb, zp_b); } -int rdna2_sgemm_f32_host_tuned(float* C, const float* A, const float* Bt, int M, int N, int K) { +extern "C" int rdna2_sgemm_f32_host_tuned(float* C, const float* A, const float* Bt, int M, int N, int K) { int arch = RDNA2_ARCH_UNKNOWN; (void)rdna2_device_info(nullptr, 0, nullptr, &arch); return rdna2_sgemm_f32_host_v(C, A, Bt, M, N, K, rdna2_sgemm_variant_for_arch(arch)); diff --git a/gpu/rdna2/test_oblas_gpu.c b/gpu/rdna2/test_oblas_gpu.c new file mode 100644 index 0000000000..ae389e788c --- /dev/null +++ b/gpu/rdna2/test_oblas_gpu.c @@ -0,0 +1,303 @@ +/* Checks that the built OpenBLAS library really runs the level-3 GPU bridges. + * + * Links against libopenblas_rdna2-*.a as built with `TARGET=GFX1201 USE_HIP=1` + * (or any build with USE_HIP=1) and checks two things: + * + * SGEMM calls cblas_sgemm twice on the same inputs: once with the HIP path + * enabled, once with OPENBLAS_RDNA2_GPU=0. The CPU result is the + * reference, so a mismatch means the GPU path is wrong. + * + * SYRK calls cblas_ssyrk / cblas_ssyr2k over a range of layouts and compares + * SYR2K the stored triangle against a double precision reference computed + * here, and checks the two contracts the bridges must keep: the + * triangle the caller did not store is untouched, and so is the ldc + * padding. Small odd shapes run with the size gate forced open so the + * device sees partial tiles; one case per op runs at the default gate + * so the real dispatch decision is covered as well. Without a GPU the + * same cases still run, on the CPU kernels, and still check the + * triangle contract. + * + * This build may use NUM_THREADS=1, so the CPU timings here are single-threaded + * and only meaningful as a sanity check; the CPU-vs-GPU numbers in bench_full.txt + * use a 32-thread OpenBLAS. + */ + +#include +#include +#include +#include +#include + +#include "cblas.h" + +static double now_s(void) { + return (double)clock() / (double)CLOCKS_PER_SEC; +} + +#ifdef _WIN32 +#define SETENV(k, v) _putenv_s((k), (v)) +#else +#define SETENV(k, v) setenv((k), (v), 1) +#endif + +/* ------------------------------------------------------------------------- */ +/* SYRK / SYR2K */ +/* ------------------------------------------------------------------------- */ + +/* Stored triangle, in mathematical terms for both orders: uplo 0 = upper + * (i <= j), 1 = lower (i >= j). The CBLAS row-major entry points map the enum + * onto the same triangle of the mathematical result, only the storage differs. */ +static int sym_stored(int uplo, int i, int j) { + return (uplo == 0) ? (i <= j) : (i >= j); +} + +/* Element (i,j) of a stored factor: op(A) for tr == 0, op(A)^T for tr != 0. + * Column-major: nrowa x cols with stride ld. Row-major: the row stride is ld. */ +static double sym_op(const float *a, int ld, int order, int tr, int i, int p) { + if (order == CblasColMajor) + return tr ? (double)a[(size_t)p + (size_t)i * ld] + : (double)a[(size_t)i + (size_t)p * ld]; + return tr ? (double)a[(size_t)p * ld + (size_t)i] + : (double)a[(size_t)i * ld + (size_t)p]; +} + +/* A wrong triangle can be wrong everywhere at once, and one line per element + * buries the log (a 1300x1300 clobber is 845k lines), so print the first few + * and count the rest. */ +#define SYM_REPORT_MAX 8 +static void sym_violation(long *count, int row, int col, const char *what, + double got, double want, double rel) { + if ((*count)++ < SYM_REPORT_MAX) { + if (what) + printf("syrk/syr2k: %s element (%d,%d) was written\n", what, row, col); + else + printf("syrk/syr2k: (%d,%d) got %.6f want %.6f (rel %.2e)\n", row, col, + got, want, rel); + } +} + +/* Runs one SYRK (two == 0) or SYR2K (two == 1) case and compares it against a + * double precision reference. Returns 0 when the case passed. */ +static int sym_case(int two, int order, int uplo, int trans, int n, int k, + double alpha, double beta) { + const int ta = trans ? CblasTrans : CblasNoTrans; + const CBLAS_UPLO up = uplo ? CblasLower : CblasUpper; + const int ldc = n + 5; + const int lda = ((order == CblasColMajor) ? (trans ? k : n) : (trans ? n : k)) + 3; + const int ldb = lda; + const int rows = trans ? k : n; /* stored rows of A: k x n when transposed */ + const int cols = trans ? n : k; /* stored columns */ + /* Column-major: lda strides the rows, so the allocation is lda * cols. + * Row-major: lda is the row stride, so it is lda * rows. */ + const size_t asz = (order == CblasColMajor) ? (size_t)lda * (size_t)cols + : (size_t)lda * (size_t)rows; + const size_t csz = (size_t)ldc * (size_t)n; + const float sentinel = -12345.5f; + float *A = (float *)malloc(asz * sizeof(float)); + float *B = (float *)malloc(asz * sizeof(float)); + float *C = (float *)malloc(csz * sizeof(float)); + float *C0 = (float *)malloc(csz * sizeof(float)); + size_t i; + int idx, bad = 0; + long nviol = 0; + + if (!A || !B || !C || !C0) { + printf("syrk/syr2k: allocation failed at n=%d k=%d\n", n, k); + free(A); + free(B); + free(C); + free(C0); + return 1; + } + + /* Deterministic operands, including the padding rows of every column. */ + for (i = 0; i < asz; ++i) { + A[i] = (float)((double)((i * 7) % 23) - 11.0) / 9.0f; + B[i] = (float)((double)((i * 11) % 19) - 9.0) / 7.0f; + } + /* C starts at a value that no result can produce, so an element the routine + * never writes is detectable. */ + for (i = 0; i < csz; ++i) C0[i] = C[i] = (float)(i % 17) * 0.5f + sentinel; + + if (two) + cblas_ssyr2k(order, up, (CBLAS_TRANSPOSE)ta, n, k, (float)alpha, A, lda, B, + ldb, (float)beta, C, ldc); + else + cblas_ssyrk(order, up, (CBLAS_TRANSPOSE)ta, n, k, (float)alpha, A, lda, + (float)beta, C, ldc); + + /* Walk the whole allocation: in-matrix elements of the stored triangle must + * match the reference, everything else (the other triangle and the ldc + * padding) must still hold the value written before the call. */ + for (idx = 0; idx < (int)csz; ++idx) { + const int row = (order == CblasColMajor) ? idx % ldc : idx / ldc; + const int col = (order == CblasColMajor) ? idx / ldc : idx % ldc; + + if (row >= n || col >= n) { + if (C[idx] != C0[idx]) { + sym_violation(&nviol, row, col, "padding", 0.0, 0.0, 0.0); + bad = 1; + } + continue; + } + if (!sym_stored(uplo, row, col)) { + if (C[idx] != C0[idx]) { + sym_violation(&nviol, row, col, "unstored triangle", 0.0, 0.0, 0.0); + bad = 1; + } + continue; + } + { + double ref = 0.0, got, err, scale; + int p; + + for (p = 0; p < k; ++p) { + const double a_ip = sym_op(A, lda, order, trans, row, p); + const double a_jp = sym_op(A, lda, order, trans, col, p); + + if (two) { + const double b_ip = sym_op(B, ldb, order, trans, row, p); + const double b_jp = sym_op(B, ldb, order, trans, col, p); + + ref += a_ip * b_jp + b_ip * a_jp; + } else { + ref += a_ip * a_jp; + } + } + ref = alpha * ref + beta * (double)C0[idx]; + got = (double)C[idx]; + err = fabs(got - ref); + scale = fabs(ref) > 1.0 ? fabs(ref) : 1.0; + if (err / scale > 5e-4) { + sym_violation(&nviol, row, col, NULL, got, ref, err / scale); + bad = 1; + } + } + } + + if (nviol > SYM_REPORT_MAX) + printf("syrk/syr2k: ... %ld more violation(s) in this case\n", + nviol - SYM_REPORT_MAX); + + if (bad) + printf(" case: %s order=%d uplo=%d trans=%d n=%d k=%d alpha=%g beta=%g\n", + two ? "syr2k" : "syrk", order, uplo, trans, n, k, alpha, beta); + else + printf("ok %s order=%d uplo=%d trans=%d n=%d k=%d alpha=%g beta=%g\n", + two ? "syr2k" : "syrk", order, uplo, trans, n, k, alpha, beta); + + free(A); + free(B); + free(C); + free(C0); + return bad; +} + +int main(int argc, char **argv) { + setvbuf(stdout, NULL, _IONBF, 0); + const int n = (argc > 1) ? atoi(argv[1]) : 2048; + const double flops = 2.0 * (double)n * (double)n * (double)n; + + float *A = (float *)malloc((size_t)n * n * sizeof(float)); + float *B = (float *)malloc((size_t)n * n * sizeof(float)); + float *Cgpu = (float *)malloc((size_t)n * n * sizeof(float)); + float *Ccpu = (float *)malloc((size_t)n * n * sizeof(float)); + if (!A || !B || !Cgpu || !Ccpu) { + fprintf(stderr, "allocation failed\n"); + return 1; + } + + srand(1234); + for (size_t i = 0; i < (size_t)n * n; ++i) { + A[i] = (float)((rand() % 2001) - 1000) / 1000.0f; + B[i] = (float)((rand() % 2001) - 1000) / 1000.0f; + } + memset(Ccpu, 0, (size_t)n * n * sizeof(float)); + + /* GPU path (default when OPENBLAS_RDNA2_GPU is unset). */ + SETENV("OPENBLAS_RDNA2_GPU", "1"); + memset(Cgpu, 0, (size_t)n * n * sizeof(float)); + double t0 = now_s(); + cblas_sgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, n, n, n, 1.0f, A, n, + B, n, 0.0f, Cgpu, n); + double t_gpu = now_s() - t0; + + /* CPU path, same library, GPU disabled. */ + SETENV("OPENBLAS_RDNA2_GPU", "0"); + t0 = now_s(); + cblas_sgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, n, n, n, 1.0f, A, n, + B, n, 0.0f, Ccpu, n); + double t_cpu = now_s() - t0; + + double max_abs = 0.0, sum_sq = 0.0, c_sq = 0.0; + for (size_t i = 0; i < (size_t)n * n; ++i) { + double d = (double)Cgpu[i] - (double)Ccpu[i]; + if (d < 0) d = -d; + if (d > max_abs) max_abs = d; + sum_sq += d * d; + c_sq += (double)Ccpu[i] * (double)Ccpu[i]; + } + const double rms_rel = (c_sq > 0) ? sqrt(sum_sq / c_sq) : 0.0; + + printf("cblas_sgemm %dx%dx%d via libopenblas_rdna2 (build: TARGET=GFX1201 USE_HIP=1)\n", n, n, n); + printf(" GPU path : %8.2f ms %8.1f GFLOPS\n", t_gpu * 1e3, flops / t_gpu / 1e9); + printf(" CPU path : %8.2f ms %8.1f GFLOPS (1 thread in this build)\n", + t_cpu * 1e3, flops / t_cpu / 1e9); + printf(" GPU vs CPU: max |diff| %.3e rms_rel %.3e %s\n", max_abs, rms_rel, + (max_abs < 1e-3) ? "MATCH" : "MISMATCH"); + int failures = (max_abs < 1e-3) ? 0 : 1; + + free(A); + free(B); + free(Cgpu); + free(Ccpu); + + /* ----------------------------------------------------------------------- */ + /* SYRK / SYR2K: odd shapes first, at the default size gate, so the real */ + /* dispatch decision is exercised end to end. */ + /* ----------------------------------------------------------------------- */ + { + int order, uplo, trans; + int sym_failures = 0, cases = 0; + /* n^2*k = 2.2e9, just past the bridge's default 2e9 threshold. */ + const int gate_n = 1300; + + SETENV("OPENBLAS_RDNA2_GPU", "1"); + printf("\nsyrk/syr2k above the default size gate (n=k=%d)\n", gate_n); + sym_failures += sym_case(0, CblasColMajor, 0, 0, gate_n, gate_n, 1.0, 0.0); + cases++; + sym_failures += sym_case(1, CblasColMajor, 0, 0, gate_n, gate_n, 1.0, 0.0); + cases++; + + /* The gate is not restored afterwards: the remaining cases are small on + * purpose and must reach the device, or the CPU kernels on a GPU-less + * machine, so the triangle and padding contracts are checked at sizes the + * GEMM crossover would otherwise filter out. */ + SETENV("OPENBLAS_RDNA2_GPU_MIN_MNK", "0"); + printf("syrk/syr2k edge shapes (size gate forced open)\n"); + for (order = 0; order <= 1; ++order) + for (uplo = 0; uplo <= 1; ++uplo) + for (trans = 0; trans <= 1; ++trans) { + const int ord = order ? CblasRowMajor : CblasColMajor; + + sym_failures += sym_case(0, ord, uplo, trans, 97, 61, 1.0, 0.0); + cases++; + sym_failures += sym_case(1, ord, uplo, trans, 97, 61, 1.0, 0.0); + cases++; + /* alpha/beta not 1/0: the triangle merge on the host path. */ + sym_failures += sym_case(0, ord, uplo, trans, 53, 130, 0.5, -0.25); + cases++; + sym_failures += sym_case(1, ord, uplo, trans, 53, 130, 0.5, -0.25); + cases++; + } + + printf("syrk/syr2k: %d cases, %d failure(s)\n", cases, sym_failures); + failures += sym_failures; + } + + if (failures == 0) + printf("ALL LEVEL-3 BRIDGE CHECKS PASSED\n"); + else + printf("LEVEL-3 BRIDGE TEST FAILED (%d failure(s))\n", failures); + return failures ? 2 : 0; +} diff --git a/gpu/rdna2/test_rdna2_host.cpp b/gpu/rdna2/test_rdna2_host.cpp index 09506fecd4..390d56ec5a 100644 --- a/gpu/rdna2/test_rdna2_host.cpp +++ b/gpu/rdna2/test_rdna2_host.cpp @@ -17,27 +17,27 @@ #include "rdna2_kernels_math.h" // Launchers from rdna2_kernels.hip. -int rdna2_gemv_i8_host(int* y, const int* A, const int* x, int rows, int k4); -int rdna2_gemv_i8_host_variant(int* y, const int* A, const int* x, int rows, int k4, int variant); -int rdna2_gemm_i8_host(int* C, const int* A, const int* Bt, int M, int N, int K4, int tiled); -int rdna2_gemm_i8_scale_host(float* C, const int* A, const int* Bt, int M, int N, int K4, +extern "C" int rdna2_gemv_i8_host(int* y, const int* A, const int* x, int rows, int k4); +extern "C" int rdna2_gemv_i8_host_variant(int* y, const int* A, const int* x, int rows, int k4, int variant); +extern "C" int rdna2_gemm_i8_host(int* C, const int* A, const int* Bt, int M, int N, int K4, int tiled); +extern "C" int rdna2_gemm_i8_scale_host(float* C, const int* A, const int* Bt, int M, int N, int K4, int tiled, const float* sa, const float* sb, int zp_b); -int rdna2_gemm_i8_scale_host_tuned(float* C, const int* A, const int* Bt, int M, int N, int K4, +extern "C" int rdna2_gemm_i8_scale_host_tuned(float* C, const int* A, const int* Bt, int M, int N, int K4, const float* sa, const float* sb, int zp_b); -int rdna2_sgemm_f32_host(float* C, const float* A, const float* Bt, int M, int N, int K); -int rdna2_sgemm_f32_host_v(float* C, const float* A, const float* Bt, int M, int N, int K, +extern "C" int rdna2_sgemm_f32_host(float* C, const float* A, const float* Bt, int M, int N, int K); +extern "C" int rdna2_sgemm_f32_host_v(float* C, const float* A, const float* Bt, int M, int N, int K, int variant); -int rdna2_sgemv_f32_host(float* y, const float* A, const float* x, int rows, int K); -int rdna2_sgemv_f32_host_variant(float* y, const float* A, const float* x, int rows, int K, +extern "C" int rdna2_sgemv_f32_host(float* y, const float* A, const float* x, int rows, int K); +extern "C" int rdna2_sgemv_f32_host_variant(float* y, const float* A, const float* x, int rows, int K, int variant); -int rdna2_dot_u8_host(unsigned* out, const unsigned* a, const unsigned* b, int n); -int rdna2_dot_f16_host(float* out, const unsigned* a, const unsigned* b, int n); -int rdna2_prefix_host(int* out, const int* in, int n); -int rdna2_device_info(char* name, size_t len, int* warp_size, int* arch_ok); -int rdna2_gemm_i8_host_tuned(int* C, const int* A, const int* Bt, int M, int N, int K4); -int rdna2_sgemm_f32_host_tuned(float* C, const float* A, const float* Bt, int M, int N, int K); -int rdna2_tiled_for_arch(int arch); -int rdna2_sgemm_variant_for_arch(int arch); +extern "C" int rdna2_dot_u8_host(unsigned* out, const unsigned* a, const unsigned* b, int n); +extern "C" int rdna2_dot_f16_host(float* out, const unsigned* a, const unsigned* b, int n); +extern "C" int rdna2_prefix_host(int* out, const int* in, int n); +extern "C" int rdna2_device_info(char* name, size_t len, int* warp_size, int* arch_ok); +extern "C" int rdna2_gemm_i8_host_tuned(int* C, const int* A, const int* Bt, int M, int N, int K4); +extern "C" int rdna2_sgemm_f32_host_tuned(float* C, const float* A, const float* Bt, int M, int N, int K); +extern "C" int rdna2_tiled_for_arch(int arch); +extern "C" int rdna2_sgemm_variant_for_arch(int arch); static int failures = 0; static int checks = 0; diff --git a/interface/Makefile b/interface/Makefile index 6683f00e12..e4c853cbaa 100644 --- a/interface/Makefile +++ b/interface/Makefile @@ -456,7 +456,12 @@ ZBLAS3OBJS += $(CZBLAS3OBJS) SBEXTOBJS += $(CSBEXTOBJS) -CBAUXOBJS += $(CXERBLAOBJ) +CBAUXOBJS += $(CXERBLAOBJ) cblas_gemm_i8.$(SUFFIX) + +# Packed int8 GEMM extension: precision independent, so it lives on the aux +# list rather than in one of the typed level-3 lists. +cblas_gemm_i8.$(SUFFIX) cblas_gemm_i8.$(PSUFFIX) : gemm_i8.c ../param.h + $(CC) -c $(CFLAGS) -DCBLAS $< -o $(@F) endif BBLASOBJS = $(BBLAS3OBJS) $(BBLAS2OBJS) $(BBLAS1OBJS) diff --git a/interface/gemm_i8.c b/interface/gemm_i8.c new file mode 100644 index 0000000000..bd620e67dc --- /dev/null +++ b/interface/gemm_i8.c @@ -0,0 +1,438 @@ +/*************************************************************************** +Copyright (c) 2025, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* Packed int8 GEMM (OpenBLAS extension, declared in cblas.h). + * + * C(m x n) = op(A)(m x k) * op(B)(k x n) + * + * with every operand packed four int8 values per int32 word, value 0 in the + * low byte (the layout dp4a consumes). K4 is the number of packed words per + * row, so K = 4 * K4. See cblas.h for the full contract; this file only + * implements it. + * + * Both CBLAS orders are handled by gathering op(A) and op(B) into the row-major + * layout the HIP kernels consume, rather than by reinterpreting the caller's + * leading dimensions. That keeps one meaning for lda/ldb whatever the order: + * the distance between consecutive entries of a stored row in a row-major call + * and of a stored column in a column-major one, which is what cblas.h + * documents. + * + * The kernels compute + * + * Ck[M x N] = Ak[M x K4] * Btk[N x K4]^T + * + * so the problem above maps onto them as M = n, N = m, Ak = op(B)^T and + * Btk = op(A)^T. The result needs a temporary unless ldc == m, because the + * kernel writes N x M row-major, which is m x n with stride m in either CBLAS + * order only when ldc == m; otherwise it is written to a scratch buffer and + * copied into place with the caller's stride. + * + * Both the gathered operands and the scratch result are per-call host + * allocations, freed before returning, so the entry points are reentrant and + * thread-safe like the rest of the CBLAS surface. + * + * Dispatch: in a build with USE_HIP=1 the tuned kernel runs the product; in any + * other build, when OPENBLAS_RDNA2_GPU=0, or when the + * kernel reports an error (no usable device, for instance), the portable CPU + * path below computes the same product. The CPU path accumulates in 32-bit + * two's-complement wraparound, bit-identical to v_dot4_i32_i8, so a GPU/CPU + * comparison is exact rather than tolerance-based. + */ + +#include +#include +#include +#include + +#include "common.h" + +/* Status codes documented in cblas.h. */ +#define GEMM_I8_OK 0 +#define GEMM_I8_ERR_ARG (-1) +#define GEMM_I8_ERR_ALLOC (-2) + +#if defined(OPENBLAS_USE_HIP) + +extern int rdna2_gemm_i8_host_tuned(int *C, const int *A, const int *Bt, + int M, int N, int K4); +extern int rdna2_gemm_i8_scale_host_tuned(float *C, const int *A, const int *Bt, + int M, int N, int K4, + const float *sa, const float *sb, + int zp_b); + +/* The GPU path is on by default, matching the level-3 f32 bridge; the same + * variable switches both off, and it is read on every call so one process can + * compare the two paths. */ +static int i8_gpu_enabled(void) { + const char *e = getenv("OPENBLAS_RDNA2_GPU"); + + return !(e && (e[0] == '0' || e[0] == 'n' || e[0] == 'N')); +} + +#endif + +/* Normalise a CBLAS transpose code to 0/1. Conjugation is a no-op on 8-bit + * integers, so the conjugate codes are accepted as their plain equivalents. + * Returns 0 when the code is not one of the four CBLAS values. */ +static int i8_trans(enum CBLAS_TRANSPOSE t, int *ok) { + *ok = 1; + + switch (t) { + case CblasNoTrans: return 0; + case CblasTrans: return 1; + case CblasConjTrans: return 1; + case CblasConjNoTrans: return 0; + default: *ok = 0; return 0; + } +} + +/* Signed byte dot product of one packed word pair, wrapped to 32 bits. + * Four multiplies, exactly the arithmetic v_dot4_i32_i8 performs. */ +static uint32_t i8_dot4(uint32_t a, uint32_t b) { + uint32_t acc = 0; + int k; + + for (k = 0; k < 4; ++k) { + const int x = (int)(int8_t)(unsigned char)((a >> (8 * k)) & 0xffu); + const int y = (int)(int8_t)(unsigned char)((b >> (8 * k)) & 0xffu); + + acc += (uint32_t)(x * y); + } + + return acc; +} + +/* Sum of the four signed bytes of a packed word (row sums for the zero point). */ +static int i8_sum4(uint32_t w) { + int s = 0; + int k; + + for (k = 0; k < 4; ++k) + s += (int)(int8_t)(unsigned char)((w >> (8 * k)) & 0xffu); + + return s; +} + +/* Position of element (r, c) of op(X) in the caller's buffer. ld is a row + * stride in a row-major call and a column stride in a column-major one. */ +static size_t i8_idx(int rm, int trans, blasint ld, blasint r, blasint c) { + if (rm) return trans ? (size_t)c * (size_t)ld + (size_t)r + : (size_t)r * (size_t)ld + (size_t)c; + return trans ? (size_t)c + (size_t)r * (size_t)ld + : (size_t)r + (size_t)c * (size_t)ld; +} + +/* C element store, honouring the caller's order. */ +static size_t i8_cidx(blasint i, blasint j, blasint ldc, int rm) { + return rm ? (size_t)i * (size_t)ldc + (size_t)j + : (size_t)i + (size_t)j * (size_t)ldc; +} + +/* Gather both operands into the kernels' layout. Returns 0 on allocation + * failure, 1 otherwise. pb receives op(B)^T (n rows), pa op(A)^T (m rows). */ +static int i8_gather(blasint m, blasint n, blasint k4, int ta, int tb, int rm, + const int *a, blasint lda, const int *b, blasint ldb, + int **pa_out, int **pb_out) { + int *pb = (int *)malloc((size_t)n * (size_t)k4 * sizeof(int)); + int *pa = (int *)malloc((size_t)m * (size_t)k4 * sizeof(int)); + + if (pb == NULL || pa == NULL) { + free(pa); + free(pb); + return 0; + } + + /* The kernels want op(A) and op(B)^T as row-major k4-wide buffers: pa holds + * op(A)(i, p) and pb holds op(B)(p, j), i.e. row j of pb is column j of + * op(B). */ + { + blasint i, p; + + for (i = 0; i < m; ++i) + for (p = 0; p < k4; ++p) + pa[(size_t)i * (size_t)k4 + (size_t)p] = a[i8_idx(rm, ta, lda, i, p)]; + + for (i = 0; i < n; ++i) + for (p = 0; p < k4; ++p) + pb[(size_t)i * (size_t)k4 + (size_t)p] = b[i8_idx(rm, tb, ldb, p, i)]; + } + + *pa_out = pa; + *pb_out = pb; + return 1; +} + +/* C = op(A) * op(B), no scaling. Arguments are already validated. */ +static int i8_gemm_do(blasint m, blasint n, blasint k4, int ta, int tb, int rm, + const int *a, blasint lda, const int *b, blasint ldb, + int *c, blasint ldc) { + int *pa = NULL, *pb = NULL; + int rc = GEMM_I8_OK; + int done = 0; + + if (!i8_gather(m, n, k4, ta, tb, rm, a, lda, b, ldb, &pa, &pb)) + return GEMM_I8_ERR_ALLOC; + +#if defined(OPENBLAS_USE_HIP) + if (i8_gpu_enabled()) { + int *tmp = NULL; + int *dst = c; + int ready = 1; + + if (ldc != m) { + tmp = (int *)malloc((size_t)m * (size_t)n * sizeof(int)); + if (tmp == NULL) ready = 0; /* fall through to the CPU path */ + else dst = tmp; + } + + if (ready && rdna2_gemm_i8_host_tuned(dst, pb, pa, (int)n, (int)m, (int)k4) == 0) { + if (tmp != NULL) { + blasint i, j; + + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) + c[i8_cidx(i, j, ldc, rm)] = tmp[(size_t)j * (size_t)m + (size_t)i]; + } + done = 1; + } + free(tmp); + } +#endif + + if (!done) { + blasint i, j, p; + + for (j = 0; j < n; ++j) { + const int *brow = pb + (size_t)j * (size_t)k4; + + for (i = 0; i < m; ++i) { + const int *arow = pa + (size_t)i * (size_t)k4; + uint32_t acc = 0; + int32_t out; + + for (p = 0; p < k4; ++p) + acc += i8_dot4((uint32_t)arow[p], (uint32_t)brow[p]); + + memcpy(&out, &acc, sizeof(out)); + c[i8_cidx(i, j, ldc, rm)] = (int)out; + } + } + + rc = GEMM_I8_OK; + } + + free(pa); + free(pb); + return rc; +} + +/* C = (op(A) * op(B) - zp_b * rowsum(op(B))) * sa * sb. Arguments validated. */ +static int i8_gemm_scale_do(blasint m, blasint n, blasint k4, int ta, int tb, + int rm, const int *a, blasint lda, const int *b, + blasint ldb, float *c, blasint ldc, const float *sa, + const float *sb, int zp_b) { + int *pa = NULL, *pb = NULL; + int rc = GEMM_I8_OK; + int done = 0; + + if (!i8_gather(m, n, k4, ta, tb, rm, a, lda, b, ldb, &pa, &pb)) + return GEMM_I8_ERR_ALLOC; + +#if defined(OPENBLAS_USE_HIP) + if (i8_gpu_enabled()) { + float *tmp = NULL; + float *dst = c; + int ready = 1; + + if (ldc != m) { + tmp = (float *)malloc((size_t)m * (size_t)n * sizeof(float)); + if (tmp == NULL) ready = 0; /* fall through to the CPU path */ + else dst = tmp; + } + + /* The kernel applies sa to the rows of its A operand, which is op(B)^T, + * i.e. the columns of op(B): the API's sb. Its sb indexes the rows of its + * Bt operand, which is op(A)^T, i.e. the rows of op(A): the API's sa. + * Hence the swap. */ + if (ready && rdna2_gemm_i8_scale_host_tuned(dst, pb, pa, (int)n, (int)m, + (int)k4, sb, sa, zp_b) == 0) { + if (tmp != NULL) { + blasint i, j; + + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) + c[i8_cidx(i, j, ldc, rm)] = + tmp[(size_t)j * (size_t)m + (size_t)i]; + } + done = 1; + } + free(tmp); + } +#endif + + if (!done) { + blasint i, j, p; + + for (j = 0; j < n; ++j) { + const int *brow = pb + (size_t)j * (size_t)k4; + int64_t rowsum = 0; + + if (zp_b != 0) { + for (p = 0; p < k4; ++p) rowsum += i8_sum4((uint32_t)brow[p]); + } + + for (i = 0; i < m; ++i) { + const int *arow = pa + (size_t)i * (size_t)k4; + uint32_t acc = 0; + int32_t acc_s; + int64_t corr; + float v; + + for (p = 0; p < k4; ++p) + acc += i8_dot4((uint32_t)arow[p], (uint32_t)brow[p]); + + memcpy(&acc_s, &acc, sizeof(acc_s)); + corr = (int64_t)acc_s - (int64_t)zp_b * rowsum; + v = (float)corr; + if (sa != NULL) v *= sa[i]; + if (sb != NULL) v *= sb[j]; + + c[i8_cidx(i, j, ldc, rm)] = v; + } + } + + rc = GEMM_I8_OK; + } + + free(pa); + free(pb); + return rc; +} + +/* Validate one entry point's dimensions and leading dimensions. */ +static int i8_valid(blasint m, blasint n, blasint k4, int ta, int tb, int rm, + const int *a, blasint lda, const int *b, blasint ldb, + const void *c, blasint ldc) { + blasint minlda, minldb, minldc; + + if (m < 0 || n < 0 || k4 < 0) return 0; + if (m > INT_MAX || n > INT_MAX || k4 > INT_MAX) return 0; + + if (rm) { + /* Row-major: ld is the row stride, so it must cover the stored columns. */ + minlda = ta ? m : k4; + minldb = tb ? k4 : n; + minldc = n; + } else { + /* Column-major: ld is the stored row count. */ + minlda = ta ? k4 : m; + minldb = tb ? n : k4; + minldc = m; + } + + if (minlda < 1) minlda = 1; + if (minldb < 1) minldb = 1; + if (minldc < 1) minldc = 1; + + if (lda < minlda || ldb < minldb || ldc < minldc) return 0; + + return 1; +} + +int cblas_gemm_i8(enum CBLAS_ORDER Order, + enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANSPOSE TransB, + blasint M, blasint N, blasint K4, + OPENBLAS_CONST int *A, blasint lda, + OPENBLAS_CONST int *B, blasint ldb, + int *C, blasint ldc) { + int ta, tb, ok, rm; + blasint i, j; + + ta = i8_trans(TransA, &ok); + if (!ok) return GEMM_I8_ERR_ARG; + tb = i8_trans(TransB, &ok); + if (!ok) return GEMM_I8_ERR_ARG; + + if (Order == CblasRowMajor) rm = 1; + else if (Order == CblasColMajor) rm = 0; + else return GEMM_I8_ERR_ARG; + + if (!i8_valid(M, N, K4, ta, tb, rm, A, lda, B, ldb, C, ldc)) + return GEMM_I8_ERR_ARG; + + /* Quick return before the data pointers are required: no element is + * referenced when either extent is zero, as in BLAS. */ + if (M == 0 || N == 0) return GEMM_I8_OK; + if (C == NULL) return GEMM_I8_ERR_ARG; + if (K4 > 0 && (A == NULL || B == NULL)) return GEMM_I8_ERR_ARG; + + /* Empty product: C = 0, the only value a k = 0 product can have. */ + if (K4 == 0) { + for (j = 0; j < N; ++j) + for (i = 0; i < M; ++i) C[i8_cidx(i, j, ldc, rm)] = 0; + return GEMM_I8_OK; + } + + return i8_gemm_do(M, N, K4, ta, tb, rm, A, lda, B, ldb, C, ldc); +} + +int cblas_gemm_i8_scale(enum CBLAS_ORDER Order, + enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANSPOSE TransB, + blasint M, blasint N, blasint K4, + OPENBLAS_CONST int *A, blasint lda, + OPENBLAS_CONST int *B, blasint ldb, + float *C, blasint ldc, + OPENBLAS_CONST float *sa, OPENBLAS_CONST float *sb, + int zp_b) { + int ta, tb, ok, rm; + blasint i, j; + + ta = i8_trans(TransA, &ok); + if (!ok) return GEMM_I8_ERR_ARG; + tb = i8_trans(TransB, &ok); + if (!ok) return GEMM_I8_ERR_ARG; + + if (Order == CblasRowMajor) rm = 1; + else if (Order == CblasColMajor) rm = 0; + else return GEMM_I8_ERR_ARG; + + if (!i8_valid(M, N, K4, ta, tb, rm, A, lda, B, ldb, C, ldc)) + return GEMM_I8_ERR_ARG; + + if (M == 0 || N == 0) return GEMM_I8_OK; + if (C == NULL) return GEMM_I8_ERR_ARG; + if (K4 > 0 && (A == NULL || B == NULL)) return GEMM_I8_ERR_ARG; + + if (K4 == 0) { + for (j = 0; j < N; ++j) + for (i = 0; i < M; ++i) C[i8_cidx(i, j, ldc, rm)] = 0.0f; + return GEMM_I8_OK; + } + + return i8_gemm_scale_do(M, N, K4, ta, tb, rm, A, lda, B, ldb, C, ldc, + sa, sb, zp_b); +} diff --git a/interface/syr2k.c b/interface/syr2k.c index 4b005d0880..20018bb041 100644 --- a/interface/syr2k.c +++ b/interface/syr2k.c @@ -181,6 +181,23 @@ void NAME(char *UPLO, char *TRANS, return; } +#if defined(OPENBLAS_USE_HIP) && !defined(COMPLEX) && !defined(DOUBLE) && !defined(XDOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) + { + /* Symmetric rank-2k update on the GPU; only the uplo triangle of C is read + * or written. Returns 0 when the CPU path should run instead. */ + extern int rdna2_oblas_ssyr2k_cm(BLASLONG, BLASLONG, double, const float *, + BLASLONG, const float *, BLASLONG, double, + float *, BLASLONG, int, int); + + if (rdna2_oblas_ssyr2k_cm(args.n, args.k, (double)*(const FLOAT *)args.alpha, + (const float *)args.a, args.lda, + (const float *)args.b, args.ldb, + (double)*(const FLOAT *)args.beta, + (float *)args.c, args.ldc, uplo, trans)) + return; + } +#endif + #else void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE Trans, @@ -349,6 +366,21 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE Tr if (args.n == 0) return; +#if defined(OPENBLAS_USE_HIP) && !defined(COMPLEX) && !defined(DOUBLE) && !defined(XDOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) + { + /* Symmetric rank-2k update on the GPU; only the uplo triangle of C is read + * or written. Returns 0 when the CPU path should run instead. */ + extern int rdna2_oblas_ssyr2k_cm(BLASLONG, BLASLONG, double, const float *, + BLASLONG, const float *, BLASLONG, double, + float *, BLASLONG, int, int); + + if (rdna2_oblas_ssyr2k_cm(args.n, args.k, (double)alpha, (const float *)args.a, + args.lda, (const float *)args.b, args.ldb, + (double)beta, (float *)args.c, args.ldc, uplo, trans)) + return; + } +#endif + #ifdef DYNAMIC_ARCH extern char* gotoblas_corename(void); #endif diff --git a/interface/syrk.c b/interface/syrk.c index bfce536a06..adfbb52177 100644 --- a/interface/syrk.c +++ b/interface/syrk.c @@ -192,6 +192,21 @@ void NAME(char *UPLO, char *TRANS, return; } +#if defined(OPENBLAS_USE_HIP) && !defined(COMPLEX) && !defined(DOUBLE) && !defined(XDOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) + { + /* n x n symmetric rank-k update on the GPU; only the uplo triangle of C is + * read or written. Returns 0 when the CPU path should run instead. */ + extern int rdna2_oblas_ssyrk_cm(BLASLONG, BLASLONG, double, const float *, + BLASLONG, double, float *, BLASLONG, int, int); + + if (rdna2_oblas_ssyrk_cm(args.n, args.k, (double)*(const FLOAT *)args.alpha, + (const float *)args.a, args.lda, + (double)*(const FLOAT *)args.beta, + (float *)args.c, args.ldc, uplo, trans)) + return; + } +#endif + #else void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE Trans, @@ -343,6 +358,20 @@ double NNK; if (args.n == 0) return; +#if defined(OPENBLAS_USE_HIP) && !defined(COMPLEX) && !defined(DOUBLE) && !defined(XDOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) + { + /* n x n symmetric rank-k update on the GPU; only the uplo triangle of C is + * read or written. Returns 0 when the CPU path should run instead. */ + extern int rdna2_oblas_ssyrk_cm(BLASLONG, BLASLONG, double, const float *, + BLASLONG, double, float *, BLASLONG, int, int); + + if (rdna2_oblas_ssyrk_cm(args.n, args.k, (double)alpha, (const float *)args.a, + args.lda, (double)beta, (float *)args.c, args.ldc, + uplo, trans)) + return; + } +#endif + #ifdef DYNAMIC_ARCH extern char* gotoblas_corename(void); #endif diff --git a/kernel/Makefile b/kernel/Makefile index b1aa985da1..fad70b4b9d 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -118,6 +118,56 @@ endif -include $(KERNELDIR)/KERNEL.$(TARGET_CORE) +# RDNA2/RDNA4 HIP kernels. Built whenever USE_HIP is set, whatever the CPU core +# is: the CPU side keeps that core's own kernels and these add the GPU side, so +# an ordinary build (TARGET=zen, TARGET=haswell, ...) gets both. A GPU-core +# build (TARGET=GFX1201) is just the special case where there is no separate CPU +# core. Defined before Makefile.L3 so SKERNELOBJS picks the objects up (SBLASOBJS +# is expanded from it at parse time). +ifdef USE_HIP +USE_RDNA2 = 1 +endif + +ifdef USE_RDNA2 +HIPCC ?= hipcc +# $(SUFFIX) keeps these out of the profiled object list, which would otherwise +# compile the same file a second time with -pg. +RDNA2_OBJS = rdna2/rdna2_kernels.$(SUFFIX) rdna2/rdna2_oblas.$(SUFFIX) +# GPU_ARCH selects the device architecture(s) independently of the CPU core; +# a GPU-core target supplies it implicitly. One --offload-arch per entry, so +# GPU_ARCH="gfx1031 gfx1201" produces a fat object that runs on both families. +ifneq ($(strip $(GPU_ARCH)),) +RDNA2_GPU_ARCH = $(GPU_ARCH) +else +RDNA2_GPU_ARCH = $(if $(filter GFX%,$(TARGET_CORE)),gfx$(patsubst GFX%,%,$(TARGET_CORE)),gfx1201) +endif +RDNA2_OFFLOAD = $(addprefix --offload-arch=,$(RDNA2_GPU_ARCH)) +# The kernels have a single source of truth, gpu/rdna2/, shared with the +# standalone test and benchmark harnesses. +RDNA2_HIP_SRC = $(TOPDIR)/gpu/rdna2/rdna2_kernels.hip +# The architecture list is baked into the device object but make cannot see +# flags, so record it in a stamp the object depends on: changing GPU_ARCH then +# relinks the library instead of silently keeping the previous cards' code. +RDNA2_ARCH_STAMP = rdna2/rdna2_gpu_arch.stamp +$(RDNA2_ARCH_STAMP): FORCE + @printf '%s\n' '$(RDNA2_GPU_ARCH)' | cmp -s - $@ || printf '%s\n' '$(RDNA2_GPU_ARCH)' > $@ +rdna2/rdna2_kernels.$(SUFFIX): $(RDNA2_HIP_SRC) $(RDNA2_ARCH_STAMP) + $(HIPCC) $(HIPFLAGS) $(RDNA2_OFFLOAD) -x hip -c $< -o $@ +rdna2/rdna2_oblas.$(SUFFIX): rdna2/rdna2_oblas.c + $(CC) $(CFLAGS) -c $< -o $@ +SKERNELOBJS += $(RDNA2_OBJS) +endif + +# The device objects carry an -offload-arch list inside them, so they must not +# outlive a clean: obj/o are spelled out because $(SUFFIX) is empty here. +clean :: + @rm -f rdna2/rdna2_gpu_arch.stamp \ + rdna2/rdna2_kernels.obj rdna2/rdna2_kernels.o \ + rdna2/rdna2_oblas.obj rdna2/rdna2_oblas.o + +.PHONY: FORCE +FORCE: + include $(KERNELDIR)/KERNEL include Makefile.L1 @@ -229,35 +279,3 @@ clean :: #endif include $(TOPDIR)/Makefile.tail - -# RDNA2/RDNA4 HIP kernels -ifdef USE_HIP -ifeq ($(TARGET_CORE), RDNA2) -USE_RDNA2 = 1 -endif -ifeq ($(TARGET_CORE), GFX1030) -USE_RDNA2 = 1 -endif -ifeq ($(TARGET_CORE), GFX1031) -USE_RDNA2 = 1 -endif -ifeq ($(TARGET_CORE), GFX1200) -USE_RDNA2 = 1 -endif -ifeq ($(TARGET_CORE), GFX1201) -USE_RDNA2 = 1 -endif -endif - -ifdef USE_RDNA2 -HIPCC ?= hipcc -RDNA2_OBJS = rdna2/rdna2_kernels.o rdna2/rdna2_oblas.o -rdna2/rdna2_kernels.o: rdna2/rdna2_kernels.hip - $(HIPCC) $(HIPFLAGS) --offload-arch=$(TARGET_CORE) -x hip -c $< -o $@ -rdna2/rdna2_oblas.o: rdna2/rdna2_oblas.c - $(CC) $(CFLAGS) -c $< -o $@ -endif - -ifdef USE_RDNA2 -SKERNELOBJS += $(RDNA2_OBJS) -endif diff --git a/kernel/rdna2/rdna2_oblas.c b/kernel/rdna2/rdna2_oblas.c index 38b56205a6..efa200d1fb 100644 --- a/kernel/rdna2/rdna2_oblas.c +++ b/kernel/rdna2/rdna2_oblas.c @@ -36,8 +36,8 @@ int rdna2_sgemm_f32_host_tuned(float *C, const float *A, const float *Bt, * kernel's N x K left operand; a column-major A (m x k) is contiguous in m and * is transposed once into a temporary. * - * All four bridges funnel into rdna2_gemm_cm_core(), so every level-3 routine - * runs the same GEMM path that the standalone harness verifies against the CPU. + * Every bridge funnels into rdna2_gemm_cm_core(), so every level-3 routine runs + * the same GEMM path that the standalone harness verifies against the CPU. */ /* Below this the CPU path wins, because the host entry point allocates, copies @@ -400,6 +400,212 @@ int rdna2_oblas_strmm_cm(BLASLONG m, BLASLONG n, double alpha, return ok; } +/* C := alpha*T + beta*C, only on the stored triangle of the symmetric result. + * C is n x n column-major with stride ldc, T is the same shape with stride n. + * uplo: 0 = upper (i <= j), 1 = lower (i >= j). The triangle the caller did not + * store is neither read nor written, which is what SYRK/SYR2K promise, so the + * GEMM core may compute both triangles and only half of them is kept. */ +static void rdna2_sym_triangle(double alpha, double beta, float *c, BLASLONG ldc, + const float *t, BLASLONG n, int uplo) { + BLASLONG i, j; + + for (j = 0; j < n; ++j) { + const BLASLONG i0 = (uplo == 0) ? 0 : j; + const BLASLONG i1 = (uplo == 0) ? j + 1 : n; + + for (i = i0; i < i1; ++i) { + const float d = t[(size_t)i + (size_t)j * (size_t)n]; + float *cp = c + (size_t)i + (size_t)j * (size_t)ldc; + + if (alpha == 1.0 && beta == 0.0) + *cp = d; + else if (beta == 0.0) + *cp = (float)(alpha * (double)d); + else + *cp = (float)(alpha * (double)d + beta * (double)*cp); + } + } +} + +/* Materialises X = op(A) as an n x k array in the two layouts the GEMM core + * needs: *rm is X stored row-major with row stride k, and *cm is X stored + * column-major with stride n when op(A) is not already inside the caller's + * array (*cm is NULL then, since A itself is X). + * + * tr == 0: op(A) = A, an n x k column-major array with stride lda. + * tr != 0: op(A) = A^T, an n x k result of a k x n array with stride lda. + * + * The row-major form is what the core's second operand wants: its row j is + * op(B)(j,:), so passing the row-major form of op(A) as that operand makes the + * core compute X*X^T (SYRK) or X*Y^T (SYR2K). Returns 0 on allocation + * failure, in which case the caller falls back to the CPU path. + */ +static int rdna2_sym_pack(const float *a, BLASLONG lda, BLASLONG n, BLASLONG k, + int tr, float **cm_out, float **rm_out) { + BLASLONG i, p; + float *rm = (float *)malloc((size_t)n * (size_t)k * sizeof(float)); + + *cm_out = NULL; + *rm_out = NULL; + if (rm == NULL) return 0; + + if (!tr) { + /* rm[i*k + p] = A(i,p) = a[i + p*lda] */ + rdna2_cm_to_rm(a, lda, rm, k, n, k); + } else { + float *cm = (float *)malloc((size_t)n * (size_t)k * sizeof(float)); + + if (cm == NULL) { + free(rm); + return 0; + } + + /* op(A)(i,p) = A(p,i) = a[p + i*lda] */ + for (i = 0; i < n; ++i) { + for (p = 0; p < k; ++p) { + const float v = a[(size_t)p + (size_t)i * (size_t)lda]; + + cm[(size_t)i + (size_t)p * (size_t)n] = v; + rm[(size_t)i * (size_t)k + (size_t)p] = v; + } + } + *cm_out = cm; + } + + *rm_out = rm; + return 1; +} + +/* SSYRK: C := alpha*A*A^T + beta*C (trans=0) or alpha*A^T*A + beta*C (trans=1), + * with C n x n symmetric and only the uplo triangle stored. + * + * A is n x k for trans=0 and k x n for trans=1, column-major with stride lda, + * matching the interface's already-normalised nrowa rule. The product is an + * ordinary GEMM over the full square (the kernel has no triangle mode); only + * the stored triangle of the result is applied to C, so the other triangle is + * left exactly as the caller left it. Half of that arithmetic is therefore + * discarded, so the size gate below still measures the GEMM that is actually + * run, not SYRK's n*n*k/2 flop count. + * + * uplo: 0 = upper, 1 = lower. Returns 1 when C was produced on the GPU. + */ +int rdna2_oblas_ssyrk_cm(BLASLONG n, BLASLONG k, double alpha, + const float *a, BLASLONG lda, + double beta, float *c, BLASLONG ldc, + int uplo, int trans) { + const int tr = trans & 1; + float *cm = NULL, *rm = NULL, *tmp = NULL; + const float *x; + BLASLONG ldxa; + int ok; + clock_t t0; + + if (!rdna2_gpu_enabled()) return 0; + if (n <= 0 || k <= 0) return 0; + if (n > RDNA2_MAX_DIM || k > RDNA2_MAX_DIM) return 0; + if ((double)n * (double)n * (double)k < rdna2_min_mnk()) return 0; + if (ldc < n) return 0; + if (lda < (tr ? k : n)) return 0; + + if (!rdna2_sym_pack(a, lda, n, k, tr, &cm, &rm)) return 0; + + tmp = (float *)malloc((size_t)n * (size_t)n * sizeof(float)); + if (tmp == NULL) { + free(cm); + free(rm); + return 0; + } + + x = tr ? cm : a; + ldxa = tr ? n : lda; + + t0 = clock(); + ok = rdna2_gemm_cm_core(n, n, k, 1.0, x, ldxa, rm, k, 0.0, tmp, n); + + if (ok) + rdna2_sym_triangle(alpha, beta, c, ldc, tmp, n, uplo); + + if (rdna2_debug()) + fprintf(stderr, "[rdna2] ssyrk %ldx%ld trans=%d uplo=%d handled=%d %ldms\n", + (long)n, (long)k, tr, uplo, ok, rdna2_ms(t0)); + + free(tmp); + free(cm); + free(rm); + return ok; +} + +/* SSYR2K: C := alpha*A*B^T + alpha*B*A^T + beta*C (trans=0) or + * alpha*A^T*B + alpha*B^T*A + beta*C (trans=1), with C n x n symmetric and only + * the uplo triangle stored. A and B are n x k for trans=0 and k x n for + * trans=1, column-major with strides lda and ldb. + * + * The rank-2k update is a rank-2k GEMM: one core call for A*B^T and a second + * that accumulates B*A^T into the same buffer, after which the stored triangle + * is merged into C. uplo: 0 = upper, 1 = lower. Returns 1 when C was produced + * on the GPU. + */ +int rdna2_oblas_ssyr2k_cm(BLASLONG n, BLASLONG k, double alpha, + const float *a, BLASLONG lda, + const float *b, BLASLONG ldb, + double beta, float *c, BLASLONG ldc, + int uplo, int trans) { + const int tr = trans & 1; + const BLASLONG nrowa = tr ? k : n; + float *cma = NULL, *rma = NULL, *cmb = NULL, *rmb = NULL, *tmp = NULL; + const float *xa, *xb; + BLASLONG ldxa, ldxb; + int ok; + clock_t t0; + + if (!rdna2_gpu_enabled()) return 0; + if (n <= 0 || k <= 0) return 0; + if (n > RDNA2_MAX_DIM || k > RDNA2_MAX_DIM) return 0; + if ((double)n * (double)n * (double)k < rdna2_min_mnk()) return 0; + if (ldc < n) return 0; + if (lda < nrowa || ldb < nrowa) return 0; + + if (!rdna2_sym_pack(a, lda, n, k, tr, &cma, &rma)) return 0; + if (!rdna2_sym_pack(b, ldb, n, k, tr, &cmb, &rmb)) { + free(cma); + free(rma); + return 0; + } + + tmp = (float *)malloc((size_t)n * (size_t)n * sizeof(float)); + if (tmp == NULL) { + free(cmb); + free(rmb); + free(cma); + free(rma); + return 0; + } + + xa = tr ? cma : a; + xb = tr ? cmb : b; + ldxa = tr ? n : lda; + ldxb = tr ? n : ldb; + + t0 = clock(); + ok = rdna2_gemm_cm_core(n, n, k, 1.0, xa, ldxa, rmb, k, 0.0, tmp, n); + if (ok) + ok = rdna2_gemm_cm_core(n, n, k, 1.0, xb, ldxb, rma, k, 1.0, tmp, n); + + if (ok) + rdna2_sym_triangle(alpha, beta, c, ldc, tmp, n, uplo); + + if (rdna2_debug()) + fprintf(stderr, "[rdna2] ssyr2k %ldx%ld trans=%d uplo=%d handled=%d %ldms\n", + (long)n, (long)k, tr, uplo, ok, rdna2_ms(t0)); + + free(tmp); + free(cmb); + free(rmb); + free(cma); + free(rma); + return ok; +} + /* Solve T * X = Y in place: T is the dense k x k triangular factor with stride * ldt, X is k x n column-major with stride ldx, and Y is already in X. * uplo: 0 = upper, 1 = lower. Returns 1 on success. diff --git a/kernel/x86_64/KERNEL.RDNA2_SQUARE b/kernel/x86_64/KERNEL.RDNA2_SQUARE new file mode 100644 index 0000000000..a8bcdd91fe --- /dev/null +++ b/kernel/x86_64/KERNEL.RDNA2_SQUARE @@ -0,0 +1,30 @@ +# Rules for the in-direction copies of square-tiled real/complex kernels. +# +# kernel/Makefile.L3 only defines $(KDIR)gemm_incopy/itcopy when the +# UNROLL_M and UNROLL_N of that type differ, on the assumption that a square +# tile can reuse the on/ot copies. The symbols themselves are still called +# directly by the level 3 code (syr2k, her2k, symm, trmm), so the objects have to +# exist. For a square tile the in-copy is the same operation as the on-copy, so +# these compile the same sources under the in-copy names. +# +# Included by KERNEL.GFX1030/GFX1031/GFX1200/GFX1201, whose tiles are square. + +$(KDIR)sgemm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMONCOPY) + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ +$(KDIR)sgemm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMOTCOPY) + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ + +$(KDIR)dgemm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMONCOPY) + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ +$(KDIR)dgemm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMOTCOPY) + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ + +$(KDIR)cgemm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMONCOPY) + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX $< -o $@ +$(KDIR)cgemm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMOTCOPY) + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX $< -o $@ + +$(KDIR)zgemm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMONCOPY) + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX $< -o $@ +$(KDIR)zgemm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMOTCOPY) + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX $< -o $@ diff --git a/param.h b/param.h index 42512e07b6..5aaaa747ac 100644 --- a/param.h +++ b/param.h @@ -4518,5 +4518,57 @@ Until then, just keep it different than DGEMM_DEFAULT_UNROLL_N to keep copy rout #define SHUFPS_39 shufps $0x39, #endif +/* RDNA2 / RDNA4 GPU targets: the host is x86_64 but the core is not one of the + * x86 cores above, so the block chain never reaches a set of GEMM defaults. + * These are the values getarch's FORCE_GFX blocks publish and they match the + * generic C kernels KERNEL.GFX* selects (4x4 f32/f64, 2x2 complex). They only + * describe host-side packing; the GEMM math itself runs on the GPU. */ +#if defined(RDNA2) +#ifndef SGEMM_DEFAULT_UNROLL_M +#define SGEMM_DEFAULT_UNROLL_M 4 +#define SGEMM_DEFAULT_UNROLL_N 4 +#define DGEMM_DEFAULT_UNROLL_M 4 +#define DGEMM_DEFAULT_UNROLL_N 4 +#define QGEMM_DEFAULT_UNROLL_M 2 +#define QGEMM_DEFAULT_UNROLL_N 2 +#define CGEMM_DEFAULT_UNROLL_M 2 +#define CGEMM_DEFAULT_UNROLL_N 2 +#define ZGEMM_DEFAULT_UNROLL_M 2 +#define ZGEMM_DEFAULT_UNROLL_N 2 +#define XGEMM_DEFAULT_UNROLL_M 1 +#define XGEMM_DEFAULT_UNROLL_N 1 +#define SGEMM_DEFAULT_P sgemm_p +#define DGEMM_DEFAULT_P dgemm_p +#define QGEMM_DEFAULT_P qgemm_p +#define CGEMM_DEFAULT_P cgemm_p +#define ZGEMM_DEFAULT_P zgemm_p +#define XGEMM_DEFAULT_P xgemm_p +#define SGEMM_DEFAULT_R sgemm_r +#define DGEMM_DEFAULT_R dgemm_r +#define QGEMM_DEFAULT_R qgemm_r +#define CGEMM_DEFAULT_R cgemm_r +#define ZGEMM_DEFAULT_R zgemm_r +#define XGEMM_DEFAULT_R xgemm_r +#define SGEMM_DEFAULT_Q 128 +#define DGEMM_DEFAULT_Q 64 +#define QGEMM_DEFAULT_Q 64 +#define CGEMM_DEFAULT_Q 64 +#define ZGEMM_DEFAULT_Q 32 +#define XGEMM_DEFAULT_Q 32 +#define GEMM_DEFAULT_OFFSET_A 64 +#define GEMM_DEFAULT_OFFSET_B 256 +#define GEMM_DEFAULT_ALIGN (BLASLONG)0xffffUL +#define SNUMOPT 4 +#define DNUMOPT 2 +#define SYMV_P 16 +#ifndef DTB_DEFAULT_ENTRIES +#define DTB_DEFAULT_ENTRIES 64 +#endif +#ifndef ITB_DEFAULT_ENTRIES +#define ITB_DEFAULT_ENTRIES 64 +#endif +#endif +#endif + #endif \ No newline at end of file diff --git a/test/Makefile b/test/Makefile index 949eb2faec..dd128de618 100644 --- a/test/Makefile +++ b/test/Makefile @@ -247,6 +247,11 @@ endif ifeq ($(BUILD_HFLOAT16),1) H3 = test_shgemm$(EXE) endif +# The int8 extension entry points are part of the C interface and are built for +# every precision, so their test depends only on CBLAS being enabled. +ifneq ($(NO_CBLAS), 1) +I83 = test_gemm_i8$(EXE) +endif ifeq ($(BUILD_SINGLE),1) S3=sblat3$(EXE) endif @@ -274,9 +279,9 @@ endif ifeq ($(SUPPORT_GEMM3M),1) -level3: $(BF3) $(B3) $(H3) $(S3) $(D3) $(C3) $(Z3) level3_3m +level3: $(BF3) $(B3) $(H3) $(I83) $(S3) $(D3) $(C3) $(Z3) level3_3m else -level3: $(BF3) $(B3) $(H3) $(S3) $(D3) $(C3) $(Z3) +level3: $(BF3) $(B3) $(H3) $(I83) $(S3) $(D3) $(C3) $(Z3) endif ifneq ($(CROSS), 1) @@ -291,6 +296,10 @@ ifeq ($(BUILD_HFLOAT16),1) OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_shgemm > SHBLAT3.SUMM $(call CHECK_TEST_SUMMARY,SHBLAT3.SUMM) endif +ifneq ($(NO_CBLAS), 1) + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_gemm_i8$(EXE) > I8GEMM.SUMM + $(call CHECK_TEST_SUMMARY,I8GEMM.SUMM) +endif ifeq ($(BUILD_SINGLE),1) OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./sblat3$(EXE) < ./sblat3.dat $(call CHECK_TEST_SUMMARY,SBLAT3.SUMM) @@ -506,6 +515,10 @@ test_shgemv$(EXE) : compare_sgemv_shgemv.c ../$(LIBNAME) $(CC) $(CLDFLAGS) -o test_shgemv compare_sgemv_shgemv.c ../$(LIBNAME) $(EXTRALIB) $(FEXTRALIB) $(CEXTRALIB) endif +ifneq ($(NO_CBLAS), 1) +test_gemm_i8$(EXE) : test_gemm_i8.c ../$(LIBNAME) + $(CC) $(CLDFLAGS) -I$(TOPDIR) -o test_gemm_i8 test_gemm_i8.c ../$(LIBNAME) $(EXTRALIB) $(FEXTRALIB) $(CEXTRALIB) +endif ifeq ($(BUILD_COMPLEX),1) cblat3_3m$(EXE) : cblat3_3m.$(SUFFIX) $(XERBLA_TEST) ../$(LIBNAME) @@ -523,7 +536,7 @@ clean: @rm -f *.$(SUFFIX) *.$(PSUFFIX) gmon.$(SUFFIX)ut *.SUMM *.cxml *.exe *.pdb *.dwf \ sblat1$(EXE) dblat1$(EXE) cblat1$(EXE) zblat1$(EXE) \ sblat2$(EXE) dblat2$(EXE) cblat2$(EXE) zblat2$(EXE) \ - test_bgemm$(EXE) test_bgemv$(EXE) test_sbgemm$(EXE) test_sbgemv$(EXE) test_shgemm$(EXE) test_shgemv$(EXE) sblat3$(EXE) dblat3$(EXE) cblat3$(EXE) zblat3$(EXE) \ + test_bgemm$(EXE) test_bgemv$(EXE) test_sbgemm$(EXE) test_sbgemv$(EXE) test_shgemm$(EXE) test_shgemv$(EXE) test_gemm_i8$(EXE) sblat3$(EXE) dblat3$(EXE) cblat3$(EXE) zblat3$(EXE) \ sblat1p$(EXE) dblat1p$(EXE) cblat1p$(EXE) zblat1p$(EXE) \ sblat2p$(EXE) dblat2p$(EXE) cblat2p$(EXE) zblat2p$(EXE) \ sblat3p$(EXE) dblat3p$(EXE) cblat3p$(EXE) zblat3p$(EXE) \ From 309ae7aeacc4b291db892a18e9c3f260df2192e5 Mon Sep 17 00:00:00 2001 From: rr Date: Fri, 6 Nov 2026 12:12:25 +0530 Subject: [PATCH 07/13] Use actions/checkout v5 in the level-3 workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first run flagged that v4 still targets Node.js 20, which the runners now force onto Node.js 24. Generated with Codebuff 🤖 Co-Authored-By: Codebuff --- .github/workflows/rdna2_level3.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rdna2_level3.yml b/.github/workflows/rdna2_level3.yml index 2d2d8f514e..7e0664a2d3 100644 --- a/.github/workflows/rdna2_level3.yml +++ b/.github/workflows/rdna2_level3.yml @@ -47,7 +47,7 @@ jobs: name: bridge test (linux, CPU build) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Install toolchain run: | @@ -69,7 +69,7 @@ jobs: run: shell: msys2 {0} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: msys2/setup-msys2@v2 with: @@ -94,7 +94,7 @@ jobs: # name: bridge test (self-hosted, USE_HIP) # runs-on: [self-hosted, gpu, rocm] # steps: - # - uses: actions/checkout@v4 + # - uses: actions/checkout@v5 # - name: Build library with the device kernels # run: | # make clean From 4bd1d69296bdc8ab386acedef0281ce8ac6aaf9d Mon Sep 17 00:00:00 2001 From: rr Date: Fri, 6 Nov 2026 12:45:28 +0530 Subject: [PATCH 08/13] Cover SYMM, TRMM and TRSM in the level-3 bridge test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test only reached SGEMM and SYRK/SYR2K, so three of the six bridged routines could break without anything noticing. Each new op runs over its full layout range and is compared against a double-precision reference that shares none of the bridge's triangle logic (TRSM's reference solves a dense copy with a pivoting LU). The checks also cover what the bridges must not do: the unstored triangle of a symmetric or triangular factor and the stored diagonal of a unit-diagonal one hold a value no correct result can produce, so reading them fails loudly rather than silently, and the ldb/ldc padding must be untouched. Breaking the ssymm triangle read, the unit-diagonal handling and the strsm triangle each made the test fail with only the affected op failing. Generated with Codebuff 🤖 Co-Authored-By: Codebuff --- gpu/rdna2/README.md | 15 +- gpu/rdna2/test_oblas_gpu.c | 495 ++++++++++++++++++++++++++++++++++++- 2 files changed, 499 insertions(+), 11 deletions(-) diff --git a/gpu/rdna2/README.md b/gpu/rdna2/README.md index 388fd627dd..efcc7a3273 100644 --- a/gpu/rdna2/README.md +++ b/gpu/rdna2/README.md @@ -315,9 +315,18 @@ the static archive must add the same. make level3-test # build the library, then run the test make -C gpu/rdna2 oblas-test # run it against an existing library -The test checks the CBLAS SGEMM/SYRK/SYR2K entry points against its own -double-precision reference, including the contract that the unstored triangle -and the `ldc` padding are left untouched. It builds and passes on a CPU-only +The test checks every dispatched level-3 entry point -- SGEMM, SYMM, SYRK, +SYR2K, TRMM and TRSM -- against its own double-precision reference (an LU solve +for TRSM, so the reference does not share the bridge's triangle logic). Each op +runs over its full layout range: both store orders, side, uplo, trans and diag. +The contracts the bridges must keep are checked too: the unstored triangle and +the `ldb`/`ldc` padding are left untouched, and the unused triangle of a +symmetric or triangular factor plus the stored diagonal of a unit-diagonal one +are never read (they hold a value that no correct result can produce), which is +what fails loudly if a bridge reads the wrong half. + +All six ops are seen at the default size gate as well, so the real dispatch +decision is covered end to end, not only the forced-open small shapes. It builds and passes on a CPU-only library, so it runs in ordinary CI (see `.github/workflows/rdna2_level3.yml`); against a `USE_HIP` library it exercises the real dispatch, which `OPENBLAS_RDNA2_GPU_DEBUG=1` makes visible as one `handled=1` line per call. diff --git a/gpu/rdna2/test_oblas_gpu.c b/gpu/rdna2/test_oblas_gpu.c index ae389e788c..53eda9dab3 100644 --- a/gpu/rdna2/test_oblas_gpu.c +++ b/gpu/rdna2/test_oblas_gpu.c @@ -17,6 +17,13 @@ * same cases still run, on the CPU kernels, and still check the * triangle contract. * + * SYMM cover every order/side/uplo/trans/diag combination against a double + * TRMM precision reference (an LU solve for TRSM, so the reference shares + * TRSM none of the triangle logic under test). The unstored triangle of the + * factor, and the stored diagonal of a unit-diagonal one, are filled + * with a value that no correct result can produce, so a bridge that + * reads them fails loudly; the ldb/ldc padding must be untouched. + * * This build may use NUM_THREADS=1, so the CPU timings here are single-threaded * and only meaningful as a sanity check; the CPU-vs-GPU numbers in bench_full.txt * use a 32-thread OpenBLAS. @@ -65,17 +72,55 @@ static double sym_op(const float *a, int ld, int order, int tr, int i, int p) { * buries the log (a 1300x1300 clobber is 845k lines), so print the first few * and count the rest. */ #define SYM_REPORT_MAX 8 -static void sym_violation(long *count, int row, int col, const char *what, - double got, double want, double rel) { +static void violation(long *count, const char *tag, int row, int col, + const char *what, double got, double want, double rel) { if ((*count)++ < SYM_REPORT_MAX) { if (what) - printf("syrk/syr2k: %s element (%d,%d) was written\n", what, row, col); + printf("%s: %s element (%d,%d) was written\n", tag, what, row, col); else - printf("syrk/syr2k: (%d,%d) got %.6f want %.6f (rel %.2e)\n", row, col, - got, want, rel); + printf("%s: (%d,%d) got %.6f want %.6f (rel %.2e)\n", tag, row, col, got, + want, rel); } } +/* Element (i,j) of an m x n matrix with stride ld, in either store order. */ +static double mat_get(const float *x, int ld, int order, int i, int j) { + return (order == CblasColMajor) ? (double)x[(size_t)i + (size_t)j * ld] + : (double)x[(size_t)i * ld + (size_t)j]; +} + +static void mat_put(float *x, int ld, int order, int i, int j, float v) { + if (order == CblasColMajor) + x[(size_t)i + (size_t)j * ld] = v; + else + x[(size_t)i * ld + (size_t)j] = v; +} + +/* Bytes to allocate for an m x n matrix with row or column stride ld. */ +static size_t mat_size(int ld, int order, int m, int n) { + return (order == CblasColMajor) ? (size_t)ld * (size_t)n + : (size_t)ld * (size_t)m; +} + +/* Element (i,p) of a symmetric factor given only its uplo triangle. The mirror + * image is read out of the stored triangle, never from the unused one. */ +static double sym_el(const float *a, int ld, int order, int uplo, int i, int p) { + const int stored = sym_stored(uplo, i, p); + return mat_get(a, ld, order, stored ? i : p, stored ? p : i); +} + +/* Element (i,p) of op(T) for a triangular factor T: op(T) is T (tr == 0) or + * T^T, zero outside the stored triangle, with an implicit unit diagonal when + * unit is set. */ +static double tri_el(const float *a, int ld, int order, int uplo, int tr, + int unit, int i, int p) { + const int si = tr ? p : i; + const int sj = tr ? i : p; + if (!sym_stored(uplo, si, sj)) return 0.0; + if (si == sj && unit) return 1.0; + return mat_get(a, ld, order, si, sj); +} + /* Runs one SYRK (two == 0) or SYR2K (two == 1) case and compares it against a * double precision reference. Returns 0 when the case passed. */ static int sym_case(int two, int order, int uplo, int trans, int n, int k, @@ -135,14 +180,15 @@ static int sym_case(int two, int order, int uplo, int trans, int n, int k, if (row >= n || col >= n) { if (C[idx] != C0[idx]) { - sym_violation(&nviol, row, col, "padding", 0.0, 0.0, 0.0); + violation(&nviol, "syrk/syr2k", row, col, "padding", 0.0, 0.0, 0.0); bad = 1; } continue; } if (!sym_stored(uplo, row, col)) { if (C[idx] != C0[idx]) { - sym_violation(&nviol, row, col, "unstored triangle", 0.0, 0.0, 0.0); + violation(&nviol, "syrk/syr2k", row, col, "unstored triangle", 0.0, 0.0, + 0.0); bad = 1; } continue; @@ -169,7 +215,7 @@ static int sym_case(int two, int order, int uplo, int trans, int n, int k, err = fabs(got - ref); scale = fabs(ref) > 1.0 ? fabs(ref) : 1.0; if (err / scale > 5e-4) { - sym_violation(&nviol, row, col, NULL, got, ref, err / scale); + violation(&nviol, "syrk/syr2k", row, col, NULL, got, ref, err / scale); bad = 1; } } @@ -193,6 +239,359 @@ static int sym_case(int two, int order, int uplo, int trans, int n, int k, return bad; } +/* ------------------------------------------------------------------------- */ +/* SYMM / TRMM / TRSM */ +/* ------------------------------------------------------------------------- */ + +/* Written into every element the operation must not look at (the unstored + * triangle of a symmetric or triangular factor, a unit diagonal). A pipeline + * that reads it turns this into errors of this order, far above the tolerances + * below, without producing the NaNs that would also poison a correct build. */ +#define UNUSED_FILL 1.0e6f + +/* Everything outside the logical m x n block of a matrix with stride ld must + * still hold what it was filled with. */ +static int pad_intact(const char *tag, const float *x, const float *x0, int ld, + int order, int m, int n, long *nviol) { + int i, j, bad = 0; + + if (order == CblasColMajor) { + for (j = 0; j < n; ++j) + for (i = m; i < ld; ++i) + if (x[(size_t)i + (size_t)j * ld] != x0[(size_t)i + (size_t)j * ld]) { + violation(nviol, tag, i, j, "padding", 0.0, 0.0, 0.0); + bad = 1; + } + } else { + for (i = 0; i < m; ++i) + for (j = n; j < ld; ++j) + if (x[(size_t)i * ld + (size_t)j] != x0[(size_t)i * ld + (size_t)j]) { + violation(nviol, tag, i, j, "padding", 0.0, 0.0, 0.0); + bad = 1; + } + } + return bad; +} + +/* Compares one result element against the reference and reports a mismatch. */ +static void check_el(const char *tag, int i, int j, double got, double want, + double tol, long *nviol, int *bad) { + const double err = fabs(got - want); + const double scale = fabs(want) > 1.0 ? fabs(want) : 1.0; + + if (err / scale > tol) { + violation(nviol, tag, i, j, NULL, got, want, err / scale); + *bad = 1; + } +} + +/* C := alpha*A*B + beta*C (side=left) or alpha*B*A + beta*C (side=right), A + * symmetric with only its uplo triangle stored. Checks every element of C, the + * ldc padding, and that A's other triangle is never read. */ +static int symm_case(int order, int side, int uplo, int m, int n, double alpha, + double beta) { + const int kdim = side ? n : m; + const int lda = kdim + 3; + const int ldb = (m > n ? m : n) + 4; + const int ldc = (m > n ? m : n) + 5; + const float sentinel = -12345.5f; + const size_t asz = mat_size(lda, order, kdim, kdim); + const size_t bsz = mat_size(ldb, order, m, n); + const size_t csz = mat_size(ldc, order, m, n); + float *A = (float *)malloc(asz * sizeof(float)); + float *B = (float *)malloc(bsz * sizeof(float)); + float *C = (float *)malloc(csz * sizeof(float)); + float *C0 = (float *)malloc(csz * sizeof(float)); + size_t k; + int i, j, bad = 0; + long nviol = 0; + + if (!A || !B || !C || !C0) { + printf("symm: allocation failed at m=%d n=%d\n", m, n); + free(A); + free(B); + free(C); + free(C0); + return 1; + } + + for (k = 0; k < asz; ++k) A[k] = UNUSED_FILL; + for (k = 0; k < bsz; ++k) B[k] = sentinel; + for (k = 0; k < csz; ++k) C0[k] = C[k] = sentinel; + + for (j = 0; j < kdim; ++j) + for (i = 0; i < kdim; ++i) + if (sym_stored(uplo, i, j)) + mat_put(A, lda, order, i, j, + (float)((double)((i * 5 + j * 3) % 17) - 8.0) / 8.0f); + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) { + const float v = (float)((double)((i * 7 + j * 11) % 23) - 11.0) / 9.0f; + + mat_put(B, ldb, order, i, j, v); + mat_put(C0, ldc, order, i, j, v * 0.5f + 0.25f); + } + memcpy(C, C0, csz * sizeof(float)); + + cblas_ssymm(order, side ? CblasRight : CblasLeft, + uplo ? CblasLower : CblasUpper, m, n, (float)alpha, A, lda, B, ldb, + (float)beta, C, ldc); + + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) { + double ref = 0.0; + int p; + + for (p = 0; p < kdim; ++p) + ref += (side ? mat_get(B, ldb, order, i, p) * + sym_el(A, lda, order, uplo, p, j) + : sym_el(A, lda, order, uplo, i, p) * + mat_get(B, ldb, order, p, j)); + ref = alpha * ref + beta * (double)mat_get(C0, ldc, order, i, j); + check_el("symm", i, j, mat_get(C, ldc, order, i, j), ref, 5e-4, &nviol, + &bad); + } + bad |= pad_intact("symm", C, C0, ldc, order, m, n, &nviol); + + if (nviol > SYM_REPORT_MAX) + printf("symm: ... %ld more violation(s) in this case\n", + nviol - SYM_REPORT_MAX); + printf("%s symm order=%d side=%d uplo=%d m=%d n=%d alpha=%g beta=%g\n", + bad ? "FAIL" : "ok ", order, side, uplo, m, n, alpha, beta); + + free(A); + free(B); + free(C); + free(C0); + return bad; +} + +/* B := alpha*op(A)*B (side=left) or alpha*B*op(A) (side=right), in place, A + * triangular with only its uplo triangle stored. The unused triangle and the + * stored diagonal of a unit-diagonal factor carry UNUSED_FILL, so reading + * either one shows up as a large error. */ +static int trmm_case(int order, int side, int uplo, int trans, int unit, int m, + int n, double alpha) { + const int kdim = side ? n : m; + const int lda = kdim + 3; + const int ldb = (m > n ? m : n) + 4; + const float sentinel = -12345.5f; + const size_t asz = mat_size(lda, order, kdim, kdim); + const size_t bsz = mat_size(ldb, order, m, n); + float *A = (float *)malloc(asz * sizeof(float)); + float *B = (float *)malloc(bsz * sizeof(float)); + float *B0 = (float *)malloc(bsz * sizeof(float)); + size_t k; + int i, j, bad = 0; + long nviol = 0; + + if (!A || !B || !B0) { + printf("trmm: allocation failed at m=%d n=%d\n", m, n); + free(A); + free(B); + free(B0); + return 1; + } + + for (k = 0; k < asz; ++k) A[k] = UNUSED_FILL; + for (j = 0; j < kdim; ++j) + for (i = 0; i < kdim; ++i) + if (sym_stored(uplo, i, j) && !(i == j && unit)) + mat_put(A, lda, order, i, j, + (float)((double)((i * 5 + j * 3) % 17) - 8.0) / 16.0f); + for (k = 0; k < bsz; ++k) B0[k] = B[k] = sentinel; + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) + mat_put(B0, ldb, order, i, j, + (float)((double)((i * 7 + j * 11) % 23) - 11.0) / 11.0f); + memcpy(B, B0, bsz * sizeof(float)); + + cblas_strmm(order, side ? CblasRight : CblasLeft, + uplo ? CblasLower : CblasUpper, + trans ? CblasTrans : CblasNoTrans, + unit ? CblasUnit : CblasNonUnit, m, n, (float)alpha, A, lda, B, + ldb); + + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) { + double ref = 0.0; + int p; + + for (p = 0; p < kdim; ++p) + ref += (side ? tri_el(A, lda, order, uplo, trans, unit, p, j) * + mat_get(B0, ldb, order, i, p) + : tri_el(A, lda, order, uplo, trans, unit, i, p) * + mat_get(B0, ldb, order, p, j)); + check_el("trmm", i, j, mat_get(B, ldb, order, i, j), alpha * ref, 5e-4, + &nviol, &bad); + } + bad |= pad_intact("trmm", B, B0, ldb, order, m, n, &nviol); + + if (nviol > SYM_REPORT_MAX) + printf("trmm: ... %ld more violation(s) in this case\n", + nviol - SYM_REPORT_MAX); + printf("%s trmm order=%d side=%d uplo=%d trans=%d diag=%d m=%d n=%d alpha=%g\n", + bad ? "FAIL" : "ok ", order, side, uplo, trans, unit, m, n, alpha); + + free(A); + free(B); + free(B0); + return bad; +} + +/* Solves M*Y = X in double precision with partial pivoting. M (n x n) is + * destroyed, X is n x nrhs with row stride ldx. Returns 0 if M is singular. */ +static int lu_solve(double *M, int n, double *X, int nrhs, int ldx) { + int k, i, j, r; + + for (k = 0; k < n; ++k) { + int piv = k; + double best = fabs(M[(size_t)k + (size_t)k * n]); + + for (i = k + 1; i < n; ++i) { + const double v = fabs(M[(size_t)i + (size_t)k * n]); + + if (v > best) { + best = v; + piv = i; + } + } + if (best < 1e-9) return 0; + if (piv != k) { + for (j = 0; j < n; ++j) { + const double t = M[(size_t)k + (size_t)j * n]; + + M[(size_t)k + (size_t)j * n] = M[(size_t)piv + (size_t)j * n]; + M[(size_t)piv + (size_t)j * n] = t; + } + for (r = 0; r < nrhs; ++r) { + const double t = X[(size_t)k + (size_t)r * ldx]; + + X[(size_t)k + (size_t)r * ldx] = X[(size_t)piv + (size_t)r * ldx]; + X[(size_t)piv + (size_t)r * ldx] = t; + } + } + for (i = k + 1; i < n; ++i) { + const double f = + M[(size_t)i + (size_t)k * n] / M[(size_t)k + (size_t)k * n]; + + if (f == 0.0) continue; + for (j = k + 1; j < n; ++j) + M[(size_t)i + (size_t)j * n] -= f * M[(size_t)k + (size_t)j * n]; + for (r = 0; r < nrhs; ++r) + X[(size_t)i + (size_t)r * ldx] -= f * X[(size_t)k + (size_t)r * ldx]; + } + } + + for (r = 0; r < nrhs; ++r) + for (i = n - 1; i >= 0; --i) { + double s = X[(size_t)i + (size_t)r * ldx]; + + for (j = i + 1; j < n; ++j) + s -= M[(size_t)i + (size_t)j * n] * X[(size_t)j + (size_t)r * ldx]; + X[(size_t)i + (size_t)r * ldx] = s / M[(size_t)i + (size_t)i * n]; + } + return 1; +} + +/* Solves op(A)*X = alpha*B (side=left) or X*op(A) = alpha*B (side=right), in + * place on B, A triangular. The reference solves a dense copy of op(A) with a + * pivoting LU, so it does not share the triangle logic under test. */ +static int trsm_case(int order, int side, int uplo, int trans, int unit, int m, + int n, double alpha) { + const int kdim = side ? n : m; + const int lda = kdim + 3; + const int ldb = (m > n ? m : n) + 4; + const int nrhs = side ? m : n; + const float sentinel = -12345.5f; + const size_t asz = mat_size(lda, order, kdim, kdim); + const size_t bsz = mat_size(ldb, order, m, n); + float *A = (float *)malloc(asz * sizeof(float)); + float *B = (float *)malloc(bsz * sizeof(float)); + float *B0 = (float *)malloc(bsz * sizeof(float)); + double *M = (double *)malloc((size_t)kdim * (size_t)kdim * sizeof(double)); + double *X = (double *)malloc((size_t)kdim * (size_t)nrhs * sizeof(double)); + size_t k; + int i, j, bad = 0; + long nviol = 0; + + if (!A || !B || !B0 || !M || !X) { + printf("trsm: allocation failed at m=%d n=%d\n", m, n); + free(A); + free(B); + free(B0); + free(M); + free(X); + return 1; + } + + for (k = 0; k < asz; ++k) A[k] = UNUSED_FILL; + for (j = 0; j < kdim; ++j) + for (i = 0; i < kdim; ++i) + if (sym_stored(uplo, i, j) && !(i == j && unit)) + mat_put(A, lda, order, i, j, + (i == j) ? (float)(1.0 + 0.5 * ((j % 3) + 1) / 4.0) + : (float)((double)((i * 5 + j * 3) % 7) - 3.0) / 32.0f); + for (k = 0; k < bsz; ++k) B0[k] = B[k] = sentinel; + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) + mat_put(B0, ldb, order, i, j, + (float)((double)((i * 7 + j * 11) % 23) - 11.0) / 11.0f); + memcpy(B, B0, bsz * sizeof(float)); + + cblas_strsm(order, side ? CblasRight : CblasLeft, + uplo ? CblasLower : CblasUpper, + trans ? CblasTrans : CblasNoTrans, + unit ? CblasUnit : CblasNonUnit, m, n, (float)alpha, A, lda, B, + ldb); + + /* side=right is solved as op(A)^T * Y = alpha*B^T, so the dense copy is + * transposed there; side=left solves against op(A) directly. */ + for (j = 0; j < kdim; ++j) + for (i = 0; i < kdim; ++i) + M[(size_t)i + (size_t)j * kdim] = tri_el( + A, lda, order, uplo, trans, unit, side ? j : i, side ? i : j); + /* X := alpha*B, or alpha*B^T for side=right, so both sides are a plain + * solve against a matrix the code under test built nothing of. */ + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) { + const double v = alpha * mat_get(B0, ldb, order, i, j); + + if (side) + X[(size_t)j + (size_t)i * kdim] = v; + else + X[(size_t)i + (size_t)j * kdim] = v; + } + + if (!lu_solve(M, kdim, X, nrhs, kdim)) { + printf("FAIL trsm: reference matrix is singular (m=%d n=%d)\n", m, n); + bad = 1; + } else { + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) { + const double ref = side ? X[(size_t)j + (size_t)i * kdim] + : X[(size_t)i + (size_t)j * kdim]; + + check_el("trsm", i, j, mat_get(B, ldb, order, i, j), ref, 1e-3, &nviol, + &bad); + } + bad |= pad_intact("trsm", B, B0, ldb, order, m, n, &nviol); + } + + if (nviol > SYM_REPORT_MAX) + printf("trsm: ... %ld more violation(s) in this case\n", + nviol - SYM_REPORT_MAX); + printf("%s trsm order=%d side=%d uplo=%d trans=%d diag=%d m=%d n=%d alpha=%g\n", + bad ? "FAIL" : "ok ", order, side, uplo, trans, unit, m, n, alpha); + + free(A); + free(B); + free(B0); + free(M); + free(X); + return bad; +} + int main(int argc, char **argv) { setvbuf(stdout, NULL, _IONBF, 0); const int n = (argc > 1) ? atoi(argv[1]) : 2048; @@ -295,6 +694,86 @@ int main(int argc, char **argv) { failures += sym_failures; } + /* ----------------------------------------------------------------------- */ + /* SYMM / TRMM / TRSM: one case per op above the default size gate, then */ + /* every layout combination at shapes the gate would otherwise filter out. */ + /* ----------------------------------------------------------------------- */ + { + int order, side, uplo, trans, diag; + int cases, sec_failures; + /* m*n*kdim = 2.2e9, just past the bridge's default 2e9 threshold. */ + const int gate_mn = 1300; + + SETENV("OPENBLAS_RDNA2_GPU", "1"); + SETENV("OPENBLAS_RDNA2_GPU_MIN_MNK", "2000000000"); + printf("\nsymm/trmm/trsm at the default size gate (m=n=%d)\n", gate_mn); + cases = sec_failures = 0; + sec_failures += symm_case(CblasColMajor, 0, 0, gate_mn, gate_mn, 1.0, 0.0); + cases++; + sec_failures += trmm_case(CblasColMajor, 0, 0, 0, 0, gate_mn, gate_mn, 1.0); + cases++; + sec_failures += trsm_case(CblasColMajor, 0, 0, 0, 0, gate_mn, gate_mn, 1.0); + cases++; + printf("gate cases: %d cases, %d failure(s)\n", cases, sec_failures); + failures += sec_failures; + + /* Every remaining case is small on purpose, so it reaches the device, or + * the CPU kernels on a GPU-less machine, at shapes a GEMM crossover would + * never route there. */ + SETENV("OPENBLAS_RDNA2_GPU_MIN_MNK", "0"); + + cases = sec_failures = 0; + printf("symm edge shapes (size gate forced open)\n"); + for (order = 0; order <= 1; ++order) + for (side = 0; side <= 1; ++side) + for (uplo = 0; uplo <= 1; ++uplo) { + const int ord = order ? CblasRowMajor : CblasColMajor; + + sec_failures += symm_case(ord, side, uplo, 97, 61, 1.0, 0.0); + cases++; + sec_failures += symm_case(ord, side, uplo, 53, 130, 0.5, -0.25); + cases++; + } + printf("symm: %d cases, %d failure(s)\n", cases, sec_failures); + failures += sec_failures; + + cases = sec_failures = 0; + printf("trmm edge shapes (size gate forced open)\n"); + for (order = 0; order <= 1; ++order) + for (side = 0; side <= 1; ++side) + for (uplo = 0; uplo <= 1; ++uplo) + for (trans = 0; trans <= 1; ++trans) + for (diag = 0; diag <= 1; ++diag) { + const int ord = order ? CblasRowMajor : CblasColMajor; + + sec_failures += trmm_case(ord, side, uplo, trans, diag, 97, 61, 1.0); + cases++; + sec_failures += + trmm_case(ord, side, uplo, trans, diag, 53, 130, -0.75); + cases++; + } + printf("trmm: %d cases, %d failure(s)\n", cases, sec_failures); + failures += sec_failures; + + cases = sec_failures = 0; + printf("trsm edge shapes (size gate forced open)\n"); + for (order = 0; order <= 1; ++order) + for (side = 0; side <= 1; ++side) + for (uplo = 0; uplo <= 1; ++uplo) + for (trans = 0; trans <= 1; ++trans) + for (diag = 0; diag <= 1; ++diag) { + const int ord = order ? CblasRowMajor : CblasColMajor; + + sec_failures += trsm_case(ord, side, uplo, trans, diag, 97, 61, 1.0); + cases++; + sec_failures += + trsm_case(ord, side, uplo, trans, diag, 53, 130, 0.75); + cases++; + } + printf("trsm: %d cases, %d failure(s)\n", cases, sec_failures); + failures += sec_failures; + } + if (failures == 0) printf("ALL LEVEL-3 BRIDGE CHECKS PASSED\n"); else From 5351ca159ba730744e9b71943d058ca4555b8e6c Mon Sep 17 00:00:00 2001 From: rr Date: Fri, 6 Nov 2026 13:51:20 +0530 Subject: [PATCH 09/13] Sweep zero, negative and large scalars in the level-3 bridge test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every level-3 case used alpha=1 or a small negative value and beta=0 or -0.25, so a bridge that mishandled a scalar edge went unnoticed. Each op now also runs alpha and beta at zero, negative and large values, and the test asserts what a zero means: alpha==0 leaves beta*C and nothing else, beta==0 drops the old C, and both zero means exact zeros. TRMM and TRSM have no beta and sweep alpha. Making the TRSM cases reliable took two changes. Its factor was loosely conditioned: ||A^-1|| reached 2e8 at 1300 rows, so the verdict depended on the order the substitution happened to sum its terms, and a mutant that could not change the alpha=1 result still flipped that case by 0.86 relative. The factor is now row-wise diagonally dominant, and the tolerance follows the row count, because a float solve over 1300 rows is only good to about a percent whatever the build. The gate case says so and is a dispatch check; the layout and scalar cases keep the tight tolerances. Verified against three mutants of the bridges, each failing only where it should: an alpha==0 early-out that forgets beta*C fails 96 cases, dropping alpha from the signed and scaled paths fails 400, and ignoring the trans/side triangle fails 64 of the 160 TRSM cases. Generated with Codebuff 🤖 Co-Authored-By: Codebuff --- gpu/rdna2/Makefile | 7 +- gpu/rdna2/README.md | 22 +++- gpu/rdna2/test_oblas_gpu.c | 220 ++++++++++++++++++++++++++++++++----- 3 files changed, 216 insertions(+), 33 deletions(-) diff --git a/gpu/rdna2/Makefile b/gpu/rdna2/Makefile index 505a8fab74..47839f43c9 100644 --- a/gpu/rdna2/Makefile +++ b/gpu/rdna2/Makefile @@ -57,9 +57,10 @@ bench-run: bench ./bench_rdna2 "$(OB_DLL)" # Level-3 bridge test against the library built in this tree: checks the -# interface-side kernel plumbing (SGEMM, and SYRK/SYR2K where the dispatch is -# compiled in) against an independent double-precision reference, including the -# SYRK contract that the unstored triangle and the ldc padding are untouched. +# interface-side kernel plumbing (SGEMM, SYMM, TRMM, TRSM, SYRK and SYR2K where +# the dispatch is compiled in) against an independent double-precision reference, +# including the contract that the unstored triangle and the ldb/ldc padding are +# untouched, over every layout and over zero, negative and large scalars. # The same binary is useful either way -- the device path is exercised only when # OBLAS_LIB is a USE_HIP build, while the reference checks still run on a # CPU-only build. Point OBLAS_LIB/LDFLAGS at another tree or a ROCm runtime diff --git a/gpu/rdna2/README.md b/gpu/rdna2/README.md index efcc7a3273..f50684152e 100644 --- a/gpu/rdna2/README.md +++ b/gpu/rdna2/README.md @@ -326,8 +326,26 @@ are never read (they hold a value that no correct result can produce), which is what fails loudly if a bridge reads the wrong half. All six ops are seen at the default size gate as well, so the real dispatch -decision is covered end to end, not only the forced-open small shapes. It builds and passes on a CPU-only -library, so it runs in ordinary CI (see `.github/workflows/rdna2_level3.yml`); +decision is covered end to end, not only the forced-open small shapes. + +The scalars get their own pass on top of the layouts: `alpha` and `beta` at +zero, negative and large values (0, -0.5, -2, 1000). A zero scalar has to drop +its term exactly -- `alpha == 0` leaves `beta*C` and nothing else, `beta == 0` +drops the old `C`, both zero means exact zeros -- and the large values keep the +accumulation honest. TRMM and TRSM have no `beta`, so they sweep `alpha` only. + +The TRSM factor is built row-wise diagonally dominant, each stored row summing +to a quarter of its diagonal. A triangular solve over a merely "small" +off-diagonal grows by orders of magnitude across a thousand rows, and the answer +then depends on the order the terms happened to be summed in rather than on the +triangle: the loosely conditioned 1300-row version reached `||A^-1|| ~ 2e8` and +flipped between builds of equivalent code. Dominance keeps the reference usable +at the gate size. The above-gate TRSM case is a dispatch check rather than an +accuracy one: at 1300 rows a float solve on this path is only good to about a +percent however it is compiled, so the layout and scalar cases carry the tight +tolerances. + +It builds and passes on a CPU-only library, so it runs in ordinary CI (see `.github/workflows/rdna2_level3.yml`); against a `USE_HIP` library it exercises the real dispatch, which `OPENBLAS_RDNA2_GPU_DEBUG=1` makes visible as one `handled=1` line per call. The link line adds the HIP runtime by itself when the archive contains the diff --git a/gpu/rdna2/test_oblas_gpu.c b/gpu/rdna2/test_oblas_gpu.c index 53eda9dab3..70b3c4e370 100644 --- a/gpu/rdna2/test_oblas_gpu.c +++ b/gpu/rdna2/test_oblas_gpu.c @@ -19,7 +19,8 @@ * * SYMM cover every order/side/uplo/trans/diag combination against a double * TRMM precision reference (an LU solve for TRSM, so the reference shares - * TRSM none of the triangle logic under test). The unstored triangle of the + * TRSM none of the triangle logic under test), at alpha and beta values + * including zero, negative and large ones. The unstored triangle of the * factor, and the stored diagonal of a unit-diagonal one, are filled * with a value that no correct result can produce, so a bridge that * reads them fails loudly; the ldb/ldc padding must be untouched. @@ -83,6 +84,16 @@ static void violation(long *count, const char *tag, int row, int col, } } +/* Tolerance for one element: the structural tolerance the test has always + * used, plus the rounding the reference itself carries when a scalar edge + * inflates the magnitudes. A result that is small next to the terms that went + * into it (a large beta*C, heavy cancellation) cannot be judged below its own + * representation error, while a wrong triangle or a dropped term is off by + * orders of magnitude more, so the cases keep their power. */ +static double tol_for(double scale, double mag) { + return 5e-4 + 1e-6 * mag / scale; +} + /* Element (i,j) of an m x n matrix with stride ld, in either store order. */ static double mat_get(const float *x, int ld, int order, int i, int j) { return (order == CblasColMajor) ? (double)x[(size_t)i + (size_t)j * ld] @@ -194,7 +205,7 @@ static int sym_case(int two, int order, int uplo, int trans, int n, int k, continue; } { - double ref = 0.0, got, err, scale; + double ref = 0.0, got, err, scale, mag = 0.0, tol; int p; for (p = 0; p < k; ++p) { @@ -206,15 +217,18 @@ static int sym_case(int two, int order, int uplo, int trans, int n, int k, const double b_jp = sym_op(B, ldb, order, trans, col, p); ref += a_ip * b_jp + b_ip * a_jp; + mag += fabs(a_ip * b_jp) + fabs(b_ip * a_jp); } else { ref += a_ip * a_jp; + mag += fabs(a_ip * a_jp); } } ref = alpha * ref + beta * (double)C0[idx]; got = (double)C[idx]; err = fabs(got - ref); scale = fabs(ref) > 1.0 ? fabs(ref) : 1.0; - if (err / scale > 5e-4) { + tol = tol_for(scale, fabs(alpha) * mag + fabs(beta) * fabs((double)C0[idx])); + if (err / scale > tol) { violation(&nviol, "syrk/syr2k", row, col, NULL, got, ref, err / scale); bad = 1; } @@ -339,17 +353,24 @@ static int symm_case(int order, int side, int uplo, int m, int n, double alpha, for (j = 0; j < n; ++j) for (i = 0; i < m; ++i) { - double ref = 0.0; + double ref = 0.0, mag = 0.0, c0; int p; - for (p = 0; p < kdim; ++p) - ref += (side ? mat_get(B, ldb, order, i, p) * - sym_el(A, lda, order, uplo, p, j) - : sym_el(A, lda, order, uplo, i, p) * - mat_get(B, ldb, order, p, j)); - ref = alpha * ref + beta * (double)mat_get(C0, ldc, order, i, j); - check_el("symm", i, j, mat_get(C, ldc, order, i, j), ref, 5e-4, &nviol, - &bad); + for (p = 0; p < kdim; ++p) { + const double t = side ? mat_get(B, ldb, order, i, p) * + sym_el(A, lda, order, uplo, p, j) + : sym_el(A, lda, order, uplo, i, p) * + mat_get(B, ldb, order, p, j); + + ref += t; + mag += fabs(t); + } + c0 = mat_get(C0, ldc, order, i, j); + ref = alpha * ref + beta * c0; + check_el("symm", i, j, mat_get(C, ldc, order, i, j), ref, + tol_for(fabs(ref) > 1.0 ? fabs(ref) : 1.0, + fabs(alpha) * mag + fabs(beta) * fabs(c0)), + &nviol, &bad); } bad |= pad_intact("symm", C, C0, ldc, order, m, n, &nviol); @@ -494,11 +515,20 @@ static int lu_solve(double *M, int n, double *X, int nrhs, int ldx) { return 1; } +/* Deterministic off-diagonal of a triangular factor, in [-3/32, 3/32]. */ +static double tri_raw(int i, int j) { + return (double)((i * 5 + j * 3) % 7 - 3) / 32.0; +} + +/* Stored diagonal of a non-unit factor, deliberately not 1 so a bridge that + * reads it when the caller said unit diagonal is caught. */ +static double tri_diag(int i) { return 1.0 + 0.5 * ((i % 3) + 1) / 4.0; } + /* Solves op(A)*X = alpha*B (side=left) or X*op(A) = alpha*B (side=right), in * place on B, A triangular. The reference solves a dense copy of op(A) with a * pivoting LU, so it does not share the triangle logic under test. */ static int trsm_case(int order, int side, int uplo, int trans, int unit, int m, - int n, double alpha) { + int n, double alpha, double floor_tol) { const int kdim = side ? n : m; const int lda = kdim + 3; const int ldb = (m > n ? m : n) + 4; @@ -511,27 +541,49 @@ static int trsm_case(int order, int side, int uplo, int trans, int unit, int m, float *B0 = (float *)malloc(bsz * sizeof(float)); double *M = (double *)malloc((size_t)kdim * (size_t)kdim * sizeof(double)); double *X = (double *)malloc((size_t)kdim * (size_t)nrhs * sizeof(double)); + double *scale = (double *)malloc((size_t)kdim * sizeof(double)); size_t k; int i, j, bad = 0; long nviol = 0; - if (!A || !B || !B0 || !M || !X) { + if (!A || !B || !B0 || !M || !X || !scale) { printf("trsm: allocation failed at m=%d n=%d\n", m, n); free(A); free(B); free(B0); free(M); free(X); + free(scale); return 1; } + /* The factor is made row-wise diagonally dominant, with each stored row's + * off-diagonals summing to a quarter of its diagonal. A merely small + * off-diagonal still grows enormously over a thousand rows in a triangular + * solve: the 1300-row gate case below reached ||A^-1|| ~ 2e8, which is far + * into the range where a float solve's answer depends on the order the terms + * happen to be summed in. Such a case cannot tell a broken triangle from a + * differently compiled correct one, so it is not worth having. Dominance + * keeps the reference a reliable oracle at every size, while the diagonal + * stays away from 1 so a bridge that ignores the unit-diagonal flag is still + * caught reading the stored diagonal. */ + for (i = 0; i < kdim; ++i) { + double sum = 0.0; + const int j0 = (uplo == 0) ? i : 0; + const int j1 = (uplo == 0) ? kdim - 1 : i; + + for (j = j0; j <= j1; ++j) + if (!(i == j && unit)) sum += fabs(tri_raw(i, j)); + scale[i] = 0.25 * tri_diag(i) / (sum > 1.0 ? sum : 1.0); + } + for (k = 0; k < asz; ++k) A[k] = UNUSED_FILL; for (j = 0; j < kdim; ++j) for (i = 0; i < kdim; ++i) if (sym_stored(uplo, i, j) && !(i == j && unit)) mat_put(A, lda, order, i, j, - (i == j) ? (float)(1.0 + 0.5 * ((j % 3) + 1) / 4.0) - : (float)((double)((i * 5 + j * 3) % 7) - 3.0) / 32.0f); + (i == j) ? (float)tri_diag(i) + : (float)(tri_raw(i, j) * scale[i])); for (k = 0; k < bsz; ++k) B0[k] = B[k] = sentinel; for (j = 0; j < n; ++j) for (i = 0; i < m; ++i) @@ -566,15 +618,23 @@ static int trsm_case(int order, int side, int uplo, int trans, int unit, int m, if (!lu_solve(M, kdim, X, nrhs, kdim)) { printf("FAIL trsm: reference matrix is singular (m=%d n=%d)\n", m, n); bad = 1; - } else { - for (j = 0; j < n; ++j) - for (i = 0; i < m; ++i) { - const double ref = side ? X[(size_t)j + (size_t)i * kdim] - : X[(size_t)i + (size_t)j * kdim]; - - check_el("trsm", i, j, mat_get(B, ldb, order, i, j), ref, 1e-3, &nviol, - &bad); - } + } else { /* A solve accumulates over kdim rows, so its float result drifts with the + * size of the factor, and where in that range it lands depends on the + * order the terms happen to be summed in rather than on the answer being + * right. The tolerance follows the row count on top of the caller's + * floor. A broken triangle or a dropped scalar is off by orders of + * magnitude more than this (the sentinel alone is 1e6), so the layout + * cases give nothing up. */ + const double tol = floor_tol + 1e-5 * (double)kdim; + + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) { + const double ref = side ? X[(size_t)j + (size_t)i * kdim] + : X[(size_t)i + (size_t)j * kdim]; + + check_el("trsm", i, j, mat_get(B, ldb, order, i, j), ref, tol, &nviol, + &bad); + } bad |= pad_intact("trsm", B, B0, ldb, order, m, n, &nviol); } @@ -589,6 +649,7 @@ static int trsm_case(int order, int side, int uplo, int trans, int unit, int m, free(B0); free(M); free(X); + free(scale); return bad; } @@ -690,6 +751,35 @@ int main(int argc, char **argv) { cases++; } + /* Scalar edges. alpha == 0 must drop the product term exactly, leaving + * beta*C and nothing else; beta == 0 must drop the old C, so both zero + * means exact zeros. The negative and large values catch a scalar applied + * to one term only, or a sign lost where the two triangles are merged. */ + printf("syrk/syr2k scalar edges (size gate forced open)\n"); + for (order = 0; order <= 1; ++order) + for (uplo = 0; uplo <= 1; ++uplo) + for (trans = 0; trans <= 1; ++trans) { + const int ord = order ? CblasRowMajor : CblasColMajor; + int two; + + for (two = 0; two <= 1; ++two) { + sym_failures += sym_case(two, ord, uplo, trans, 97, 61, 0.0, 1.0); + cases++; + sym_failures += sym_case(two, ord, uplo, trans, 97, 61, 0.0, -0.5); + cases++; + sym_failures += sym_case(two, ord, uplo, trans, 97, 61, 0.0, 0.0); + cases++; + sym_failures += sym_case(two, ord, uplo, trans, 53, 130, -2.0, 1.0); + cases++; + sym_failures += + sym_case(two, ord, uplo, trans, 53, 130, 1000.0, 0.0); + cases++; + sym_failures += + sym_case(two, ord, uplo, trans, 53, 130, 1.0, 1000.0); + cases++; + } + } + printf("syrk/syr2k: %d cases, %d failure(s)\n", cases, sym_failures); failures += sym_failures; } @@ -712,7 +802,13 @@ int main(int argc, char **argv) { cases++; sec_failures += trmm_case(CblasColMajor, 0, 0, 0, 0, gate_mn, gate_mn, 1.0); cases++; - sec_failures += trsm_case(CblasColMajor, 0, 0, 0, 0, gate_mn, gate_mn, 1.0); + /* The gate case is here for the dispatch decision, not for accuracy: at + * 1300 rows the device solve is only good to about a percent whatever the + * build, so the tolerance is loose enough not to argue about roundoff that + * this path cannot avoid. The triangle, layout and scalar checks all live + * in the small cases below, where a float solve is accurate to ~1e-5. */ + sec_failures += + trsm_case(CblasColMajor, 0, 0, 0, 0, gate_mn, gate_mn, 1.0, 5e-2); cases++; printf("gate cases: %d cases, %d failure(s)\n", cases, sec_failures); failures += sec_failures; @@ -734,6 +830,29 @@ int main(int argc, char **argv) { sec_failures += symm_case(ord, side, uplo, 53, 130, 0.5, -0.25); cases++; } + + /* Scalar edges, same reasoning as for SYRK: alpha == 0 leaves beta*C, beta + * == 0 drops the old C, and a large scalar on one term only shows up as a + * mismatch rather than as a rounding artefact. */ + printf("symm scalar edges (size gate forced open)\n"); + for (order = 0; order <= 1; ++order) + for (side = 0; side <= 1; ++side) + for (uplo = 0; uplo <= 1; ++uplo) { + const int ord = order ? CblasRowMajor : CblasColMajor; + + sec_failures += symm_case(ord, side, uplo, 97, 61, 0.0, 1.0); + cases++; + sec_failures += symm_case(ord, side, uplo, 97, 61, 0.0, -0.5); + cases++; + sec_failures += symm_case(ord, side, uplo, 97, 61, 0.0, 0.0); + cases++; + sec_failures += symm_case(ord, side, uplo, 53, 130, -2.0, 1.0); + cases++; + sec_failures += symm_case(ord, side, uplo, 53, 130, 1000.0, 0.0); + cases++; + sec_failures += symm_case(ord, side, uplo, 53, 130, 1.0, 1000.0); + cases++; + } printf("symm: %d cases, %d failure(s)\n", cases, sec_failures); failures += sec_failures; @@ -752,6 +871,28 @@ int main(int argc, char **argv) { trmm_case(ord, side, uplo, trans, diag, 53, 130, -0.75); cases++; } + + /* alpha edges: zero must give exact zeros, the negative value must keep + * its sign, and the large one must not be folded into the accumulation at + * reduced precision. */ + printf("trmm alpha edges (size gate forced open)\n"); + for (order = 0; order <= 1; ++order) + for (side = 0; side <= 1; ++side) + for (uplo = 0; uplo <= 1; ++uplo) + for (trans = 0; trans <= 1; ++trans) + for (diag = 0; diag <= 1; ++diag) { + const int ord = order ? CblasRowMajor : CblasColMajor; + + sec_failures += + trmm_case(ord, side, uplo, trans, diag, 97, 61, 0.0); + cases++; + sec_failures += + trmm_case(ord, side, uplo, trans, diag, 97, 61, -2.0); + cases++; + sec_failures += + trmm_case(ord, side, uplo, trans, diag, 53, 130, 1000.0); + cases++; + } printf("trmm: %d cases, %d failure(s)\n", cases, sec_failures); failures += sec_failures; @@ -764,10 +905,33 @@ int main(int argc, char **argv) { for (diag = 0; diag <= 1; ++diag) { const int ord = order ? CblasRowMajor : CblasColMajor; - sec_failures += trsm_case(ord, side, uplo, trans, diag, 97, 61, 1.0); + sec_failures += + trsm_case(ord, side, uplo, trans, diag, 97, 61, 1.0, 1e-3); + cases++; + sec_failures += + trsm_case(ord, side, uplo, trans, diag, 53, 130, 0.75, 1e-3); + cases++; + } + + /* alpha edges for the solve too: zero gives exact zeros, the negative + * value must keep its sign, and the large one must scale the whole right + * hand side rather than only part of it. */ + printf("trsm alpha edges (size gate forced open)\n"); + for (order = 0; order <= 1; ++order) + for (side = 0; side <= 1; ++side) + for (uplo = 0; uplo <= 1; ++uplo) + for (trans = 0; trans <= 1; ++trans) + for (diag = 0; diag <= 1; ++diag) { + const int ord = order ? CblasRowMajor : CblasColMajor; + + sec_failures += + trsm_case(ord, side, uplo, trans, diag, 97, 61, 0.0, 1e-3); + cases++; + sec_failures += + trsm_case(ord, side, uplo, trans, diag, 97, 61, -2.0, 1e-3); cases++; sec_failures += - trsm_case(ord, side, uplo, trans, diag, 53, 130, 0.75); + trsm_case(ord, side, uplo, trans, diag, 53, 130, 1000.0, 1e-3); cases++; } printf("trsm: %d cases, %d failure(s)\n", cases, sec_failures); From 4377a76d5e7745c413265751bc5ea0b9928a56b7 Mon Sep 17 00:00:00 2001 From: rr Date: Fri, 6 Nov 2026 14:29:43 +0530 Subject: [PATCH 10/13] Let the level-3 comparisons see a non-finite result MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A tolerance test cannot detect one: every comparison against NaN is false, so a bridge that answered 0.0 for a NaN input, or produced a NaN where a number was due, passed silently. Results are now compared against NaN and the infinities explicitly, and the level-3 case helpers take a planted value so an operand can carry one. Generated with Codebuff 🤖 Co-Authored-By: Codebuff --- gpu/rdna2/test_oblas_gpu.c | 183 +++++++++++++++++++++++++++---------- 1 file changed, 134 insertions(+), 49 deletions(-) diff --git a/gpu/rdna2/test_oblas_gpu.c b/gpu/rdna2/test_oblas_gpu.c index 70b3c4e370..fd924bbb42 100644 --- a/gpu/rdna2/test_oblas_gpu.c +++ b/gpu/rdna2/test_oblas_gpu.c @@ -107,6 +107,53 @@ static void mat_put(float *x, int ld, int order, int i, int j, float v) { x[(size_t)i * ld + (size_t)j] = v; } +/* ------------------------------------------------------------------------- */ +/* Non-finite inputs */ +/* ------------------------------------------------------------------------- */ + +/* NaN and the two infinities are the three ways a float stops being an ordinary + * number, and a BLAS must carry each of them through: a NaN that arrives as 0.0, + * or an infinity folded into a finite result, is a silently wrong answer, which + * is the worst failure a library like this can have. The values are planted in + * an operand the routine has to read, so the result either says NaN/inf where + * the arithmetic says it must, or the case fails. */ +enum { PLANT_NONE = 0, PLANT_NAN, PLANT_POS_INF, PLANT_NEG_INF }; + +static float plant_value(int kind) { + switch (kind) { + case PLANT_NAN: + return (float)NAN; + case PLANT_POS_INF: + return (float)INFINITY; + case PLANT_NEG_INF: + return -(float)INFINITY; + default: + return 0.0f; + } +} + +/* Plants one value at (i,j) of an operand. The position is always on the + * diagonal of the operand: every uplo stores it, and it means the same element + * under transpose, so one position serves every layout. */ +static void plant(float *x, int ld, int order, int i, int j, int kind) { + if (kind != PLANT_NONE) mat_put(x, ld, order, i, j, plant_value(kind)); +} + +/* Plants one value in a stored off-diagonal of a triangular factor, which is + * read whatever the diag flag says (the diagonal is not, when unit is set). */ +static void plant_tri(float *a, int ld, int order, int uplo, int kind) { + if (kind == PLANT_NONE) return; + if (uplo == 0) + mat_put(a, ld, order, 0, 1, plant_value(kind)); + else + mat_put(a, ld, order, 1, 0, plant_value(kind)); +} + +/* Defined with the SYMM/TRMM/TRSM helpers below, used by the SYRK reference + * above them as well. */ +static void check_el(const char *tag, int i, int j, double got, double want, + double tol, long *nviol, int *bad); + /* Bytes to allocate for an m x n matrix with row or column stride ld. */ static size_t mat_size(int ld, int order, int m, int n) { return (order == CblasColMajor) ? (size_t)ld * (size_t)n @@ -135,7 +182,7 @@ static double tri_el(const float *a, int ld, int order, int uplo, int tr, /* Runs one SYRK (two == 0) or SYR2K (two == 1) case and compares it against a * double precision reference. Returns 0 when the case passed. */ static int sym_case(int two, int order, int uplo, int trans, int n, int k, - double alpha, double beta) { + double alpha, double beta, int plant_a, int plant_b) { const int ta = trans ? CblasTrans : CblasNoTrans; const CBLAS_UPLO up = uplo ? CblasLower : CblasUpper; const int ldc = n + 5; @@ -171,6 +218,8 @@ static int sym_case(int two, int order, int uplo, int trans, int n, int k, A[i] = (float)((double)((i * 7) % 23) - 11.0) / 9.0f; B[i] = (float)((double)((i * 11) % 19) - 9.0) / 7.0f; } + plant(A, lda, order, 1, 1, plant_a); + plant(B, ldb, order, 1, 1, plant_b); /* C starts at a value that no result can produce, so an element the routine * never writes is detectable. */ for (i = 0; i < csz; ++i) C0[i] = C[i] = (float)(i % 17) * 0.5f + sentinel; @@ -205,7 +254,7 @@ static int sym_case(int two, int order, int uplo, int trans, int n, int k, continue; } { - double ref = 0.0, got, err, scale, mag = 0.0, tol; + double ref = 0.0, mag = 0.0, tol; int p; for (p = 0; p < k; ++p) { @@ -224,14 +273,9 @@ static int sym_case(int two, int order, int uplo, int trans, int n, int k, } } ref = alpha * ref + beta * (double)C0[idx]; - got = (double)C[idx]; - err = fabs(got - ref); - scale = fabs(ref) > 1.0 ? fabs(ref) : 1.0; - tol = tol_for(scale, fabs(alpha) * mag + fabs(beta) * fabs((double)C0[idx])); - if (err / scale > tol) { - violation(&nviol, "syrk/syr2k", row, col, NULL, got, ref, err / scale); - bad = 1; - } + tol = tol_for(fabs(ref) > 1.0 ? fabs(ref) : 1.0, + fabs(alpha) * mag + fabs(beta) * fabs((double)C0[idx])); + check_el("syrk/syr2k", row, col, (double)C[idx], ref, tol, &nviol, &bad); } } @@ -287,11 +331,42 @@ static int pad_intact(const char *tag, const float *x, const float *x0, int ld, return bad; } -/* Compares one result element against the reference and reports a mismatch. */ +/* Compares one result element against the reference and reports a mismatch. + * + * The non-finite cases have to be spelled out rather than left to the + * tolerance: every comparison against NaN is false, so `fabs(got - want) > tol` + * quietly passes when the reference is NaN and also when the result is, and a + * result of 0.0 where the arithmetic says NaN would look perfect. So a NaN + * reference demands a NaN result, an infinity demands an infinity of the same + * sign, and a finite reference demands a finite result before the tolerance is + * allowed to judge it. */ static void check_el(const char *tag, int i, int j, double got, double want, double tol, long *nviol, int *bad) { - const double err = fabs(got - want); - const double scale = fabs(want) > 1.0 ? fabs(want) : 1.0; + double err, scale; + + if (isnan(want) || isinf(want) || !isfinite(got)) { + const int ok = isnan(want) ? isnan(got) + : isinf(want) ? (isinf(got) && ((got > 0.0) == (want > 0.0))) + : 0; + + if (!ok) { + if ((*nviol)++ < SYM_REPORT_MAX) { + if (isnan(want)) + printf("%s: (%d,%d) got %g, expected NaN\n", tag, i, j, got); + else if (isinf(want)) + printf("%s: (%d,%d) got %g, expected %s infinity\n", tag, i, j, got, + want > 0.0 ? "+" : "-"); + else + printf("%s: (%d,%d) got %g, expected a finite value\n", tag, i, j, + got); + } + *bad = 1; + } + return; + } + + err = fabs(got - want); + scale = fabs(want) > 1.0 ? fabs(want) : 1.0; if (err / scale > tol) { violation(nviol, tag, i, j, NULL, got, want, err / scale); @@ -303,7 +378,7 @@ static void check_el(const char *tag, int i, int j, double got, double want, * symmetric with only its uplo triangle stored. Checks every element of C, the * ldc padding, and that A's other triangle is never read. */ static int symm_case(int order, int side, int uplo, int m, int n, double alpha, - double beta) { + double beta, int plant_a, int plant_b) { const int kdim = side ? n : m; const int lda = kdim + 3; const int ldb = (m > n ? m : n) + 4; @@ -345,6 +420,8 @@ static int symm_case(int order, int side, int uplo, int m, int n, double alpha, mat_put(B, ldb, order, i, j, v); mat_put(C0, ldc, order, i, j, v * 0.5f + 0.25f); } + plant(A, lda, order, 1, 1, plant_a); + plant(B, ldb, order, 1, 1, plant_b); memcpy(C, C0, csz * sizeof(float)); cblas_ssymm(order, side ? CblasRight : CblasLeft, @@ -392,7 +469,7 @@ static int symm_case(int order, int side, int uplo, int m, int n, double alpha, * stored diagonal of a unit-diagonal factor carry UNUSED_FILL, so reading * either one shows up as a large error. */ static int trmm_case(int order, int side, int uplo, int trans, int unit, int m, - int n, double alpha) { + int n, double alpha, int plant_a, int plant_b) { const int kdim = side ? n : m; const int lda = kdim + 3; const int ldb = (m > n ? m : n) + 4; @@ -425,6 +502,8 @@ static int trmm_case(int order, int side, int uplo, int trans, int unit, int m, for (i = 0; i < m; ++i) mat_put(B0, ldb, order, i, j, (float)((double)((i * 7 + j * 11) % 23) - 11.0) / 11.0f); + plant_tri(A, lda, order, uplo, plant_a); + plant(B, ldb, order, 1, 1, plant_b); memcpy(B, B0, bsz * sizeof(float)); cblas_strmm(order, side ? CblasRight : CblasLeft, @@ -528,7 +607,7 @@ static double tri_diag(int i) { return 1.0 + 0.5 * ((i % 3) + 1) / 4.0; } * place on B, A triangular. The reference solves a dense copy of op(A) with a * pivoting LU, so it does not share the triangle logic under test. */ static int trsm_case(int order, int side, int uplo, int trans, int unit, int m, - int n, double alpha, double floor_tol) { + int n, double alpha, double floor_tol, int plant_b) { const int kdim = side ? n : m; const int lda = kdim + 3; const int ldb = (m > n ? m : n) + 4; @@ -589,6 +668,12 @@ static int trsm_case(int order, int side, int uplo, int trans, int unit, int m, for (i = 0; i < m; ++i) mat_put(B0, ldb, order, i, j, (float)((double)((i * 7 + j * 11) % 23) - 11.0) / 11.0f); + /* Only the right hand side is poisoned here: a NaN in B stays in its own + * column, which the reference reproduces exactly. A NaN in the factor would + * be the more natural target, but the LU reference cannot tell a NaN pivot + * from a good one, so it would answer a question of its own rather than the + * bridge's -- TRMM and SYMM cover a poisoned factor instead. */ + plant(B, ldb, order, 1, 1, plant_b); memcpy(B, B0, bsz * sizeof(float)); cblas_strsm(order, side ? CblasRight : CblasLeft, @@ -724,9 +809,9 @@ int main(int argc, char **argv) { SETENV("OPENBLAS_RDNA2_GPU", "1"); printf("\nsyrk/syr2k above the default size gate (n=k=%d)\n", gate_n); - sym_failures += sym_case(0, CblasColMajor, 0, 0, gate_n, gate_n, 1.0, 0.0); + sym_failures += sym_case(0, CblasColMajor, 0, 0, gate_n, gate_n, 1.0, 0.0, 0, 0); cases++; - sym_failures += sym_case(1, CblasColMajor, 0, 0, gate_n, gate_n, 1.0, 0.0); + sym_failures += sym_case(1, CblasColMajor, 0, 0, gate_n, gate_n, 1.0, 0.0, 0, 0); cases++; /* The gate is not restored afterwards: the remaining cases are small on @@ -740,14 +825,14 @@ int main(int argc, char **argv) { for (trans = 0; trans <= 1; ++trans) { const int ord = order ? CblasRowMajor : CblasColMajor; - sym_failures += sym_case(0, ord, uplo, trans, 97, 61, 1.0, 0.0); + sym_failures += sym_case(0, ord, uplo, trans, 97, 61, 1.0, 0.0, 0, 0); cases++; - sym_failures += sym_case(1, ord, uplo, trans, 97, 61, 1.0, 0.0); + sym_failures += sym_case(1, ord, uplo, trans, 97, 61, 1.0, 0.0, 0, 0); cases++; /* alpha/beta not 1/0: the triangle merge on the host path. */ - sym_failures += sym_case(0, ord, uplo, trans, 53, 130, 0.5, -0.25); + sym_failures += sym_case(0, ord, uplo, trans, 53, 130, 0.5, -0.25, 0, 0); cases++; - sym_failures += sym_case(1, ord, uplo, trans, 53, 130, 0.5, -0.25); + sym_failures += sym_case(1, ord, uplo, trans, 53, 130, 0.5, -0.25, 0, 0); cases++; } @@ -763,19 +848,19 @@ int main(int argc, char **argv) { int two; for (two = 0; two <= 1; ++two) { - sym_failures += sym_case(two, ord, uplo, trans, 97, 61, 0.0, 1.0); + sym_failures += sym_case(two, ord, uplo, trans, 97, 61, 0.0, 1.0, 0, 0); cases++; - sym_failures += sym_case(two, ord, uplo, trans, 97, 61, 0.0, -0.5); + sym_failures += sym_case(two, ord, uplo, trans, 97, 61, 0.0, -0.5, 0, 0); cases++; - sym_failures += sym_case(two, ord, uplo, trans, 97, 61, 0.0, 0.0); + sym_failures += sym_case(two, ord, uplo, trans, 97, 61, 0.0, 0.0, 0, 0); cases++; - sym_failures += sym_case(two, ord, uplo, trans, 53, 130, -2.0, 1.0); + sym_failures += sym_case(two, ord, uplo, trans, 53, 130, -2.0, 1.0, 0, 0); cases++; sym_failures += - sym_case(two, ord, uplo, trans, 53, 130, 1000.0, 0.0); + sym_case(two, ord, uplo, trans, 53, 130, 1000.0, 0.0, 0, 0); cases++; sym_failures += - sym_case(two, ord, uplo, trans, 53, 130, 1.0, 1000.0); + sym_case(two, ord, uplo, trans, 53, 130, 1.0, 1000.0, 0, 0); cases++; } } @@ -798,9 +883,9 @@ int main(int argc, char **argv) { SETENV("OPENBLAS_RDNA2_GPU_MIN_MNK", "2000000000"); printf("\nsymm/trmm/trsm at the default size gate (m=n=%d)\n", gate_mn); cases = sec_failures = 0; - sec_failures += symm_case(CblasColMajor, 0, 0, gate_mn, gate_mn, 1.0, 0.0); + sec_failures += symm_case(CblasColMajor, 0, 0, gate_mn, gate_mn, 1.0, 0.0, 0, 0); cases++; - sec_failures += trmm_case(CblasColMajor, 0, 0, 0, 0, gate_mn, gate_mn, 1.0); + sec_failures += trmm_case(CblasColMajor, 0, 0, 0, 0, gate_mn, gate_mn, 1.0, 0, 0); cases++; /* The gate case is here for the dispatch decision, not for accuracy: at * 1300 rows the device solve is only good to about a percent whatever the @@ -808,7 +893,7 @@ int main(int argc, char **argv) { * this path cannot avoid. The triangle, layout and scalar checks all live * in the small cases below, where a float solve is accurate to ~1e-5. */ sec_failures += - trsm_case(CblasColMajor, 0, 0, 0, 0, gate_mn, gate_mn, 1.0, 5e-2); + trsm_case(CblasColMajor, 0, 0, 0, 0, gate_mn, gate_mn, 1.0, 5e-2, 0); cases++; printf("gate cases: %d cases, %d failure(s)\n", cases, sec_failures); failures += sec_failures; @@ -825,9 +910,9 @@ int main(int argc, char **argv) { for (uplo = 0; uplo <= 1; ++uplo) { const int ord = order ? CblasRowMajor : CblasColMajor; - sec_failures += symm_case(ord, side, uplo, 97, 61, 1.0, 0.0); + sec_failures += symm_case(ord, side, uplo, 97, 61, 1.0, 0.0, 0, 0); cases++; - sec_failures += symm_case(ord, side, uplo, 53, 130, 0.5, -0.25); + sec_failures += symm_case(ord, side, uplo, 53, 130, 0.5, -0.25, 0, 0); cases++; } @@ -840,17 +925,17 @@ int main(int argc, char **argv) { for (uplo = 0; uplo <= 1; ++uplo) { const int ord = order ? CblasRowMajor : CblasColMajor; - sec_failures += symm_case(ord, side, uplo, 97, 61, 0.0, 1.0); + sec_failures += symm_case(ord, side, uplo, 97, 61, 0.0, 1.0, 0, 0); cases++; - sec_failures += symm_case(ord, side, uplo, 97, 61, 0.0, -0.5); + sec_failures += symm_case(ord, side, uplo, 97, 61, 0.0, -0.5, 0, 0); cases++; - sec_failures += symm_case(ord, side, uplo, 97, 61, 0.0, 0.0); + sec_failures += symm_case(ord, side, uplo, 97, 61, 0.0, 0.0, 0, 0); cases++; - sec_failures += symm_case(ord, side, uplo, 53, 130, -2.0, 1.0); + sec_failures += symm_case(ord, side, uplo, 53, 130, -2.0, 1.0, 0, 0); cases++; - sec_failures += symm_case(ord, side, uplo, 53, 130, 1000.0, 0.0); + sec_failures += symm_case(ord, side, uplo, 53, 130, 1000.0, 0.0, 0, 0); cases++; - sec_failures += symm_case(ord, side, uplo, 53, 130, 1.0, 1000.0); + sec_failures += symm_case(ord, side, uplo, 53, 130, 1.0, 1000.0, 0, 0); cases++; } printf("symm: %d cases, %d failure(s)\n", cases, sec_failures); @@ -865,10 +950,10 @@ int main(int argc, char **argv) { for (diag = 0; diag <= 1; ++diag) { const int ord = order ? CblasRowMajor : CblasColMajor; - sec_failures += trmm_case(ord, side, uplo, trans, diag, 97, 61, 1.0); + sec_failures += trmm_case(ord, side, uplo, trans, diag, 97, 61, 1.0, 0, 0); cases++; sec_failures += - trmm_case(ord, side, uplo, trans, diag, 53, 130, -0.75); + trmm_case(ord, side, uplo, trans, diag, 53, 130, -0.75, 0, 0); cases++; } @@ -884,13 +969,13 @@ int main(int argc, char **argv) { const int ord = order ? CblasRowMajor : CblasColMajor; sec_failures += - trmm_case(ord, side, uplo, trans, diag, 97, 61, 0.0); + trmm_case(ord, side, uplo, trans, diag, 97, 61, 0.0, 0, 0); cases++; sec_failures += - trmm_case(ord, side, uplo, trans, diag, 97, 61, -2.0); + trmm_case(ord, side, uplo, trans, diag, 97, 61, -2.0, 0, 0); cases++; sec_failures += - trmm_case(ord, side, uplo, trans, diag, 53, 130, 1000.0); + trmm_case(ord, side, uplo, trans, diag, 53, 130, 1000.0, 0, 0); cases++; } printf("trmm: %d cases, %d failure(s)\n", cases, sec_failures); @@ -906,10 +991,10 @@ int main(int argc, char **argv) { const int ord = order ? CblasRowMajor : CblasColMajor; sec_failures += - trsm_case(ord, side, uplo, trans, diag, 97, 61, 1.0, 1e-3); + trsm_case(ord, side, uplo, trans, diag, 97, 61, 1.0, 1e-3, 0); cases++; sec_failures += - trsm_case(ord, side, uplo, trans, diag, 53, 130, 0.75, 1e-3); + trsm_case(ord, side, uplo, trans, diag, 53, 130, 0.75, 1e-3, 0); cases++; } @@ -925,13 +1010,13 @@ int main(int argc, char **argv) { const int ord = order ? CblasRowMajor : CblasColMajor; sec_failures += - trsm_case(ord, side, uplo, trans, diag, 97, 61, 0.0, 1e-3); + trsm_case(ord, side, uplo, trans, diag, 97, 61, 0.0, 1e-3, 0); cases++; sec_failures += - trsm_case(ord, side, uplo, trans, diag, 97, 61, -2.0, 1e-3); + trsm_case(ord, side, uplo, trans, diag, 97, 61, -2.0, 1e-3, 0); cases++; sec_failures += - trsm_case(ord, side, uplo, trans, diag, 53, 130, 1000.0, 1e-3); + trsm_case(ord, side, uplo, trans, diag, 53, 130, 1000.0, 1e-3, 0); cases++; } printf("trsm: %d cases, %d failure(s)\n", cases, sec_failures); From 0ed5c879ffa08747408334d25c1fe47c1d080b42 Mon Sep 17 00:00:00 2001 From: rr Date: Fri, 6 Nov 2026 15:12:34 +0530 Subject: [PATCH 11/13] Describe the six level-3 bridges in the workflow header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The header still advertised SGEMM and SYRK/SYR2K only, so a reader could not tell that SYMM, TRMM and TRSM are covered too, nor that the cases sweep every layout with zero, negative and large alpha/beta. It also leaves the two known gaps on record: nothing feeds a NaN or infinity into an input yet, and the path filters miss interface/symm.c, interface/trsm.c and gemm.c. 🤖 Generated with Codebuff Co-Authored-By: Codebuff --- .github/workflows/rdna2_level3.yml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rdna2_level3.yml b/.github/workflows/rdna2_level3.yml index 7e0664a2d3..8dc2b872ee 100644 --- a/.github/workflows/rdna2_level3.yml +++ b/.github/workflows/rdna2_level3.yml @@ -1,12 +1,26 @@ name: rdna2 level-3 bridge test # Builds the library and runs gpu/rdna2/test_oblas_gpu.c, which checks the CBLAS -# level-3 entry points (SGEMM and SYRK/SYR2K) against its own double-precision -# reference. The test needs no GPU: on a CPU-only build it still verifies values -# and the SYRK contract that the unstored triangle and the ldc padding are left -# untouched. The GPU dispatch itself is only exercised on a build made with +# level-3 entry points the rdna2 bridges serve -- SGEMM, SYMM, SYRK, SYR2K, TRMM +# and TRSM -- against double-precision references the test computes itself. It +# covers every order/side/uplo/trans/diag combination, alpha and beta including +# zero, negative and large values, and the contracts the bridges owe the caller: +# the triangle they were not asked to store, the stored diagonal of a +# unit-diagonal factor and the ldb/ldc padding have to come back untouched. +# The per-op comparisons treat a non-finite result as a failure instead of +# letting it past a tolerance test that is always false for NaN. Nothing here +# needs a GPU: on a CPU-only build the same cases still check the values and +# those contracts. The dispatch itself is only exercised by a build made with # USE_HIP=1 on a machine with an AMD card, which hosted runners do not have -- # the "gpu" job at the bottom is the place to add that on a self-hosted runner. +# +# Two gaps are written down here rather than left implicit. No case feeds a NaN +# or an infinity into an input yet, so the non-finite handling above guards a +# result that arithmetic produced; it is not a tested scenario, and the SGEMM +# block still judges by max |diff| alone, where a NaN result passes silently. +# The path filters below also omit interface/symm.c, interface/trsm.c (TRMM and +# TRSM) and interface/gemm.c, so an edit to one of those bridge files does not +# start this workflow on its own. on: push: From a82eae5b5bb400cbb03492bab20eceadf3491e9c Mon Sep 17 00:00:00 2001 From: rr Date: Fri, 6 Nov 2026 16:43:56 +0530 Subject: [PATCH 12/13] Record RDNA2 measurements on an RX 6700 XT and where the library gain goes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README still described the port as never having run on hardware. Add the Windows/ROCm build and DLL-resolution steps that made it run, the level-3 and int8 results, and per-shape GEMM timings: the device wins 4.5x over the CPU end to end, but the bridge's per-call column-major to row-major pass over A costs more than the kernel and leaves 1.2-1.6x at the library. 🤖 Generated with Codebuff Co-Authored-By: Codebuff --- gpu/rdna2/README.md | 127 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 122 insertions(+), 5 deletions(-) diff --git a/gpu/rdna2/README.md b/gpu/rdna2/README.md index f50684152e..99a6fbad21 100644 --- a/gpu/rdna2/README.md +++ b/gpu/rdna2/README.md @@ -310,6 +310,34 @@ The resulting library needs the HIP runtime at link time: `make` adds `-L$(HIP_LIB_PATH) -lamdhip64 -lstdc++` for its own tests, and a program linking the static archive must add the same. +On Windows with the ROCm HIP SDK three extra steps are environment, not code: + +- `hipcc` cannot find the device libraries unless it is told where they are. + clang looks under `lib/amdgcn`, while HIP 7.x installs them in + `lib/llvm/amdgcn/bitcode`, so `rdna2_kernels.hip` fails with *cannot find + ROCm device library*: + + make USE_HIP=1 GPU_ARCH=gfx1031 HIPFLAGS="-O2 -std=c++17 --rocm-path=C:/Rocm10/lib/llvm" libs + + `--rocm-device-lib-path` looks like the right flag for this and is not: clang + 23 accepts it and still fails to resolve the bitcode. + +- The SDK ships only the MSVC import library (`amdhip64.lib`), so a mingw link + needs one generated from the DLL: + + gendef C:/Rocm10/bin/amdhip64_7.dll + dlltool -d amdhip64_7.def -l libamdhip64.a + +- Which `amdhip64_*.dll` the process loads decides whether it sees a device at + all. The SDK installs a versioned runtime next to `hipcc` and the display + driver installs another one into `System32`, which Windows resolves first + unless the executable sits in the SDK's `bin`. Against the driver's copy + `hipGetDeviceCount` returns `hipErrorNoDevice` (100) even though `hipInfo.exe` + lists the card, and every launcher silently takes its CPU reference path (see + *CPU fallback is silent*). Run the test binaries from the SDK `bin` + directory, or copy the SDK's `amdhip64_7.dll` beside them, before trusting any + GPU number. + ### Level-3 bridge test make level3-test # build the library, then run the test @@ -613,17 +641,106 @@ Correctness through the library (MATCH = max |diff| < 1e-3): - Library transpose is now tiled 64×64 (cache-resident); the naive copy was the ~200 ms strided-store bottleneck at k=2048. -Coverage limits: everything above is one RDNA4 card (gfx1201). - The gfx1030 and -gfx1031 paths are compiled and host-simulated but were not executed on RDNA2 -hardware, so the dp4a-in-LDS timing on RDNA2 is the same code measured only on -RDNA4. The shuffle-based emulation of the matrix instruction is compiled (its +Coverage limits: the timing tables in *Performance* are one RDNA4 card +(gfx1201) and are not re-timed on RDNA2. The gfx1030/gfx1031 path has since +been executed through the library on an RX 6700 XT - see *RDNA2 hardware* - +so the dispatch and the numerics are covered there, while the kernel-level +shapes remain RDNA4-only measurements. + +The shuffle-based emulation of the matrix instruction is compiled (its branch is visible in a fresh gfx1031 dump: 0 `v_wmma_*`, 167 `v_dot4c_i32_i8`, 83 `ds_bpermute_b32` of which 64 are the emulation's lane gathers, 0 warnings) and never selected by a dispatcher, but it is not covered by a runtime device test. Wave-level behaviour is covered on the device; the 141 checks are functional, not a stress or fuzz suite. +## RDNA2 hardware (measured on an RX 6700 XT, gfx1031) + +The gfx1031 path has now been executed on a real RDNA2 card: RX 6700 XT (20 CUs +as `hipGetDeviceProperties` reports them, 12.0 GB), HIP 7.16, clang 23, +Windows 11, 12 host threads, `-O2`, with a library built +`USE_HIP=1 GPU_ARCH=gfx1031` as described above. gfx1030 is the same ISA family +and is still compiled and host-simulated only. Through that library, end to end +and including the transfers: + +| size | CPU f32 GFLOPS (`OPENBLAS_RDNA2_GPU=0`) | GPU f32 GFLOPS | ratio | +|---|---|---|---| +| 1024^3 | 620.6 | 574.2 (below the 2e9 gate, so CPU) | 1.00x | +| 2048^3 | 589.8 | **693.3** | **1.18x** | +| 4096^3 | 654.5 | **1031.2** | **1.58x** | + +On RDNA2 the bridge is a real but modest win - about 1.2x at 2048 and 1.6x at +4096 against a 12-thread host - and the default gate keeps everything below +roughly 1260^3 on the CPU, where the device would lose. That is well short of +the 2.6-4.1x the same dispatch gives on RDNA4 (above), which is what the kernel +shapes predict: gfx12 has the 16x16x16 matrix instruction for int8 and the +128x128 f32 tile, while gfx1031 has neither and runs the 64x64 dp4a (int8) and +the 64x64 KSTEP-16 (f32) shapes. + +The level-3 bridge suite (`test_oblas_gpu.exe`) with the device visible: +**517 cases, 0 failures**, every op dispatch-confirmed, with the above-gate 1300 +cases timed at ssymm 10 ms, ssyrk 11 ms, ssyr2k 20 ms, strmm 13 ms, trsm 110 ms, +and `sgemm 2048` 201 ms on the first call (25 ms on the same call warmed up; the +first one carries the code-object load). + +int8 through the C API (`test_gemm_i8.c`): **185 checks, 0 failures**, both paths +exact, and 512x512xK4=128 in 3.0 ms on the device against 87.0 ms on the CPU +path - a 29x win. + +### Where the 1.2x goes + +Per shape, end to end through `rdna2_sgemm_f32_host_v` - allocation, upload and +download included, no bridge repacking - same card, `C = A * Bt^T`, ms / GFLOPS: + +| M = N = K | v0 (global 4x4) | v1 (64x64, KSTEP 8) | v2 (128x128) | **v3 (64x64, KSTEP 16)** | +|---|---|---|---|---| +| 1024 | 21.0 / 102 | 2.42 / 886 | 2.46 / 874 | **1.85 / 1161** | +| 2048 | 107.1 / 160 | 8.73 / 1968 | 9.31 / 1846 | **8.28 / 2074** | +| 4096 | 779.0 / 176 | 53.2 / 2584 | 49.2 / 2796 | **47.0 / 2925** | + +`rdna2_sgemm_f32_host_tuned` is v3 and reproduces that column. v3 is already the +arch default, and v0 - the shape the first version of this tree shipped - is +11-17x slower, so the kernel choice costs nothing here; the loss is elsewhere. + +The same problem through `cblas_sgemm` costs 25.0 ms at 2048 and 133 ms at 4096, +against 8.28 ms and 47.0 ms for the call above. The difference, 16.7 ms and +86.0 ms, is `rdna2_gemm_cm_core`'s unconditional host work: it allocates an +`m x k` buffer and runs `rdna2_cm_to_rm` over A on every call. Nothing else is +in it - B passes through untouched when `ldb == k`, and with `alpha = 1`, +`beta = 0`, `ldc == m` the kernel writes directly into C, so there is no output +pass and no temporary. Those copies move 16.8 MB at 2048 and 67.1 MB at 4096 at +an effective 1.0 and 0.78 GB/s: the 64x64 tiling does not make a column-major +read plus a row-strided store pay for itself, and the destination is a fresh +allocation, first-touched, on every call. + +| size | CPU (12 threads) | GPU call | bridge transpose | library total | ratio | without it | +|---|---|---|---|---|---|---| +| 2048^3 | 29.1 ms | 8.28 ms | 16.7 ms | 25.0 ms | 1.18x | 3.5x | +| 4096^3 | 210 ms | 47.0 ms | 86.0 ms | 133 ms | 1.58x | 4.5x | + +So the kernel and the transfers are not what limits RDNA2: the device does the +4096^3 product in 47 ms end to end, 4.5x the 12-thread CPU, and 2.9 TFLOP/s. A +single per-call column-major to row-major pass over A gives most of that back. +Removing it - a per-thread scratch buffer, transposing once when the caller +reuses A, or teaching the tiled staging loop to read column-major A, which is a +stride swap - is worth about 3x at the gate sizes, far more than further kernel +tuning. The same pass is the most likely reason the RDNA4 library numbers above +sit below the kernel-only tables there. + +### CPU fallback is silent + +This matters before reading any number above as a GPU result. When +`rdna2_device_count() == 0`, the host layers call the file's own `rdna2_*_ref` +CPU implementation, return `RDNA2_OK`, and the bridge logs `handled=1` exactly +as it does after a real launch. Reached through the `System32` `amdhip64_7.dll` +above, the first run of the level-3 suite reported 517 passes and 518 +`handled=1` lines **without touching the GPU**, at 3.1 GFLOPS - 200x slower than +the CPU - for 2048^3 and 4096^3 alike. The flat throughput and the absent size +dependence are the only symptoms. The same binary, run where the device is +visible, gives the 693 / 1031 GFLOPS and the 201 ms first-call timing in the +table above. Until the fallback returns a status distinct from a launch, +`handled=1` means "the call was answered", not "the device ran". + ## Not included - Full OpenBLAS (BLAS levels 1-3, LAPACK) on the GPU. OpenBLAS has no GPU backend; From d50ef9164811380892373c3c284f320c1d5a0ba4 Mon Sep 17 00:00:00 2001 From: rr Date: Fri, 6 Nov 2026 17:00:40 +0530 Subject: [PATCH 13/13] Add the RDNA2 CPU-vs-GPU benchmark harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Makefile has had bench and bench-run targets pointing at gpu/rdna2/bench_rdna2.cpp while the source itself was never in the tree, so neither target could run. Add it, now that its two reporting defects are fixed: the throughput column prints TF/s rather than a second copy of the GFLOPS figure, and the program refuses to time the CPU fallback by asking the library for its own device view and checking every launcher return code. 🤖 Generated with Codebuff Co-Authored-By: Codebuff --- gpu/rdna2/bench_rdna2.cpp | 434 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 434 insertions(+) create mode 100644 gpu/rdna2/bench_rdna2.cpp diff --git a/gpu/rdna2/bench_rdna2.cpp b/gpu/rdna2/bench_rdna2.cpp new file mode 100644 index 0000000000..c7333499dd --- /dev/null +++ b/gpu/rdna2/bench_rdna2.cpp @@ -0,0 +1,434 @@ +// CPU vs GPU comparison for the RDNA2 / RDNA4 HIP GEMM kernels. +// +// Everything below runs the *same* operands through both sides and reports +// measured time, GFLOPS and the GPU's numerical error against the CPU result, +// so a speedup is only ever claimed when this program printed one. +// +// GPU timings are end-to-end: the public host entry points allocate, copy in, +// launch, and copy out, exactly like a full cblas_sgemm call is end-to-end on +// the CPU. Scaling matrices to a device that cannot read host memory is not a +// transfer we can hide, so it is included rather than quietly excluded; the +// copy-only time is printed separately so the split is visible. +// +// The GPU columns are only produced when the library itself can see a device. +// Its host entry points run a scalar CPU reference when they cannot, and they +// return success while doing it, so no timing distinguishes that from a slow +// GPU: this program asks the library first and exits instead of printing it. +// +// CPU baselines: +// cpu1t - this file's blocked -O3 single-thread f32 gemm (always available) +// openblas - the OpenBLAS DLL shipped inside numpy, multithreaded, when the +// path is given or found. This is a tuned, threaded BLAS: the fair +// floor for the GPU to beat. +// +// Usage: +// bench_rdna2 [openblas.dll] [max_size] +// +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif + +extern "C" int rdna2_sgemm_f32_host_tuned(float* C, const float* A, const float* Bt, int M, int N, + int K); +extern "C" int rdna2_gemm_i8_host_tuned(int* C, const int* A, const int* Bt, int M, int N, int K4); +// sa is the M-element row scale (NULL for unit), sb the N-element column scale +// (NULL for unit); the zero point is B's, and the correction is subtracted from +// the finished int32 dot, never from a stored weight. +extern "C" int rdna2_gemm_i8_scale_host_tuned(float* C, const int* A, const int* Bt, int M, int N, + int K4, const float* sa, const float* sb, int zp_b); +// The library's own view of the device: the same hipGetDeviceCount() the host +// entry points use, so this is what decides whether they compute or fall back. +extern "C" int rdna2_device_info(char* name, size_t len, int* warp_size, int* arch_ok); + +// --------------------------------------------------------------------------- +// CPU reference: blocked f32 GEMM, C[MxN] = A[MxK] * Bt[NxK]^T +// +// Bt is packed row-major N x K (one row per output column), which is the layout +// the GPU kernels take. A tuned BLAS also repacks operands before its main +// loop, so transposing Bt into a K x N panel once, outside the timed region, +// mirrors that and keeps the loop below a plain k-outer accumulation. +// --------------------------------------------------------------------------- +static void cpu_sgemm_blocked(float* C, const float* A, const float* Bt_kn, int M, int N, int K) { + const int BS = 64; + for (int i = 0; i < M * N; ++i) C[i] = 0.0f; + for (int ii = 0; ii < M; ii += BS) { + const int iend = std::min(ii + BS, M); + for (int kk = 0; kk < K; kk += BS) { + const int kend = std::min(kk + BS, K); + for (int jj = 0; jj < N; jj += BS) { + const int jend = std::min(jj + BS, N); + for (int i = ii; i < iend; ++i) { + float* c = C + (size_t)i * N; + const float* a = A + (size_t)i * K; + for (int k = kk; k < kend; ++k) { + const float av = a[k]; + const float* b = Bt_kn + (size_t)k * N; + for (int j = jj; j < jend; ++j) c[j] += av * b[j]; + } + } + } + } + } +} + +// CPU reference for the int8 path: int32 accumulation of 4 dot4 groups, exactly +// the arithmetic the dp4a kernels perform, in scalar form. +static void cpu_gemm_i8(int* C, const int* A, const int* Bt, int M, int N, int K4) { + for (int i = 0; i < M; ++i) { + const int* a = A + (size_t)i * K4; + for (int j = 0; j < N; ++j) { + const int* b = Bt + (size_t)j * K4; + int acc = 0; + for (int k = 0; k < K4; ++k) { + int p = a[k], q = b[k]; + for (int t = 0; t < 4; ++t) { + int8_t x = (int8_t)((p >> (8 * t)) & 0xFF); + int8_t y = (int8_t)((q >> (8 * t)) & 0xFF); + acc += (int)x * (int)y; + } + } + C[(size_t)i * N + j] = acc; + } + } +} + +// --------------------------------------------------------------------------- +// Optional OpenBLAS baseline, loaded at run time so the bench still builds and +// runs where no such library exists. +// --------------------------------------------------------------------------- +typedef void (*cblas_sgemm64_fn)(int order, int transA, int transB, int64_t M, int64_t N, + int64_t K, float alpha, const float* A, int64_t lda, + const float* B, int64_t ldb, float beta, float* C, int64_t ldc); + +struct OpenBlas { + cblas_sgemm64_fn sgemm = nullptr; + void* lib = nullptr; + std::string path; + + bool open(const char* p) { + if (!p || !*p) return false; +#ifdef _WIN32 + lib = (void*)LoadLibraryA(p); + if (!lib) return false; + sgemm = (cblas_sgemm64_fn)GetProcAddress((HMODULE)lib, "scipy_cblas_sgemm64_"); + if (!sgemm) sgemm = (cblas_sgemm64_fn)GetProcAddress((HMODULE)lib, "cblas_sgemm"); +#else + lib = dlopen(p, RTLD_NOW); + if (!lib) return false; + sgemm = (cblas_sgemm64_fn)dlsym(lib, "scipy_cblas_sgemm64_"); + if (!sgemm) sgemm = (cblas_sgemm64_fn)dlsym(lib, "cblas_sgemm"); +#endif + path = p; + return sgemm != nullptr; + } + // C[MxN] = A[MxK] * Bt[NxK]^T + void run(float* C, const float* A, const float* Bt, int M, int N, int K) { + // CblasRowMajor=101, CblasNoTrans=111, CblasTrans=112 + sgemm(101, 111, 112, M, N, K, 1.0f, A, K, Bt, K, 0.0f, C, N); + } +}; + +// --------------------------------------------------------------------------- +static double now_s() { + using clock = std::chrono::steady_clock; + return std::chrono::duration(clock::now().time_since_epoch()).count(); +} + +template +static double best_of(int reps, F&& f) { + double best = 1e30; + for (int r = 0; r < reps; ++r) { + const double t0 = now_s(); + f(); + const double dt = now_s() - t0; + best = std::min(best, dt); + } + return best; +} + +static double gflops(double secs, int M, int N, int K) { + return (2.0 * (double)M * (double)N * (double)K) / (secs * 1e9); +} + +// best_of() for the launchers. They return 0 on success and non-zero when the +// call could not be made on the device, and a failed call returns immediately, +// so timing one would print as an excellent result. Stop at the first non-zero +// return code instead. +template +static double best_of_gpu(int reps, const char* op, int n, F&& f) { + double best = 1e30; + for (int r = 0; r < reps; ++r) { + const double t0 = now_s(); + const int rc = f(); + const double dt = now_s() - t0; + if (rc != 0) { + printf("bench: %s at n=%d returned %d - not a GPU run, aborting\n", op, n, rc); + exit(2); + } + best = std::min(best, dt); + } + return best; +} + +// Without a device the launchers run the scalar CPU reference and report +// success, so no timing can tell that apart from a slow GPU. Ask the library +// what it sees and refuse to run at all when the answer is "no device". +static void require_library_device(void) { + char name[256] = {0}; + int warp = 0, arch = 0; + const int rc = rdna2_device_info(name, sizeof(name), &warp, &arch); + printf("bench: library sees rc=%d name='%s' warp=%d arch=%d\n", rc, name, warp, arch); + if (rc != 0 || arch == 0 || name[0] == '\0') { + printf( + "bench: the library cannot see a device, so every GPU column below would be\n" + "bench: the CPU reference. Refusing to report that as a GPU result. Run this\n" + "bench: binary where the HIP runtime finds the card - on the ROCm Windows\n" + "bench: build, from the bin directory that holds amdhip64_*.dll.\n"); + exit(1); + } +} + +static void fill_rand(float* p, size_t n, unsigned seed) { + unsigned s = seed; + for (size_t i = 0; i < n; ++i) { + s = s * 1664525u + 1013904223u; + p[i] = (float)((s >> 8) & 0xFFFF) / 65535.0f - 0.5f; + } +} + +static void fill_rand_i8_packed(int* p, size_t n, unsigned seed) { + unsigned s = seed; + for (size_t i = 0; i < n; ++i) { + unsigned w = 0; + for (int t = 0; t < 4; ++t) { + s = s * 1664525u + 1013904223u; + w |= ((s >> 16) & 0xFF) << (8 * t); + } + p[i] = (int)w; + } +} + +static void report_error(const char* label, const float* got, const float* want, size_t n) { + double maxabs = 0.0, num = 0.0, den = 0.0, maxrel = 0.0; + for (size_t i = 0; i < n; ++i) { + const double d = (double)got[i] - (double)want[i]; + const double ad = d < 0 ? -d : d; + if (ad > maxabs) maxabs = ad; + num += d * d; + den += (double)want[i] * (double)want[i]; + const double ref = want[i] < 0 ? -(double)want[i] : (double)want[i]; + if (ref > 1e-3) { + const double rel = ad / ref; + if (rel > maxrel) maxrel = rel; + } + } + const double rms = den > 0 ? std::sqrt(num / den) : 0.0; + printf(" %-10s vs %-10s max_abs=%.3e max_rel=%.3e rms_rel=%.3e\n", label, "reference", + maxabs, maxrel, rms); +} + +int main(int argc, char** argv) { + setvbuf(stdout, nullptr, _IONBF, 0); + const char* ob_path = (argc > 1) ? argv[1] : nullptr; + int max_size = (argc > 2) ? atoi(argv[2]) : 4096; + if (max_size <= 0) max_size = 4096; + + printf("bench: hipsgemm cpu-vs-gpu\n"); + char devname[256] = {0}; + int ndev = 0; + const hipError_t dc = hipGetDeviceCount(&ndev); + printf("bench: hipGetDeviceCount -> %d, ndev=%d\n", (int)dc, ndev); + if (dc == hipSuccess && ndev > 0) { + hipDeviceProp_t prop{}; + if (hipGetDeviceProperties(&prop, 0) == hipSuccess) { + snprintf(devname, sizeof(devname), "%s", prop.name); + printf("GPU : %s (gcnArchName=%s, %d CUs)\n", prop.name, prop.gcnArchName, + prop.multiProcessorCount); + } + } else { + printf("GPU : none detected by the HIP runtime in this process\n"); + } + require_library_device(); + + OpenBlas ob; + if (!ob.open(ob_path)) { + printf("CPU : openblas baseline NOT loaded (pass its DLL path as argv[1])\n"); + } else { + printf("CPU : openblas baseline %s\n", ob.path.c_str()); + } + + printf("\nf32 SGEMM C[MxN] = A[MxK] * B[NxK]^T (GPU column is end-to-end, transfers included)\n"); + printf("(cpu1t, openblas and gpu are GFLOPS; the last column is the same rate in TF/s)\n"); + printf("%-16s %10s %10s %10s %10s %10s %10s\n", "M=N=K", "cpu1t", "openblas", "gpu", + "gpu/cpu1t", "gpu/oblas", "gpu_TF/s"); + + const int sizes[] = {256, 512, 1024, 2048, 4096}; + for (int si = 0; si < 5; ++si) { + const int n = sizes[si]; + if (n > max_size) break; + const size_t na = (size_t)n * n; + std::vector A(na), Bt(na), Bt_kn(na), C_cpu(na), C_ob(na), C_gpu(na); + fill_rand(A.data(), na, 12345u + n); + fill_rand(Bt.data(), na, 999u + n); + for (int j = 0; j < n; ++j) + for (int k = 0; k < n; ++k) Bt_kn[(size_t)k * n + j] = Bt[(size_t)j * n + k]; + + const int reps = n >= 2048 ? 3 : 5; + const double t_cpu = + best_of(reps, [&] { cpu_sgemm_blocked(C_cpu.data(), A.data(), Bt_kn.data(), n, n, n); }); + double t_ob = 0.0; + if (ob.sgemm) { + ob.run(C_ob.data(), A.data(), Bt.data(), n, n, n); // warm up threads + t_ob = best_of(reps, [&] { ob.run(C_ob.data(), A.data(), Bt.data(), n, n, n); }); + } + const double t_gpu = best_of_gpu(reps, "sgemm_f32", n, [&] { + return rdna2_sgemm_f32_host_tuned(C_gpu.data(), A.data(), Bt.data(), n, n, n); + }); + + const double gf_cpu = gflops(t_cpu, n, n, n), gf_gpu = gflops(t_gpu, n, n, n); + const double gf_ob = t_ob > 0 ? gflops(t_ob, n, n, n) : 0.0; + + char label[32]; + snprintf(label, sizeof(label), "%d", n); + printf("%-16s %10.1f %10.1f %10.1f %9.2fx %9s %10.3f\n", label, gf_cpu, gf_ob, gf_gpu, + gf_gpu / gf_cpu, t_ob > 0 ? (std::to_string(gf_gpu / gf_ob) + "x").c_str() : "-", + gf_gpu / 1e3); + + if (n == 1024 || n == max_size) { + report_error("gpu", C_gpu.data(), C_cpu.data(), na); + if (ob.sgemm) report_error("openblas", C_ob.data(), C_cpu.data(), na); + } + } + + // The host entry points allocate and copy on every call, so this is the part + // of the GPU column that is not math. Printed so the split is visible instead + // of being argued about. + printf("\nGPU-side fixed cost per call (hipMalloc x3 + copy in + copy out + free)\n"); + printf("%-16s %10s %10s\n", "M=N=K", "ms", "GB/s eff"); + for (int si = 0; si < 5; ++si) { + const int n = sizes[si]; + if (n > max_size) break; + const size_t bytes = (size_t)n * n * sizeof(float); + std::vector A(bytes / sizeof(float)), C(bytes / sizeof(float)); + const double t = best_of(3, [&] { + float *dA = nullptr, *dB = nullptr, *dC = nullptr; + if (hipMalloc(&dA, bytes) != hipSuccess || hipMalloc(&dB, bytes) != hipSuccess || + hipMalloc(&dC, bytes) != hipSuccess || + hipMemcpy(dA, A.data(), bytes, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(dB, A.data(), bytes, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(C.data(), dC, bytes, hipMemcpyDeviceToHost) != hipSuccess) { + printf("bench: hipMalloc/hipMemcpy failed at n=%d - no copy-only number\n", n); + exit(3); + } + (void)hipFree(dA); + (void)hipFree(dB); + (void)hipFree(dC); + }); + const double gbs = (3.0 * (double)bytes) / (t * 1e9); + char label[32]; + snprintf(label, sizeof(label), "%d", n); + printf("%-16s %10.3f %10.1f\n", label, t * 1e3, gbs); + } + + // --------------------------------------------------------------------- + // int8: packed int8 weights stay packed; only the accumulator leaves the + // dot in int32/float. GPU vs scalar CPU dp4a-equivalent arithmetic. + // --------------------------------------------------------------------- + printf("\nint8 GEMM int32 accumulate, 4 int8 per word (dp4a) - K4 words = 4*K4 columns\n"); + printf("%-16s %10s %10s %10s %12s\n", "M=N=K", "cpu1t", "gpu", "gpu/cpu1t", "gpu_Tops/s"); + const int isizes[] = {256, 512, 1024, 2048}; + for (int si = 0; si < 4; ++si) { + const int n = isizes[si]; + if (n > max_size) break; + const size_t na = (size_t)n * n; + std::vector A(na), Bt(na), C_cpu(na), C_gpu(na); + fill_rand_i8_packed(A.data(), na, 777u + n); + fill_rand_i8_packed(Bt.data(), na, 555u + n); + + const int reps = n >= 2048 ? 3 : 5; + const double t_cpu = + best_of(reps, [&] { cpu_gemm_i8(C_cpu.data(), A.data(), Bt.data(), n, n, n); }); + const double t_gpu = best_of_gpu(reps, "gemm_i8", n, [&] { + return rdna2_gemm_i8_host_tuned(C_gpu.data(), A.data(), Bt.data(), n, n, n); + }); + + const double ops = 2.0 * 4.0 * (double)n * (double)n * (double)n; // 2 flops per int8 MAC + const double tops_cpu = ops / (t_cpu * 1e12), tops_gpu = ops / (t_gpu * 1e12); + char label[32]; + snprintf(label, sizeof(label), "%d", n); + printf("%-16s %10.2f %10.2f %9.2fx %12.2f\n", label, tops_cpu, tops_gpu, tops_gpu / tops_cpu, + tops_gpu); + + long long maxdiff = 0; + for (size_t i = 0; i < na; ++i) { + long long d = (long long)C_gpu[i] - (long long)C_cpu[i]; + if (d < 0) d = -d; + if (d > maxdiff) maxdiff = d; + } + printf(" int8 gpu vs cpu: max |diff| = %lld (exact match: %s)\n", maxdiff, + maxdiff == 0 ? "yes" : "NO"); + } + + // Quantized path: int8 weights stay packed through the whole GEMM, and the + // scale / zero point is applied once to each finished int32 dot. + { + const int n = 1024, K4 = n; + const size_t na = (size_t)n * K4; + std::vector A(na), Bt(na), Ci((size_t)n * n); + std::vector C_gpu((size_t)n * n), C_ref((size_t)n * n); + std::vector sa(n), sb(n); + fill_rand_i8_packed(A.data(), na, 4242u); + fill_rand_i8_packed(Bt.data(), na, 2424u); + for (int i = 0; i < n; ++i) { + sa[i] = 1.0f / 127.0f; + sb[i] = 1.0f / 63.0f; + } + const int zp_b = 3; + + const double t_gpu = best_of_gpu(3, "gemm_i8_scale", n, [&] { + return rdna2_gemm_i8_scale_host_tuned(C_gpu.data(), A.data(), Bt.data(), n, n, K4, + sa.data(), sb.data(), zp_b); + }); + + // Reference: C[m][n] = (acc - zp_b * rowsum_a[m]) * sa[m] * sb[n], exactly + // what the epilogue does, with the int8 values never dequantised. + cpu_gemm_i8(Ci.data(), A.data(), Bt.data(), n, n, K4); + std::vector rowsum(n, 0); + for (int m = 0; m < n; ++m) { + const int* a = A.data() + (size_t)m * K4; + long long s = 0; + for (int k = 0; k < K4; ++k) + for (int t = 0; t < 4; ++t) s += (int)(int8_t)((a[k] >> (8 * t)) & 0xFF); + rowsum[m] = s; + } + for (int m = 0; m < n; ++m) { + for (int j = 0; j < n; ++j) { + long long v = (long long)Ci[(size_t)m * n + j] - (long long)zp_b * rowsum[m]; + C_ref[(size_t)m * n + j] = (float)((double)v * (double)sa[m] * (double)sb[j]); + } + } + report_error("quant epi", C_gpu.data(), C_ref.data(), C_gpu.size()); + printf(" quant epilogue 1024^3: %.3f ms (%.1f GFLOPS)\n", t_gpu * 1e3, + gflops(t_gpu, n, n, n)); + } + + return 0; +}