lib/crypto: x86/sm3: Migrate optimized code into library

Eric Biggers · Eric Biggers · commit 17ba6108d3e0 · 2026-03-23T17:50:59.000-07:00
Instead of exposing the x86-optimized SM3 code via an x86-specific crypto_shash algorithm, instead just implement the sm3_blocks() library function. This is much simpler, it makes the SM3 library functions be x86-optimized, and it fixes the longstanding issue where the x86-optimized SM3 code was disabled by default. SM3 still remains available through crypto_shash, but individual architectures no longer need to handle it. Tweak the prototype of sm3_transform_avx() to match what the library expects, including changing the block count to size_t. Note that the assembly code actually already treated this argument as size_t. Acked-by: Ard Biesheuvel <ardb@kernel.org> Link: https://lore.kernel.org/r/20260321040935.410034-10-ebiggers@kernel.org Signed-off-by: Eric Biggers <ebiggers@kernel.org>
diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig
@@ -331,17 +331,4 @@ config CRYPTO_AEGIS128_AESNI_SSE2
 	  - AES-NI (AES New Instructions)
 	  - SSE4.1 (Streaming SIMD Extensions 4.1)
 
-config CRYPTO_SM3_AVX_X86_64
-	tristate "Hash functions: SM3 (AVX)"
-	depends on 64BIT
-	select CRYPTO_HASH
-	select CRYPTO_LIB_SM3
-	help
-	  SM3 secure hash function as defined by OSCCA GM/T 0004-2012 SM3
-
-	  Architecture: x86_64 using:
-	  - AVX (Advanced Vector Extensions)
-
-	  If unsure, say N.
-
 endmenu
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
@@ -50,9 +50,6 @@ aesni-intel-$(CONFIG_64BIT) += aes-ctr-avx-x86_64.o \
 			       aes-gcm-vaes-avx512.o \
 			       aes-xts-avx-x86_64.o
 
-obj-$(CONFIG_CRYPTO_SM3_AVX_X86_64) += sm3-avx-x86_64.o
-sm3-avx-x86_64-y := sm3-avx-asm_64.o sm3_avx_glue.o
-
 obj-$(CONFIG_CRYPTO_SM4_AESNI_AVX_X86_64) += sm4-aesni-avx-x86_64.o
 sm4-aesni-avx-x86_64-y := sm4-aesni-avx-asm_64.o sm4_aesni_avx_glue.o
 
diff --git a/arch/x86/crypto/sm3_avx_glue.c b/arch/x86/crypto/sm3_avx_glue.c
diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig
@@ -282,6 +282,7 @@ config CRYPTO_LIB_SM3_ARCH
 	default y if ARM64
 	default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
 		     RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
+	default y if X86_64
 
 source "lib/crypto/tests/Kconfig"
 
diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
@@ -375,6 +375,7 @@ CFLAGS_sm3.o += -I$(src)/$(SRCARCH)
 libsm3-$(CONFIG_ARM64) += arm64/sm3-ce-core.o \
 			  arm64/sm3-neon-core.o
 libsm3-$(CONFIG_RISCV) += riscv/sm3-riscv64-zvksh-zvkb.o
+libsm3-$(CONFIG_X86) += x86/sm3-avx-asm_64.o
 endif # CONFIG_CRYPTO_LIB_SM3_ARCH
 
 ################################################################################
diff --git a/lib/crypto/x86/sm3-avx-asm_64.S b/lib/crypto/x86/sm3-avx-asm_64.S
@@ -12,10 +12,9 @@
  */
 
 #include <linux/linkage.h>
-#include <linux/cfi_types.h>
 #include <asm/frame.h>
 
-/* Context structure */
+/* State structure */
 
 #define state_h0 0
 #define state_h1 4
@@ -325,13 +324,13 @@
 /*
  * Transform nblocks*64 bytes (nblocks*16 32-bit words) at DATA.
  *
- * void sm3_transform_avx(struct sm3_state *state,
- *                        const u8 *data, int nblocks);
+ * void sm3_transform_avx(struct sm3_block_state *state,
+ *                        const u8 *data, size_t nblocks);
  */
-SYM_TYPED_FUNC_START(sm3_transform_avx)
+SYM_FUNC_START(sm3_transform_avx)
 	/* input:
-	 *	%rdi: ctx, CTX
-	 *	%rsi: data (64*nblks bytes)
+	 *	%rdi: state
+	 *	%rsi: data
 	 *	%rdx: nblocks
 	 */
 	vzeroupper;
diff --git a/lib/crypto/x86/sm3.h b/lib/crypto/x86/sm3.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * SM3 optimized for x86_64
+ *
+ * Copyright 2026 Google LLC
+ */
+#include <asm/fpu/api.h>
+#include <linux/static_call.h>
+
+asmlinkage void sm3_transform_avx(struct sm3_block_state *state,
+				  const u8 *data, size_t nblocks);
+
+static void sm3_blocks_avx(struct sm3_block_state *state,
+			   const u8 *data, size_t nblocks)
+{
+	if (likely(irq_fpu_usable())) {
+		kernel_fpu_begin();
+		sm3_transform_avx(state, data, nblocks);
+		kernel_fpu_end();
+	} else {
+		sm3_blocks_generic(state, data, nblocks);
+	}
+}
+
+DEFINE_STATIC_CALL(sm3_blocks_x86, sm3_blocks_generic);
+
+static void sm3_blocks(struct sm3_block_state *state,
+		       const u8 *data, size_t nblocks)
+{
+	static_call(sm3_blocks_x86)(state, data, nblocks);
+}
+
+#define sm3_mod_init_arch sm3_mod_init_arch
+static void sm3_mod_init_arch(void)
+{
+	if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_BMI2) &&
+	    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
+		static_call_update(sm3_blocks_x86, sm3_blocks_avx);
+}
