Skip to content

Commit 9f69f52

Browse files
author
Eric Biggers
committed
lib/crypto: arm64/sm3: Migrate optimized code into library
Instead of exposing the arm64-optimized SM3 code via arm64-specific crypto_shash algorithms, instead just implement the sm3_blocks() library function. This is much simpler, it makes the SM3 library functions be arm64-optimized, and it fixes the longstanding issue where the arm64-optimized SM3 code was disabled by default. SM3 still remains available through crypto_shash, but individual architectures no longer need to handle it. Tweak the SM3 assembly function prototypes to match what the library expects, including changing the block count from 'int' to 'size_t'. sm3_ce_transform() had to be updated to access 'x2' instead of 'w2', while sm3_neon_transform() already used 'x2'. Remove the CFI stubs which are no longer needed because the SM3 assembly functions are no longer ever indirectly called. Remove the dependency on KERNEL_MODE_NEON. It was unnecessary, because KERNEL_MODE_NEON is always enabled on arm64. Acked-by: Ard Biesheuvel <ardb@kernel.org> Link: https://lore.kernel.org/r/20260321040935.410034-8-ebiggers@kernel.org Signed-off-by: Eric Biggers <ebiggers@kernel.org>
1 parent ed065bd commit 9f69f52

10 files changed

Lines changed: 62 additions & 180 deletions

File tree

‎arch/arm64/configs/defconfig‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1916,9 +1916,9 @@ CONFIG_CRYPTO_BENCHMARK=m
19161916
CONFIG_CRYPTO_ECHAINIV=y
19171917
CONFIG_CRYPTO_MICHAEL_MIC=m
19181918
CONFIG_CRYPTO_SHA3=m
1919+
CONFIG_CRYPTO_SM3=m
19191920
CONFIG_CRYPTO_USER_API_RNG=m
19201921
CONFIG_CRYPTO_GHASH_ARM64_CE=y
1921-
CONFIG_CRYPTO_SM3_ARM64_CE=m
19221922
CONFIG_CRYPTO_AES_ARM64_CE_BLK=y
19231923
CONFIG_CRYPTO_AES_ARM64_BS=m
19241924
CONFIG_CRYPTO_AES_ARM64_CE_CCM=y

‎arch/arm64/crypto/Kconfig‎

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -14,28 +14,6 @@ config CRYPTO_GHASH_ARM64_CE
1414
Architecture: arm64 using:
1515
- ARMv8 Crypto Extensions
1616

17-
config CRYPTO_SM3_NEON
18-
tristate "Hash functions: SM3 (NEON)"
19-
depends on KERNEL_MODE_NEON
20-
select CRYPTO_HASH
21-
select CRYPTO_LIB_SM3
22-
help
23-
SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012)
24-
25-
Architecture: arm64 using:
26-
- NEON (Advanced SIMD) extensions
27-
28-
config CRYPTO_SM3_ARM64_CE
29-
tristate "Hash functions: SM3 (ARMv8.2 Crypto Extensions)"
30-
depends on KERNEL_MODE_NEON
31-
select CRYPTO_HASH
32-
select CRYPTO_LIB_SM3
33-
help
34-
SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012)
35-
36-
Architecture: arm64 using:
37-
- ARMv8.2 Crypto Extensions
38-
3917
config CRYPTO_AES_ARM64_CE_BLK
4018
tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (ARMv8 Crypto Extensions)"
4119
depends on KERNEL_MODE_NEON

‎arch/arm64/crypto/Makefile‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,6 @@
55
# Copyright (C) 2014 Linaro Ltd <ard.biesheuvel@linaro.org>
66
#
77

8-
obj-$(CONFIG_CRYPTO_SM3_NEON) += sm3-neon.o
9-
sm3-neon-y := sm3-neon-glue.o sm3-neon-core.o
10-
11-
obj-$(CONFIG_CRYPTO_SM3_ARM64_CE) += sm3-ce.o
12-
sm3-ce-y := sm3-ce-glue.o sm3-ce-core.o
13-
148
obj-$(CONFIG_CRYPTO_SM4_ARM64_CE) += sm4-ce-cipher.o
159
sm4-ce-cipher-y := sm4-ce-cipher-glue.o sm4-ce-cipher-core.o
1610

‎arch/arm64/crypto/sm3-ce-glue.c‎

Lines changed: 0 additions & 70 deletions
This file was deleted.

‎arch/arm64/crypto/sm3-neon-glue.c‎

Lines changed: 0 additions & 67 deletions
This file was deleted.

‎lib/crypto/Kconfig‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,7 @@ config CRYPTO_LIB_SM3
279279
config CRYPTO_LIB_SM3_ARCH
280280
bool
281281
depends on CRYPTO_LIB_SM3 && !UML
282+
default y if ARM64
282283

283284
source "lib/crypto/tests/Kconfig"
284285

‎lib/crypto/Makefile‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -368,13 +368,20 @@ endif # CONFIG_CRYPTO_LIB_SHA3_ARCH
368368

369369
################################################################################
370370

371+
obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o
372+
libsm3-y := sm3.o
373+
ifeq ($(CONFIG_CRYPTO_LIB_SM3_ARCH),y)
374+
CFLAGS_sm3.o += -I$(src)/$(SRCARCH)
375+
libsm3-$(CONFIG_ARM64) += arm64/sm3-ce-core.o \
376+
arm64/sm3-neon-core.o
377+
endif # CONFIG_CRYPTO_LIB_SM3_ARCH
378+
379+
################################################################################
380+
371381
obj-$(CONFIG_MPILIB) += mpi/
372382

373383
obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) += simd.o
374384

375-
obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o
376-
libsm3-y := sm3.o
377-
378385
# clean-files must be defined unconditionally
379386
clean-files += arm/sha256-core.S arm/sha512-core.S
380387
clean-files += arm64/sha256-core.S arm64/sha512-core.S

arch/arm64/crypto/sm3-ce-core.S renamed to lib/crypto/arm64/sm3-ce-core.S

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
*/
77

88
#include <linux/linkage.h>
9-
#include <linux/cfi_types.h>
109
#include <asm/assembler.h>
1110

1211
.irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
@@ -70,11 +69,11 @@
7069
.endm
7170

7271
/*
73-
* void sm3_ce_transform(struct sm3_state *sst, u8 const *src,
74-
* int blocks)
72+
* void sm3_ce_transform(struct sm3_block_state *state,
73+
* const u8 *data, size_t nblocks)
7574
*/
7675
.text
77-
SYM_TYPED_FUNC_START(sm3_ce_transform)
76+
SYM_FUNC_START(sm3_ce_transform)
7877
/* load state */
7978
ld1 {v8.4s-v9.4s}, [x0]
8079
rev64 v8.4s, v8.4s
@@ -87,7 +86,7 @@ SYM_TYPED_FUNC_START(sm3_ce_transform)
8786

8887
/* load input */
8988
0: ld1 {v0.16b-v3.16b}, [x1], #64
90-
sub w2, w2, #1
89+
sub x2, x2, #1
9190

9291
mov v15.16b, v8.16b
9392
mov v16.16b, v9.16b
@@ -123,7 +122,7 @@ CPU_LE( rev32 v3.16b, v3.16b )
123122
eor v9.16b, v9.16b, v16.16b
124123

125124
/* handled all input blocks? */
126-
cbnz w2, 0b
125+
cbnz x2, 0b
127126

128127
/* save state */
129128
rev64 v8.4s, v8.4s

arch/arm64/crypto/sm3-neon-core.S renamed to lib/crypto/arm64/sm3-neon-core.S

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
*/
1010

1111
#include <linux/linkage.h>
12-
#include <linux/cfi_types.h>
1312
#include <asm/assembler.h>
1413

1514
/* Context structure */
@@ -345,14 +344,14 @@
345344

346345

347346
/*
348-
* Transform blocks*64 bytes (blocks*16 32-bit words) at 'src'.
347+
* Transform nblocks*64 bytes (nblocks*16 32-bit words) at 'data'.
349348
*
350-
* void sm3_neon_transform(struct sm3_state *sst, u8 const *src,
351-
* int blocks)
349+
* void sm3_neon_transform(struct sm3_block_state *state,
350+
* const u8 *data, size_t nblocks)
352351
*/
353352
.text
354353
.align 3
355-
SYM_TYPED_FUNC_START(sm3_neon_transform)
354+
SYM_FUNC_START(sm3_neon_transform)
356355
ldp ra, rb, [RSTATE, #0]
357356
ldp rc, rd, [RSTATE, #8]
358357
ldp re, rf, [RSTATE, #16]

‎lib/crypto/arm64/sm3.h‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/* SPDX-License-Identifier: GPL-2.0-or-later */
2+
/*
3+
* SM3 optimized for ARM64
4+
*
5+
* Copyright 2026 Google LLC
6+
*/
7+
#include <asm/simd.h>
8+
#include <linux/cpufeature.h>
9+
10+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
11+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce);
12+
13+
asmlinkage void sm3_neon_transform(struct sm3_block_state *state,
14+
const u8 *data, size_t nblocks);
15+
asmlinkage void sm3_ce_transform(struct sm3_block_state *state,
16+
const u8 *data, size_t nblocks);
17+
18+
static void sm3_blocks(struct sm3_block_state *state,
19+
const u8 *data, size_t nblocks)
20+
{
21+
if (static_branch_likely(&have_neon) && likely(may_use_simd())) {
22+
scoped_ksimd() {
23+
if (static_branch_likely(&have_ce))
24+
sm3_ce_transform(state, data, nblocks);
25+
else
26+
sm3_neon_transform(state, data, nblocks);
27+
}
28+
} else {
29+
sm3_blocks_generic(state, data, nblocks);
30+
}
31+
}
32+
33+
#define sm3_mod_init_arch sm3_mod_init_arch
34+
static void sm3_mod_init_arch(void)
35+
{
36+
if (cpu_have_named_feature(ASIMD)) {
37+
static_branch_enable(&have_neon);
38+
if (cpu_have_named_feature(SM3))
39+
static_branch_enable(&have_ce);
40+
}
41+
}

0 commit comments

Comments
 (0)