Skip to content

Commit 879a4f7

Browse files
gcabidduherbertx
gcabiddu
authored and
herbertx
committed
crypto: qat - add support for zstd
Add support for the ZSTD algorithm for QAT GEN4, GEN5 and GEN6 via the acomp API. For GEN4 and GEN5, compression is performed in hardware using LZ4s, a QAT-specific variant of LZ4. The compressed output is post-processed to generate ZSTD sequences, and the ZSTD library is then used to produce the final ZSTD stream via zstd_compress_sequences_and_literals(). Only inputs between 8 KB and 512 KB are offloaded to the device. The minimum size restriction will be relaxed once polling support is added. The maximum size is limited by the use of pre-allocated per-CPU scratch buffers. On these generations, only compression is offloaded to hardware; decompression always falls back to software. For GEN6, both compression and decompression are offloaded to the accelerator, which natively supports the ZSTD algorithm. There is no limit on the input buffer size supported. However, since GEN6 is limited to a history size of 64 KB, decompression of frames compressed with a larger history falls back to software. Since GEN2 devices do not support ZSTD or LZ4s, add a mechanism that prevents selecting GEN2 compression instances for ZSTD or LZ4s when a GEN2 plug-in card is present on a system with an embedded GEN4, GEN5 or GEN6 device. In addition, modify the algorithm registration logic to allow registering the correct implementation, i.e. LZ4s based for GEN4 and GEN5 or native ZSTD for GEN6. Co-developed-by: Suman Kumar Chakraborty <suman.kumar.chakraborty@intel.com> Signed-off-by: Suman Kumar Chakraborty <suman.kumar.chakraborty@intel.com> Signed-off-by: Giovanni Cabiddu <giovanni.cabiddu@intel.com> Reviewed-by: Laurent M Coquerel <laurent.m.coquerel@intel.com> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
1 parent 35ecb77 commit 879a4f7

17 files changed

Lines changed: 773 additions & 30 deletions

‎drivers/crypto/intel/qat/Kconfig‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ config CRYPTO_DEV_QAT
1212
select CRYPTO_LIB_SHA1
1313
select CRYPTO_LIB_SHA256
1414
select CRYPTO_LIB_SHA512
15+
select CRYPTO_ZSTD
1516
select FW_LOADER
1617
select CRC8
1718

‎drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -488,6 +488,7 @@ void adf_init_hw_data_420xx(struct adf_hw_device_data *hw_data, u32 dev_id)
488488
hw_data->clock_frequency = ADF_420XX_AE_FREQ;
489489
hw_data->services_supported = adf_gen4_services_supported;
490490
hw_data->get_svc_slice_cnt = adf_gen4_get_svc_slice_cnt;
491+
hw_data->accel_capabilities_ext_mask = ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S;
491492

492493
adf_gen4_set_err_mask(&hw_data->dev_err_mask);
493494
adf_gen4_init_hw_csr_ops(&hw_data->csr_ops);

‎drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -473,6 +473,7 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id)
473473
hw_data->clock_frequency = ADF_4XXX_AE_FREQ;
474474
hw_data->services_supported = adf_gen4_services_supported;
475475
hw_data->get_svc_slice_cnt = adf_gen4_get_svc_slice_cnt;
476+
hw_data->accel_capabilities_ext_mask = ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S;
476477

477478
adf_gen4_set_err_mask(&hw_data->dev_err_mask);
478479
adf_gen4_init_hw_csr_ops(&hw_data->csr_ops);

‎drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@
3333
#define ADF_AE_GROUP_1 GENMASK(7, 4)
3434
#define ADF_AE_GROUP_2 BIT(8)
3535

36+
#define ASB_MULTIPLIER 9
37+
3638
struct adf_ring_config {
3739
u32 ring_mask;
3840
enum adf_cfg_service_type ring_type;
@@ -509,6 +511,9 @@ static int build_comp_block(void *ctx, enum adf_dc_algo algo)
509511
case QAT_DEFLATE:
510512
header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DYNAMIC;
511513
break;
514+
case QAT_ZSTD:
515+
header->service_cmd_id = ICP_QAT_FW_COMP_CMD_ZSTD_COMPRESS;
516+
break;
512517
default:
513518
return -EINVAL;
514519
}
@@ -519,6 +524,13 @@ static int build_comp_block(void *ctx, enum adf_dc_algo algo)
519524
cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val;
520525
cd_pars->u.sl.comp_slice_cfg_word[1] = 0;
521526

527+
/*
528+
* Store Auto Select Best (ASB) multiplier in the request template.
529+
* This will be used in the data path to set the actual threshold
530+
* value based on the input data size.
531+
*/
532+
req_tmpl->u3.asb_threshold.asb_value = ASB_MULTIPLIER;
533+
522534
return 0;
523535
}
524536

@@ -532,12 +544,16 @@ static int build_decomp_block(void *ctx, enum adf_dc_algo algo)
532544
case QAT_DEFLATE:
533545
header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS;
534546
break;
547+
case QAT_ZSTD:
548+
header->service_cmd_id = ICP_QAT_FW_COMP_CMD_ZSTD_DECOMPRESS;
549+
break;
535550
default:
536551
return -EINVAL;
537552
}
538553

539554
cd_pars->u.sl.comp_slice_cfg_word[0] = 0;
540555
cd_pars->u.sl.comp_slice_cfg_word[1] = 0;
556+
req_tmpl->u3.asb_threshold.asb_value = 0;
541557

542558
return 0;
543559
}
@@ -1030,6 +1046,7 @@ void adf_init_hw_data_6xxx(struct adf_hw_device_data *hw_data)
10301046
hw_data->num_rps = ADF_GEN6_ETR_MAX_BANKS;
10311047
hw_data->clock_frequency = ADF_6XXX_AE_FREQ;
10321048
hw_data->get_svc_slice_cnt = adf_gen6_get_svc_slice_cnt;
1049+
hw_data->accel_capabilities_ext_mask = ADF_ACCEL_CAPABILITIES_EXT_ZSTD;
10331050

10341051
adf_gen6_init_services_supported(hw_data);
10351052
adf_gen6_init_hw_csr_ops(&hw_data->csr_ops);

‎drivers/crypto/intel/qat/qat_common/Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ intel_qat-y := adf_accel_engine.o \
4141
qat_bl.o \
4242
qat_comp_algs.o \
4343
qat_compression.o \
44+
qat_comp_zstd_utils.o \
4445
qat_crypto.o \
4546
qat_hal.o \
4647
qat_mig_dev.o \

‎drivers/crypto/intel/qat/qat_common/adf_accel_devices.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,11 @@ enum adf_accel_capabilities {
5959
ADF_ACCEL_CAPABILITIES_RANDOM_NUMBER = 128
6060
};
6161

62+
enum adf_accel_capabilities_ext {
63+
ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S = BIT(0),
64+
ADF_ACCEL_CAPABILITIES_EXT_ZSTD = BIT(1),
65+
};
66+
6267
enum adf_fuses {
6368
ADF_FUSECTL0,
6469
ADF_FUSECTL1,
@@ -336,6 +341,7 @@ struct adf_hw_device_data {
336341
u32 fuses[ADF_MAX_FUSES];
337342
u32 straps;
338343
u32 accel_capabilities_mask;
344+
u32 accel_capabilities_ext_mask;
339345
u32 extended_dc_capabilities;
340346
u16 fw_capabilities;
341347
u32 clock_frequency;

‎drivers/crypto/intel/qat/qat_common/adf_common_drv.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -111,12 +111,12 @@ void qat_algs_unregister(void);
111111
int qat_asym_algs_register(void);
112112
void qat_asym_algs_unregister(void);
113113

114-
struct qat_compression_instance *qat_compression_get_instance_node(int node);
114+
struct qat_compression_instance *qat_compression_get_instance_node(int node, int alg);
115115
void qat_compression_put_instance(struct qat_compression_instance *inst);
116116
int qat_compression_register(void);
117117
int qat_compression_unregister(void);
118-
int qat_comp_algs_register(void);
119-
void qat_comp_algs_unregister(void);
118+
int qat_comp_algs_register(u32 caps);
119+
void qat_comp_algs_unregister(u32 caps);
120120
void qat_comp_alg_callback(void *resp);
121121

122122
int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev);

‎drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -504,14 +504,20 @@ static int adf_gen4_build_comp_block(void *ctx, enum adf_dc_algo algo)
504504
switch (algo) {
505505
case QAT_DEFLATE:
506506
header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DYNAMIC;
507+
hw_comp_lower_csr.algo = ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_ILZ77;
508+
hw_comp_lower_csr.lllbd = ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_ENABLED;
509+
hw_comp_lower_csr.skip_ctrl = ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_LITERAL;
510+
break;
511+
case QAT_LZ4S:
512+
header->service_cmd_id = ICP_QAT_FW_COMP_20_CMD_LZ4S_COMPRESS;
513+
hw_comp_lower_csr.algo = ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_LZ4S;
514+
hw_comp_lower_csr.lllbd = ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_DISABLED;
515+
hw_comp_lower_csr.abd = ICP_QAT_HW_COMP_20_ABD_ABD_DISABLED;
507516
break;
508517
default:
509518
return -EINVAL;
510519
}
511520

512-
hw_comp_lower_csr.skip_ctrl = ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_LITERAL;
513-
hw_comp_lower_csr.algo = ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_ILZ77;
514-
hw_comp_lower_csr.lllbd = ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_ENABLED;
515521
hw_comp_lower_csr.sd = ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_1;
516522
hw_comp_lower_csr.hash_update = ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_DONT_ALLOW;
517523
hw_comp_lower_csr.edmm = ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_ENABLED;
@@ -538,12 +544,16 @@ static int adf_gen4_build_decomp_block(void *ctx, enum adf_dc_algo algo)
538544
switch (algo) {
539545
case QAT_DEFLATE:
540546
header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS;
547+
hw_decomp_lower_csr.algo = ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE;
548+
break;
549+
case QAT_LZ4S:
550+
header->service_cmd_id = ICP_QAT_FW_COMP_20_CMD_LZ4S_DECOMPRESS;
551+
hw_decomp_lower_csr.algo = ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_LZ4S;
541552
break;
542553
default:
543554
return -EINVAL;
544555
}
545556

546-
hw_decomp_lower_csr.algo = ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE;
547557
lower_val = ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_LOWER(hw_decomp_lower_csr);
548558

549559
cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val;

‎drivers/crypto/intel/qat/qat_common/adf_init.c‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,7 @@ static int adf_dev_start(struct adf_accel_dev *accel_dev)
180180
{
181181
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
182182
struct service_hndl *service;
183+
u32 caps;
183184
int ret;
184185

185186
set_bit(ADF_STATUS_STARTING, &accel_dev->status);
@@ -253,7 +254,8 @@ static int adf_dev_start(struct adf_accel_dev *accel_dev)
253254
}
254255
set_bit(ADF_STATUS_CRYPTO_ALGS_REGISTERED, &accel_dev->status);
255256

256-
if (!list_empty(&accel_dev->compression_list) && qat_comp_algs_register()) {
257+
caps = hw_data->accel_capabilities_ext_mask;
258+
if (!list_empty(&accel_dev->compression_list) && qat_comp_algs_register(caps)) {
257259
dev_err(&GET_DEV(accel_dev),
258260
"Failed to register compression algs\n");
259261
set_bit(ADF_STATUS_STARTING, &accel_dev->status);
@@ -308,7 +310,7 @@ static void adf_dev_stop(struct adf_accel_dev *accel_dev)
308310

309311
if (!list_empty(&accel_dev->compression_list) &&
310312
test_bit(ADF_STATUS_COMP_ALGS_REGISTERED, &accel_dev->status))
311-
qat_comp_algs_unregister();
313+
qat_comp_algs_unregister(hw_data->accel_capabilities_ext_mask);
312314
clear_bit(ADF_STATUS_COMP_ALGS_REGISTERED, &accel_dev->status);
313315

314316
list_for_each_entry(service, &service_table, list) {

‎drivers/crypto/intel/qat/qat_common/icp_qat_fw.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,13 @@ struct icp_qat_fw_comn_resp {
151151
ICP_QAT_FW_COMN_CNV_FLAG_BITPOS, \
152152
ICP_QAT_FW_COMN_CNV_FLAG_MASK)
153153

154+
#define ICP_QAT_FW_COMN_ST_BLK_FLAG_BITPOS 4
155+
#define ICP_QAT_FW_COMN_ST_BLK_FLAG_MASK 0x1
156+
#define ICP_QAT_FW_COMN_HDR_ST_BLK_FLAG_GET(hdr_flags) \
157+
QAT_FIELD_GET(hdr_flags, \
158+
ICP_QAT_FW_COMN_ST_BLK_FLAG_BITPOS, \
159+
ICP_QAT_FW_COMN_ST_BLK_FLAG_MASK)
160+
154161
#define ICP_QAT_FW_COMN_HDR_CNV_FLAG_SET(hdr_t, val) \
155162
QAT_FIELD_SET((hdr_t.hdr_flags), (val), \
156163
ICP_QAT_FW_COMN_CNV_FLAG_BITPOS, \

0 commit comments

Comments
 (0)