diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index 9825a16b7b9..b08377230e4 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -44,7 +44,7 @@ jobs: - name: Install Dependencies run: | sudo apt-get update - sudo apt-get install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi gcc-aarch64-linux-gnu libc6-dev-ppc64-powerpc-cross libcurl4-gnutls-dev lib64gcc-13-dev-powerpc-cross + sudo apt-get install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi gcc-aarch64-linux-gnu libc6-dev-ppc64-powerpc-cross libcurl4-gnutls-dev gcc-multilib-powerpc-linux-gnu - name: gnu90 build run: make gnu90build && make clean - name: gnu99 build diff --git a/.github/workflows/dev-long-tests.yml b/.github/workflows/dev-long-tests.yml index fab0b96203f..35a229cc0ae 100644 --- a/.github/workflows/dev-long-tests.yml +++ b/.github/workflows/dev-long-tests.yml @@ -76,14 +76,14 @@ jobs: # lasts ~8mn tsan-zstreamtest: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1 - name: thread sanitizer zstreamtest run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream uasan-zstreamtest: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1 - name: ub + address sanitizer on zstreamtest @@ -91,7 +91,7 @@ jobs: # lasts ~11mn tsan-fuzztest: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1 - name: thread sanitizer fuzztest @@ -109,7 +109,7 @@ jobs: # lasts ~13mn gcc-8-asan-ubsan-testzstd: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1 - name: gcc-8 + ASan + UBSan + Test Zstd @@ -121,14 +121,14 @@ jobs: CC=gcc-8 make -j uasan-test-zstd = 8 */ @@ -5194,11 +5194,11 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, const void* dict, size_t dictSize, ZSTD_dictTableLoadMethod_e dtlm, ZSTD_tableFillPurpose_e tfp, - void* workspace) + void* workspace, U32* dictIDPtr) { const BYTE* dictPtr = (const BYTE*)dict; const BYTE* const dictEnd = dictPtr + dictSize; - size_t dictID; + U32 dictID; size_t eSize; ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<= 8); @@ -5214,11 +5214,12 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, FORWARD_IF_ERROR(ZSTD_loadDictionaryContent( ms, NULL, ws, params, dictPtr, dictContentSize, dtlm, tfp), ""); } - return dictID; + *dictIDPtr = dictID; + return 0; } /** ZSTD_compress_insertDictionary() : -* @return : dictID, or an error code */ +* @return : 0, or an error code. The dictionary ID is written to dictIDPtr. */ static size_t ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, ZSTD_MatchState_t* ms, @@ -5229,9 +5230,10 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, ZSTD_tableFillPurpose_e tfp, - void* workspace) + void* workspace, U32* dictIDPtr) { DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize); + *dictIDPtr = 0; if ((dict==NULL) || (dictSize<8)) { RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); return 0; @@ -5255,7 +5257,7 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, /* dict as full zstd dictionary */ return ZSTD_loadZstdDictionary( - bs, ms, ws, params, dict, dictSize, dtlm, tfp, workspace); + bs, ms, ws, params, dict, dictSize, dtlm, tfp, workspace, dictIDPtr); } #define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB) @@ -5293,19 +5295,19 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, dictContentSize, ZSTDcrp_makeClean, zbuff) , ""); - { size_t const dictID = cdict ? + { U32 dictID = 0; + size_t const dictStatus = cdict ? ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent, cdict->dictContentSize, cdict->dictContentType, dtlm, - ZSTD_tfp_forCCtx, cctx->tmpWorkspace) + ZSTD_tfp_forCCtx, cctx->tmpWorkspace, &dictID) : ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize, - dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->tmpWorkspace); - FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); - assert(dictID <= UINT_MAX); - cctx->dictID = (U32)dictID; + dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->tmpWorkspace, &dictID); + FORWARD_IF_ERROR(dictStatus, "ZSTD_compress_insertDictionary failed"); + cctx->dictID = dictID; cctx->dictContentSize = dictContentSize; } return 0; @@ -5617,13 +5619,13 @@ static size_t ZSTD_initCDict_internal( */ { params.compressionLevel = ZSTD_CLEVEL_DEFAULT; params.fParams.contentSizeFlag = 1; - { size_t const dictID = ZSTD_compress_insertDictionary( + { U32 dictID = 0; + size_t const dictStatus = ZSTD_compress_insertDictionary( &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace, ¶ms, cdict->dictContent, cdict->dictContentSize, - dictContentType, ZSTD_dtlm_full, ZSTD_tfp_forCDict, cdict->entropyWorkspace); - FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); - assert(dictID <= (size_t)(U32)-1); - cdict->dictID = (U32)dictID; + dictContentType, ZSTD_dtlm_full, ZSTD_tfp_forCDict, cdict->entropyWorkspace, &dictID); + FORWARD_IF_ERROR(dictStatus, "ZSTD_compress_insertDictionary failed"); + cdict->dictID = dictID; } } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 32360b9a58d..97cb20c292c 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -3043,6 +3043,49 @@ static int basicUnitTests(U32 const seed, double compressibility) goto _output_error; } + DISPLAYLEVEL(3, "test%3i : full range of dictionary IDs : ", testNb++); + { U32 const dictIDs[] = { 1U, 0x7FFFFFFFU, 0x80000000U, 0xFFFFFFFFU }; + BYTE dict[sizeof(dictBufferFixed)]; + size_t const srcSize = 1 KB; + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + size_t i; + CHECK(dctx != NULL); + memcpy(dict, dictBufferFixed, sizeof(dict)); + for (i = 0; i < sizeof(dictIDs) / sizeof(dictIDs[0]); ++i) { + unsigned mode; + MEM_writeLE32(dict + 4, dictIDs[i]); + for (mode = 0; mode < 3; ++mode) { + if (mode == 0) { + cSize = ZSTD_compress_usingDict(cctx, + compressedBuffer, compressedBufferSize, CNBuffer, srcSize, + dict, sizeof(dict), 1); + } else if (mode == 1) { + ZSTD_CDict* const cdict = ZSTD_createCDict(dict, sizeof(dict), 1); + CHECK(cdict != NULL); + CHECK_EQ(ZSTD_getDictID_fromCDict(cdict), dictIDs[i]); + cSize = ZSTD_compress_usingCDict(cctx, + compressedBuffer, compressedBufferSize, CNBuffer, srcSize, cdict); + ZSTD_freeCDict(cdict); + } else { + CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); + CHECK_Z(ZSTD_CCtx_loadDictionary(cctx, dict, sizeof(dict))); + cSize = ZSTD_compress2(cctx, + compressedBuffer, compressedBufferSize, CNBuffer, srcSize); + } + CHECK_Z(cSize); + CHECK_EQ(ZSTD_getDictID_fromFrame(compressedBuffer, cSize), dictIDs[i]); + { size_t const r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, + compressedBuffer, cSize, dict, sizeof(dict)); + CHECK_Z(r); + CHECK_EQ(r, srcSize); + } + CHECK(memcmp(decodedBuffer, CNBuffer, srcSize) == 0); + } + } + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : dictBuilder on cyclic data : ", testNb++); assert(compressedBufferSize >= totalSampleSize); { U32 u; for (u=0; u