From ac6e2475a827ad0e373bd3844f0e2df01c781b5d Mon Sep 17 00:00:00 2001 From: Treston Cuzzort Date: Sun, 25 Oct 2026 19:38:03 -0500 Subject: [PATCH 1/4] compress: separate dictionary IDs from error returns ZSTD_loadZstdDictionary() and ZSTD_compress_insertDictionary() returned both dictionary IDs and errors through size_t. On 32-bit builds, valid private dictionary IDs near UINT32_MAX overlap the encoded error range and are rejected by their callers. Return a status and write the ID through an output parameter instead. Raw or absent dictionaries still produce ID zero, and failed loads still return their error before the caller stores the ID. Add round trips and frame-ID checks for low and high IDs through direct dictionaries, prepared CDicts, and CCtx-loaded dictionaries. --- lib/compress/zstd_compress.c | 38 ++++++++++++++++--------------- tests/fuzzer.c | 43 ++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 18 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 38c3fc0d4cd..28d927cca17 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -5183,7 +5183,7 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#dictionary-format */ /*! ZSTD_loadZstdDictionary() : - * @return : dictID, or an error code + * @return : 0, or an error code. The dictionary ID is written to dictIDPtr. * assumptions : magic number supposed already checked * dictSize supposed >= 8 */ @@ -5194,11 +5194,11 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, const void* dict, size_t dictSize, ZSTD_dictTableLoadMethod_e dtlm, ZSTD_tableFillPurpose_e tfp, - void* workspace) + void* workspace, U32* dictIDPtr) { const BYTE* dictPtr = (const BYTE*)dict; const BYTE* const dictEnd = dictPtr + dictSize; - size_t dictID; + U32 dictID; size_t eSize; ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<= 8); @@ -5214,11 +5214,12 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, FORWARD_IF_ERROR(ZSTD_loadDictionaryContent( ms, NULL, ws, params, dictPtr, dictContentSize, dtlm, tfp), ""); } - return dictID; + *dictIDPtr = dictID; + return 0; } /** ZSTD_compress_insertDictionary() : -* @return : dictID, or an error code */ +* @return : 0, or an error code. The dictionary ID is written to dictIDPtr. */ static size_t ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, ZSTD_MatchState_t* ms, @@ -5229,9 +5230,10 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, ZSTD_tableFillPurpose_e tfp, - void* workspace) + void* workspace, U32* dictIDPtr) { DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize); + *dictIDPtr = 0; if ((dict==NULL) || (dictSize<8)) { RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); return 0; @@ -5255,7 +5257,7 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, /* dict as full zstd dictionary */ return ZSTD_loadZstdDictionary( - bs, ms, ws, params, dict, dictSize, dtlm, tfp, workspace); + bs, ms, ws, params, dict, dictSize, dtlm, tfp, workspace, dictIDPtr); } #define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB) @@ -5293,19 +5295,19 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, dictContentSize, ZSTDcrp_makeClean, zbuff) , ""); - { size_t const dictID = cdict ? + { U32 dictID = 0; + size_t const dictStatus = cdict ? ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent, cdict->dictContentSize, cdict->dictContentType, dtlm, - ZSTD_tfp_forCCtx, cctx->tmpWorkspace) + ZSTD_tfp_forCCtx, cctx->tmpWorkspace, &dictID) : ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize, - dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->tmpWorkspace); - FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); - assert(dictID <= UINT_MAX); - cctx->dictID = (U32)dictID; + dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->tmpWorkspace, &dictID); + FORWARD_IF_ERROR(dictStatus, "ZSTD_compress_insertDictionary failed"); + cctx->dictID = dictID; cctx->dictContentSize = dictContentSize; } return 0; @@ -5617,13 +5619,13 @@ static size_t ZSTD_initCDict_internal( */ { params.compressionLevel = ZSTD_CLEVEL_DEFAULT; params.fParams.contentSizeFlag = 1; - { size_t const dictID = ZSTD_compress_insertDictionary( + { U32 dictID = 0; + size_t const dictStatus = ZSTD_compress_insertDictionary( &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace, ¶ms, cdict->dictContent, cdict->dictContentSize, - dictContentType, ZSTD_dtlm_full, ZSTD_tfp_forCDict, cdict->entropyWorkspace); - FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); - assert(dictID <= (size_t)(U32)-1); - cdict->dictID = (U32)dictID; + dictContentType, ZSTD_dtlm_full, ZSTD_tfp_forCDict, cdict->entropyWorkspace, &dictID); + FORWARD_IF_ERROR(dictStatus, "ZSTD_compress_insertDictionary failed"); + cdict->dictID = dictID; } } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 32360b9a58d..97cb20c292c 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -3043,6 +3043,49 @@ static int basicUnitTests(U32 const seed, double compressibility) goto _output_error; } + DISPLAYLEVEL(3, "test%3i : full range of dictionary IDs : ", testNb++); + { U32 const dictIDs[] = { 1U, 0x7FFFFFFFU, 0x80000000U, 0xFFFFFFFFU }; + BYTE dict[sizeof(dictBufferFixed)]; + size_t const srcSize = 1 KB; + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + size_t i; + CHECK(dctx != NULL); + memcpy(dict, dictBufferFixed, sizeof(dict)); + for (i = 0; i < sizeof(dictIDs) / sizeof(dictIDs[0]); ++i) { + unsigned mode; + MEM_writeLE32(dict + 4, dictIDs[i]); + for (mode = 0; mode < 3; ++mode) { + if (mode == 0) { + cSize = ZSTD_compress_usingDict(cctx, + compressedBuffer, compressedBufferSize, CNBuffer, srcSize, + dict, sizeof(dict), 1); + } else if (mode == 1) { + ZSTD_CDict* const cdict = ZSTD_createCDict(dict, sizeof(dict), 1); + CHECK(cdict != NULL); + CHECK_EQ(ZSTD_getDictID_fromCDict(cdict), dictIDs[i]); + cSize = ZSTD_compress_usingCDict(cctx, + compressedBuffer, compressedBufferSize, CNBuffer, srcSize, cdict); + ZSTD_freeCDict(cdict); + } else { + CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); + CHECK_Z(ZSTD_CCtx_loadDictionary(cctx, dict, sizeof(dict))); + cSize = ZSTD_compress2(cctx, + compressedBuffer, compressedBufferSize, CNBuffer, srcSize); + } + CHECK_Z(cSize); + CHECK_EQ(ZSTD_getDictID_fromFrame(compressedBuffer, cSize), dictIDs[i]); + { size_t const r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, + compressedBuffer, cSize, dict, sizeof(dict)); + CHECK_Z(r); + CHECK_EQ(r, srcSize); + } + CHECK(memcmp(decodedBuffer, CNBuffer, srcSize) == 0); + } + } + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : dictBuilder on cyclic data : ", testNb++); assert(compressedBufferSize >= totalSampleSize); { U32 u; for (u=0; u Date: Sun, 25 Oct 2026 19:46:11 -0500 Subject: [PATCH 2/4] ci: use compatible toolchains for fork validation Pin jobs requiring gold or qemu-user-static to Ubuntu 24.04 and the MinGW Clang job to 21.1.8. Preserve all test targets and flags. Compare Clang 18 static analysis of the upstream base and exact source patch. Fork validation only; not part of the upstream change. --- .github/workflows/dev-long-tests.yml | 30 ++++++++-------- .github/workflows/dev-short-tests.yml | 9 +++-- .../workflows/dictionary-static-analysis.yml | 34 +++++++++++++++++++ 3 files changed, 56 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/dictionary-static-analysis.yml diff --git a/.github/workflows/dev-long-tests.yml b/.github/workflows/dev-long-tests.yml index fab0b96203f..35a229cc0ae 100644 --- a/.github/workflows/dev-long-tests.yml +++ b/.github/workflows/dev-long-tests.yml @@ -76,14 +76,14 @@ jobs: # lasts ~8mn tsan-zstreamtest: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1 - name: thread sanitizer zstreamtest run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream uasan-zstreamtest: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1 - name: ub + address sanitizer on zstreamtest @@ -91,7 +91,7 @@ jobs: # lasts ~11mn tsan-fuzztest: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1 - name: thread sanitizer fuzztest @@ -109,7 +109,7 @@ jobs: # lasts ~13mn gcc-8-asan-ubsan-testzstd: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1 - name: gcc-8 + ASan + UBSan + Test Zstd @@ -121,14 +121,14 @@ jobs: CC=gcc-8 make -j uasan-test-zstd Date: Sun, 25 Oct 2026 19:58:07 -0500 Subject: [PATCH 3/4] ci: match the PowerPC support library to its compiler Use the distribution multilib compiler package instead of a hardcoded GCC 13 support library. The latest runner uses GCC 15 and cannot link the 64-bit target with the previous dependency. Fork validation only. --- .github/workflows/commit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index 9825a16b7b9..b08377230e4 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -44,7 +44,7 @@ jobs: - name: Install Dependencies run: | sudo apt-get update - sudo apt-get install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi gcc-aarch64-linux-gnu libc6-dev-ppc64-powerpc-cross libcurl4-gnutls-dev lib64gcc-13-dev-powerpc-cross + sudo apt-get install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi gcc-aarch64-linux-gnu libc6-dev-ppc64-powerpc-cross libcurl4-gnutls-dev gcc-multilib-powerpc-linux-gnu - name: gnu90 build run: make gnu90build && make clean - name: gnu99 build From 93d04fe7ba2d4201c0adc4d870a44bf47e9547fe Mon Sep 17 00:00:00 2001 From: Treston Cuzzort Date: Sun, 25 Oct 2026 20:18:19 -0500 Subject: [PATCH 4/4] ci: stabilize historical compatibility and size environments Pin the two legacy environment checks without changing their commands or size limits. Keep the already collected exact-base/exact-fix analyzer comparison manual so infrastructure-only changes do not repeat it. Both analyzer failures remain documented. --- .github/workflows/dev-short-tests.yml | 4 ++-- .github/workflows/dictionary-static-analysis.yml | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml index d0fdc5b0017..3f5246fc051 100644 --- a/.github/workflows/dev-short-tests.yml +++ b/.github/workflows/dev-short-tests.yml @@ -343,7 +343,7 @@ jobs: # If the size grows intentionally, you can raise these numbers. # But we do need to think about binary size, since it is a concern. libzstd-size: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1 - name: libzstd size test @@ -594,7 +594,7 @@ jobs: ./simple LICENSE versions-compatibility: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1 - name: Versions Compatibility Test diff --git a/.github/workflows/dictionary-static-analysis.yml b/.github/workflows/dictionary-static-analysis.yml index aa0e7330774..0ed04c28990 100644 --- a/.github/workflows/dictionary-static-analysis.yml +++ b/.github/workflows/dictionary-static-analysis.yml @@ -2,8 +2,7 @@ name: dictionary-static-analysis on: - pull_request: - branches: [ dev ] + workflow_dispatch: permissions: read-all