compress: separate dictionary IDs from error returns

.github/workflows/commit.yml

@@ -44,7 +44,7 @@ jobs:
     - name: Install Dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi gcc-aarch64-linux-gnu libc6-dev-ppc64-powerpc-cross libcurl4-gnutls-dev lib64gcc-13-dev-powerpc-cross
+        sudo apt-get install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi gcc-aarch64-linux-gnu libc6-dev-ppc64-powerpc-cross libcurl4-gnutls-dev gcc-multilib-powerpc-linux-gnu
     - name: gnu90 build
       run: make gnu90build && make clean
     - name: gnu99 build

.github/workflows/dev-long-tests.yml

@@ -76,22 +76,22 @@ jobs:
 
   # lasts ~8mn
   tsan-zstreamtest:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
     - name: thread sanitizer zstreamtest
       run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream
 
   uasan-zstreamtest:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
     - name: ub + address sanitizer on zstreamtest
       run: CC=clang make uasan-test-zstream
 
   # lasts ~11mn
   tsan-fuzztest:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
     - name: thread sanitizer fuzztest
@@ -109,7 +109,7 @@ jobs:
 
   # lasts ~13mn
   gcc-8-asan-ubsan-testzstd:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
     - name: gcc-8 + ASan + UBSan + Test Zstd
@@ -121,14 +121,14 @@ jobs:
         CC=gcc-8 make -j uasan-test-zstd </dev/null V=1
 
   clang-asan-ubsan-testzstd:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
     - name: clang + ASan + UBSan + Test Zstd
       run: CC=clang make -j uasan-test-zstd </dev/null V=1
 
   gcc-asan-ubsan-testzstd-32bit:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
     - name: ASan + UBSan + Test Zstd, 32bit mode
@@ -142,7 +142,7 @@ jobs:
     # so any data coming from these libraries is always considered "uninitialized"
 
   gcc-8-asan-ubsan-fuzz:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
     - name: gcc-8 + ASan + UBSan + Fuzz Test
@@ -154,7 +154,7 @@ jobs:
         CC=gcc-8 FUZZER_FLAGS="--long-tests" make -j clean uasan-fuzztest
 
   clang-asan-ubsan-fuzz:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
     - name: clang + ASan + UBSan + Fuzz Test
@@ -164,14 +164,14 @@ jobs:
   # fuzzers do build these decoders, but only reach them if an input
   # happens to carry a legacy magic number.
   clang-asan-ubsan-legacy:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
     - name: clang + ASan + UBSan + legacy decoders
       run: CC=clang make uasan-test-legacy
 
   gcc-asan-ubsan-fuzz32:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
     - name: ASan + UBSan + Fuzz Test 32bit
@@ -181,7 +181,7 @@ jobs:
         CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make -j uasan-fuzztest
 
   clang-asan-fuzz32:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
     - name: clang + ASan + Fuzz Test 32bit
@@ -227,7 +227,7 @@ jobs:
       run: make -j msanregressiontest
 
   clang-msan-fuzz-unoptimized:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
       - name: clang + MSan + Fuzz Test
@@ -239,7 +239,7 @@ jobs:
           CC=clang MOREFLAGS="-O0" FUZZERTEST=-T60s ZSTREAM_TESTTIME=-T30s make clean msan-fuzztest
 
   clang-msan-fuzz:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
     - name: clang + MSan + Fuzz Test
@@ -249,7 +249,7 @@ jobs:
         CC=clang FUZZER_FLAGS="--long-tests" make clean msan-fuzztest
 
   clang-msan-testzstd:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
     - name: clang + MSan + Test Zstd
@@ -259,7 +259,7 @@ jobs:
         CC=clang make -j msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 ZSTDRTTEST= V=1
 
   armfuzz:
-      runs-on: ubuntu-latest
+      runs-on: ubuntu-24.04
       steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
       - name: Qemu ARM emulation + Fuzz Test

.github/workflows/dev-short-tests.yml

@@ -98,7 +98,7 @@ jobs:
         CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CFLAGS="-Werror -O1" make zstd
 
   armbuild:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
     - name: ARM Build Test
@@ -343,7 +343,7 @@ jobs:
   # If the size grows intentionally, you can raise these numbers.
   # But we do need to think about binary size, since it is a concern.
   libzstd-size:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
     - name: libzstd size test
@@ -505,11 +505,16 @@ jobs:
       run: pacman --noconfirm -S mingw-w64-i686-clang
     - name: install mingw clang x86_64
       if: ${{ (matrix.msystem == 'MINGW64') && (matrix.compiler == 'clang') }}
-      run: pacman --noconfirm -S mingw-w64-x86_64-clang
+      run: |
+        pacman --noconfirm -U \
+          https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-compiler-rt-21-21.1.8-5-any.pkg.tar.zst \
+          https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-llvm-21-21.1.8-5-any.pkg.tar.zst \
+          https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-clang-21-21.1.8-5-any.pkg.tar.zst
     - name: run mingw tests
       run: |
         make -v
         export CC=${{ matrix.compiler }}
+        if [ "$CC" = clang ]; then export CC=/mingw64/opt/llvm-21/bin/clang; fi
         $CC --version
         CFLAGS="${{ matrix.cflags }}" make -j allzstd
         echo "Testing $CC ${{ matrix.msystem }}"
@@ -589,7 +594,7 @@ jobs:
           ./simple LICENSE
 
   versions-compatibility:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1
     - name: Versions Compatibility Test

.github/workflows/dictionary-static-analysis.yml

@@ -0,0 +1,33 @@
+# Fork-only validation of the unchanged base and source patch.
+name: dictionary-static-analysis
+
+on:
+  workflow_dispatch:
+
+permissions: read-all
+
+jobs:
+  clang-18:
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        revision:
+          - 49cf51799ea051457f57243527aa2058f999f005
+          - ac6e2475a827ad0e373bd3844f0e2df01c781b5d
+    steps:
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+      with:
+        ref: ${{ matrix.revision }}
+    - name: Install Clang 18 analyzer
+      run: |
+        sudo apt-get -qqq update
+        sudo apt-get install -y clang-18 clang-tools-18
+    - name: Analyze the upstream base and dictionary fix
+      run: make -j2 staticAnalyze CC=clang-18 SCANBUILD="scan-build-18 -o analysis"
+    - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
+      if: always()
+      with:
+        name: static-analysis-${{ matrix.revision }}
+        path: analysis
+        if-no-files-found: ignore

lib/compress/zstd_compress.c

@@ -5183,7 +5183,7 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
  * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#dictionary-format
  */
 /*! ZSTD_loadZstdDictionary() :
- * @return : dictID, or an error code
+ * @return : 0, or an error code. The dictionary ID is written to dictIDPtr.
  *  assumptions : magic number supposed already checked
  *                dictSize supposed >= 8
  */
@@ -5194,11 +5194,11 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
                                       const void* dict, size_t dictSize,
                                       ZSTD_dictTableLoadMethod_e dtlm,
                                       ZSTD_tableFillPurpose_e tfp,
-                                      void* workspace)
+                                      void* workspace, U32* dictIDPtr)
 {
     const BYTE* dictPtr = (const BYTE*)dict;
     const BYTE* const dictEnd = dictPtr + dictSize;
-    size_t dictID;
+    U32 dictID;
     size_t eSize;
     ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog)));
     assert(dictSize >= 8);
@@ -5214,11 +5214,12 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
         FORWARD_IF_ERROR(ZSTD_loadDictionaryContent(
             ms, NULL, ws, params, dictPtr, dictContentSize, dtlm, tfp), "");
     }
-    return dictID;
+    *dictIDPtr = dictID;
+    return 0;
 }
 
 /** ZSTD_compress_insertDictionary() :
-*   @return : dictID, or an error code */
+*   @return : 0, or an error code. The dictionary ID is written to dictIDPtr. */
 static size_t
 ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
                                ZSTD_MatchState_t* ms,
@@ -5229,9 +5230,10 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
                                ZSTD_dictContentType_e dictContentType,
                                ZSTD_dictTableLoadMethod_e dtlm,
                                ZSTD_tableFillPurpose_e tfp,
-                               void* workspace)
+                               void* workspace, U32* dictIDPtr)
 {
     DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize);
+    *dictIDPtr = 0;
     if ((dict==NULL) || (dictSize<8)) {
         RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, "");
         return 0;
@@ -5255,7 +5257,7 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
 
     /* dict as full zstd dictionary */
     return ZSTD_loadZstdDictionary(
-        bs, ms, ws, params, dict, dictSize, dtlm, tfp, workspace);
+        bs, ms, ws, params, dict, dictSize, dtlm, tfp, workspace, dictIDPtr);
 }
 
 #define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB)
@@ -5293,19 +5295,19 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
     FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
                                      dictContentSize,
                                      ZSTDcrp_makeClean, zbuff) , "");
-    {   size_t const dictID = cdict ?
+    {   U32 dictID = 0;
+        size_t const dictStatus = cdict ?
                 ZSTD_compress_insertDictionary(
                         cctx->blockState.prevCBlock, &cctx->blockState.matchState,
                         &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent,
                         cdict->dictContentSize, cdict->dictContentType, dtlm,
-                        ZSTD_tfp_forCCtx, cctx->tmpWorkspace)
+                        ZSTD_tfp_forCCtx, cctx->tmpWorkspace, &dictID)
               : ZSTD_compress_insertDictionary(
                         cctx->blockState.prevCBlock, &cctx->blockState.matchState,
                         &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize,
-                        dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->tmpWorkspace);
-        FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed");
-        assert(dictID <= UINT_MAX);
-        cctx->dictID = (U32)dictID;
+                        dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->tmpWorkspace, &dictID);
+        FORWARD_IF_ERROR(dictStatus, "ZSTD_compress_insertDictionary failed");
+        cctx->dictID = dictID;
         cctx->dictContentSize = dictContentSize;
     }
     return 0;
@@ -5617,13 +5619,13 @@ static size_t ZSTD_initCDict_internal(
      */
     {   params.compressionLevel = ZSTD_CLEVEL_DEFAULT;
         params.fParams.contentSizeFlag = 1;
-        {   size_t const dictID = ZSTD_compress_insertDictionary(
+        {   U32 dictID = 0;
+            size_t const dictStatus = ZSTD_compress_insertDictionary(
                     &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace,
                     &params, cdict->dictContent, cdict->dictContentSize,
-                    dictContentType, ZSTD_dtlm_full, ZSTD_tfp_forCDict, cdict->entropyWorkspace);
-            FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed");
-            assert(dictID <= (size_t)(U32)-1);
-            cdict->dictID = (U32)dictID;
+                    dictContentType, ZSTD_dtlm_full, ZSTD_tfp_forCDict, cdict->entropyWorkspace, &dictID);
+            FORWARD_IF_ERROR(dictStatus, "ZSTD_compress_insertDictionary failed");
+            cdict->dictID = dictID;
         }
     }
 

tests/fuzzer.c

@@ -3043,6 +3043,49 @@ static int basicUnitTests(U32 const seed, double compressibility)
             goto _output_error;
         }
 
+        DISPLAYLEVEL(3, "test%3i : full range of dictionary IDs : ", testNb++);
+        {   U32 const dictIDs[] = { 1U, 0x7FFFFFFFU, 0x80000000U, 0xFFFFFFFFU };
+            BYTE dict[sizeof(dictBufferFixed)];
+            size_t const srcSize = 1 KB;
+            ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+            size_t i;
+            CHECK(dctx != NULL);
+            memcpy(dict, dictBufferFixed, sizeof(dict));
+            for (i = 0; i < sizeof(dictIDs) / sizeof(dictIDs[0]); ++i) {
+                unsigned mode;
+                MEM_writeLE32(dict + 4, dictIDs[i]);
+                for (mode = 0; mode < 3; ++mode) {
+                    if (mode == 0) {
+                        cSize = ZSTD_compress_usingDict(cctx,
+                            compressedBuffer, compressedBufferSize, CNBuffer, srcSize,
+                            dict, sizeof(dict), 1);
+                    } else if (mode == 1) {
+                        ZSTD_CDict* const cdict = ZSTD_createCDict(dict, sizeof(dict), 1);
+                        CHECK(cdict != NULL);
+                        CHECK_EQ(ZSTD_getDictID_fromCDict(cdict), dictIDs[i]);
+                        cSize = ZSTD_compress_usingCDict(cctx,
+                            compressedBuffer, compressedBufferSize, CNBuffer, srcSize, cdict);
+                        ZSTD_freeCDict(cdict);
+                    } else {
+                        CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
+                        CHECK_Z(ZSTD_CCtx_loadDictionary(cctx, dict, sizeof(dict)));
+                        cSize = ZSTD_compress2(cctx,
+                            compressedBuffer, compressedBufferSize, CNBuffer, srcSize);
+                    }
+                    CHECK_Z(cSize);
+                    CHECK_EQ(ZSTD_getDictID_fromFrame(compressedBuffer, cSize), dictIDs[i]);
+                    {   size_t const r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize,
+                            compressedBuffer, cSize, dict, sizeof(dict));
+                        CHECK_Z(r);
+                        CHECK_EQ(r, srcSize);
+                    }
+                    CHECK(memcmp(decodedBuffer, CNBuffer, srcSize) == 0);
+                }
+            }
+            ZSTD_freeDCtx(dctx);
+        }
+        DISPLAYLEVEL(3, "OK \n");
+
         DISPLAYLEVEL(3, "test%3i : dictBuilder on cyclic data : ", testNb++);
         assert(compressedBufferSize >= totalSampleSize);
         { U32 u; for (u=0; u<totalSampleSize; u++) ((BYTE*)decodedBuffer)[u] = (BYTE)u; }