lz4: replace unmaintained lz4 with maintained lz4-napi (frame-compatible)

Root cause of the Node 22+ CI failures: `lz4@0.6.5` (2021, unmaintained) is
a node-gyp native addon with NO prebuilt binaries. It fails to compile on
Node 22+, gets silently skipped as an optional dep, then tests that
require('lz4') crash with "Cannot find module 'lz4'". Node 20 passed; 22/24/26
failed — confirmed via fail-fast:false per-version run.

Replaces it with `lz4-napi@2.9.0` (maintained, napi-rs, ships prebuilds ->
no local toolchain build, works on all Node versions). Verified locally that
lz4-napi's compressFrameSync/decompressFrameSync are byte-compatible with the
LZ4 *frame* format (magic 0x184D2204) the old package used and the Databricks
server sends — round-trip and cross-decode both directions, incl. large
buffers. lib/utils/lz4.ts now wraps lz4-napi behind the same { encode, decode }
codec, so call sites are unchanged and the load-failure-tolerant contract is
kept. Also fixes #293 (deprecated Buffer() warning + xxhash build-resolution
error, both from the old lz4 package).

DIAGNOSTIC (this commit): CI temporarily uses `npm install` instead of
`npm ci` so the JFrog registry resolves lz4-napi + its @lz4-napi/* platform
packages (the dev npm proxy doesn't mirror those scoped packages, so the lock
couldn't be generated locally). If CI resolves them, I'll pin a proper lock
and restore `npm ci`.

Co-authored-by: Isaac
Signed-off-by: Vikrant Puppala <vikrant.puppala@databricks.com>

Author: vikrantpuppala

.github/workflows/main.yml

@@ -51,7 +51,7 @@ jobs:
             ${{ runner.os }}-
       - name: Check code style
         run: |
-          npm ci
+          npm install --no-audit --no-fund
           npm run prettier
           npm run lint
 
@@ -90,7 +90,7 @@ jobs:
             ${{ runner.os }}-${{ matrix.node-version }}-
       - name: Run unit tests
         run: |
-          npm ci
+          npm install --no-audit --no-fund
           npm run test
       - run: tar -cvf ${{ env.NYC_REPORT_DIR }}.tar ${{ env.NYC_REPORT_DIR }}
       - name: Store coverage report
@@ -150,7 +150,7 @@ jobs:
             ${{ runner.os }}-${{ matrix.node-version }}-
       - name: Run e2e tests
         run: |
-          npm ci
+          npm install --no-audit --no-fund
           NODE_OPTIONS="--max-old-space-size=4096" npm run e2e
       - run: tar -cvf ${{ env.NYC_REPORT_DIR }}.tar ${{ env.NYC_REPORT_DIR }}
       - name: Store coverage report

lib/utils/lz4.ts

@@ -1,38 +1,56 @@
-import type LZ4Namespace from 'lz4';
-
-type LZ4Module = typeof LZ4Namespace;
+// LZ4 frame codec, backed by `lz4-napi`.
+//
+// Historical note: this driver previously depended on the unmaintained
+// `lz4` package (last published 2021), a node-gyp native addon with no
+// prebuilt binaries. It failed to compile on Node 22+, so it was silently
+// skipped as an optional dependency and LZ4-compressed results could not be
+// decoded. `lz4-napi` is maintained and ships prebuilt binaries for all
+// supported platforms/Node versions, so no local toolchain build is needed.
+//
+// The Databricks server sends LZ4-compressed Arrow/CloudFetch payloads in
+// the LZ4 **frame** format (magic 0x184D2204). `lz4-napi`'s
+// `compressFrameSync`/`decompressFrameSync` are byte-compatible with the
+// frames the old `lz4` package produced/consumed — verified by round-trip
+// and cross-decoding both directions. We expose the same `{ encode, decode }`
+// surface the rest of the driver already calls, so call sites are unchanged.
+
+type Lz4Napi = {
+  compressFrameSync(data: Buffer): Buffer;
+  decompressFrameSync(data: Buffer): Buffer;
+};
+
+// Stable interface used across the driver (lib call sites do `LZ4()!.decode(buf)`).
+export interface LZ4Codec {
+  encode(data: Buffer): Buffer;
+  decode(data: Buffer): Buffer;
+}
 
-function tryLoadLZ4Module(): LZ4Module | undefined {
+function tryLoadLZ4Module(): LZ4Codec | undefined {
+  let napi: Lz4Napi;
   try {
-    return require('lz4'); // eslint-disable-line global-require
+    napi = require('lz4-napi'); // eslint-disable-line global-require
   } catch (err) {
-    if (!(err instanceof Error) || !('code' in err)) {
-      // eslint-disable-next-line no-console
-      console.warn('Unexpected error loading LZ4 module: Invalid error object', err);
-      return undefined;
-    }
-
-    if (err.code === 'MODULE_NOT_FOUND') {
+    // lz4-napi ships prebuilds so this should not normally happen, but keep
+    // the load-failure-tolerant contract: if it can't load, LZ4 support is
+    // reported as unavailable rather than crashing the driver.
+    if (err instanceof Error && 'code' in err && err.code === 'MODULE_NOT_FOUND') {
       return undefined;
     }
-
-    if (err.code === 'ERR_DLOPEN_FAILED') {
-      // eslint-disable-next-line no-console
-      console.warn('LZ4 native module failed to load: Architecture or version mismatch', err);
-      return undefined;
-    }
-
-    // If it's not a known error, return undefined
     // eslint-disable-next-line no-console
-    console.warn('Unknown error loading LZ4 module: Unhandled error code', err);
+    console.warn('LZ4 module (lz4-napi) failed to load; LZ4-compressed results unavailable', err);
     return undefined;
   }
+
+  return {
+    encode: (data: Buffer): Buffer => Buffer.from(napi.compressFrameSync(data)),
+    decode: (data: Buffer): Buffer => Buffer.from(napi.decompressFrameSync(data)),
+  };
 }
 
-// The null is already tried resolving that failed
-let resolvedModule: LZ4Module | null | undefined;
+// The null means we already tried resolving and it failed.
+let resolvedModule: LZ4Codec | null | undefined;
 
-function getResolvedModule() {
+function getResolvedModule(): LZ4Codec | undefined {
   if (resolvedModule === undefined) {
     resolvedModule = tryLoadLZ4Module() ?? null;
   }