Better ZIP deflate support

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+- Fix: ZIP archives compressed with Deflate64 are extracted through `inflate64` (a
+  dependency of py7zr, now declared). They required `zipfile-deflate64`, which was
+  not declared and has no wheels for Python 3.11+.
+
 ## [v0.3.0] - 2026-09-24
 
 - Fix: a download that ends before its `Content-Length` is reached now fails instead

pyproject.toml

@@ -30,6 +30,7 @@ dependencies = [
     "numpy>=2.0,<3.0",
     "pyarrow>=21.0,<24.0",
     "py7zr>=1.0,<2.0",
+    "inflate64>=1.0,<2.0",
     "multivolumefile>=0.2.3,<1.0",
     "fsspec>=2025.7.0",
     "jsonschema[format]>=4.20,<5.0",
@@ -72,6 +73,7 @@ shapely = ">=2.1,<3.0"
 numpy = ">=2.0,<3.0"
 pyarrow = ">=21.0,<24.0"
 py7zr = ">=1.0,<2.0"
+inflate64 = ">=1.0,<2.0"
 multivolumefile = ">=0.2.3,<1.0"
 unrar = ">=0.4"
 

tests/test_deflate64.py

@@ -0,0 +1,68 @@
+import random
+import struct
+import zipfile
+import zlib
+
+import inflate64
+from pytest import mark, raises
+
+from vecorel_cli.conversion.base import BaseConverter
+from vecorel_cli.conversion.deflate64 import ZIP_DEFLATED64, deflate64_zip
+
+# Compressible content fits one read; random bytes need many, which exercises the
+# streaming decompressor protocol of ZipExtFile.
+CONTENTS = {
+    "compressible": b"deflate64 " * 20_000,
+    "incompressible": random.Random(64).randbytes(2_000_000),
+}
+
+
+def _write_deflate64_zip(path, data, name="member.txt"):
+    """A single-member ZIP compressed with Deflate64, which zipfile cannot write."""
+    deflater = inflate64.Deflater()
+    compressed = deflater.deflate(data) + deflater.flush()
+    crc = zlib.crc32(data)
+    encoded = name.encode()
+    sizes = (crc, len(compressed), len(data), len(encoded))
+    local = struct.pack("<4s5H3L2H", b"PK\x03\x04", 21, 0, ZIP_DEFLATED64, 0, 0, *sizes, 0)
+    central = struct.pack(
+        "<4s6H3L5H2L", b"PK\x01\x02", 21, 21, 0, ZIP_DEFLATED64, 0, 0, *sizes, 0, 0, 0, 0, 0, 0
+    )
+    offset = len(local) + len(encoded) + len(compressed)
+    end = struct.pack("<4s4H2LH", b"PK\x05\x06", 0, 0, 1, 1, len(central) + len(encoded), offset, 0)
+    path.write_bytes(local + encoded + compressed + central + encoded + end)
+    return path
+
+
+@mark.parametrize("kind", CONTENTS)
+def test_zipfile_reads_deflate64_only_inside_the_block(tmp_path, kind):
+    archive = _write_deflate64_zip(tmp_path / "a.zip", CONTENTS[kind])
+    with raises(NotImplementedError):
+        zipfile.ZipFile(archive).read("member.txt")
+    with deflate64_zip():
+        # read() verifies the CRC, so a truncated or corrupt stream fails here
+        assert zipfile.ZipFile(archive).read("member.txt") == CONTENTS[kind]
+    with raises(NotImplementedError):
+        zipfile.ZipFile(archive).read("member.txt")
+
+
+def test_overlapping_blocks_restore_zipfile_whatever_order_they_exit_in():
+    original = zipfile._get_decompressor
+    first, second = deflate64_zip(), deflate64_zip()
+    first.__enter__()
+    second.__enter__()
+    first.__exit__(None, None, None)
+    assert zipfile._get_decompressor is not original  # the second block still needs it
+    second.__exit__(None, None, None)
+    assert zipfile._get_decompressor is original
+
+
+@mark.parametrize("kind", CONTENTS)
+def test_download_files_extracts_deflate64_archives(tmp_path, kind):
+    archive = _write_deflate64_zip(tmp_path / "source.zip", CONTENTS[kind])
+    paths = BaseConverter().download_files(
+        {str(archive): ["member.txt"]}, cache_folder=str(tmp_path / "cache")
+    )
+    assert len(paths) == 1
+    with open(paths[0][0], "rb") as f:
+        assert f.read() == CONTENTS[kind]

vecorel_cli/conversion/base.py

@@ -34,6 +34,7 @@
 from ..vecorel.schemas import Schemas
 from ..vecorel.typing import Sources
 from ..vecorel.util import get_fs, name_from_uri, stream_file, suffix_duplicate_ids
+from .deflate64 import deflate64_zip
 from .flatdict import FlatDict
 
 # a source column named `id` moves here when the converter generates its own id,
@@ -399,16 +400,8 @@ def download_files(self, uris, cache_folder=None, **kwargs):
 
             if must_extract:
                 if zipfile.is_zipfile(cache_file):
-                    try:
-                        with zipfile.ZipFile(cache_file, "r") as zip_file:
-                            zip_file.extractall(zip_folder)
-                    except NotImplementedError as e:
-                        if str(e) != "That compression method is not supported":
-                            raise e
-                        import zipfile_deflate64
-
-                        with zipfile_deflate64.ZipFile(cache_file, "r") as zip_file:
-                            zip_file.extractall(zip_folder)
+                    with deflate64_zip(), zipfile.ZipFile(cache_file, "r") as zip_file:
+                        zip_file.extractall(zip_folder)
                 elif py7zr.is_7zfile(cache_file):
                     with py7zr.SevenZipFile(cache_file, "r") as sz_file:
                         sz_file.extractall(zip_folder)

vecorel_cli/conversion/deflate64.py

@@ -0,0 +1,54 @@
+import threading
+import zipfile
+from contextlib import contextmanager
+
+import inflate64
+
+# Deflate64 ("enhanced deflate", ZIP method 9) is what Windows writes for large archives,
+# and python's zipfile cannot read it. inflate64, a dependency of py7zr, decodes it.
+ZIP_DEFLATED64 = 9
+
+_lock = threading.Lock()
+_users = 0
+_original_get_decompressor = zipfile._get_decompressor
+
+
+class _Deflate64Decompressor:
+    """The interface zipfile.ZipExtFile expects of a decompressor other than zlib's
+    (Python 3.11 to 3.14): decompress(data) and eof."""
+
+    def __init__(self):
+        self._inflater = inflate64.Inflater()
+
+    @property
+    def eof(self):
+        return self._inflater.eof
+
+    def decompress(self, data):
+        return self._inflater.inflate(data)
+
+
+def _get_decompressor(compress_type):
+    if compress_type == ZIP_DEFLATED64:
+        return _Deflate64Decompressor()
+    return _original_get_decompressor(compress_type)
+
+
+@contextmanager
+def deflate64_zip():
+    """Let zipfile read Deflate64 members for the duration of the block.
+
+    The hook is process-global, so it is counted: installed by the first block to enter
+    and restored by the last to exit, whatever order overlapping blocks exit in."""
+    global _users
+    with _lock:
+        if _users == 0:
+            zipfile._get_decompressor = _get_decompressor
+        _users += 1
+    try:
+        yield
+    finally:
+        with _lock:
+            _users -= 1
+            if _users == 0:
+                zipfile._get_decompressor = _original_get_decompressor