From 925e8cf0cf9e11b3f275327fa36a9f0786597a6b Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Thu, 24 Sep 2026 14:52:37 +0200 Subject: [PATCH] Add zip-deflate support --- CHANGELOG.md | 4 ++ pixi.lock | 3 +- pyproject.toml | 2 + tests/test_deflate64.py | 68 +++++++++++++++++++++++++++++ vecorel_cli/conversion/base.py | 13 ++---- vecorel_cli/conversion/deflate64.py | 54 +++++++++++++++++++++++ 6 files changed, 133 insertions(+), 11 deletions(-) create mode 100644 tests/test_deflate64.py create mode 100644 vecorel_cli/conversion/deflate64.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b3d645..d96ce19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +- Fix: ZIP archives compressed with Deflate64 are extracted through `inflate64` (a + dependency of py7zr, now declared). They required `zipfile-deflate64`, which was + not declared and has no wheels for Python 3.11+. + ## [v0.3.0] - 2026-09-24 - Fix: a download that ends before its `Content-Length` is reached now fails instead diff --git a/pixi.lock b/pixi.lock index 1185af6..b86f977 100644 --- a/pixi.lock +++ b/pixi.lock @@ -31645,7 +31645,7 @@ packages: - pypi: ./ name: vecorel-cli version: 0.3.0 - sha256: 6d8b4596d0e2ff27e9b759a38ed470199afecb01550c584f433e0913bceb3a41 + sha256: 9bd6cc8cdd22b31a6f24cf5c537595fa48ef920454b4701d1b33f1e8a5779f30 requires_dist: - pyyaml>=6.0,<7.0 - click>=8.1,<9.0 @@ -31655,6 +31655,7 @@ packages: - numpy>=2.0,<3.0 - pyarrow>=21.0,<24.0 - py7zr>=1.0,<2.0 + - inflate64>=1.0,<2.0 - multivolumefile>=0.2.3,<1.0 - fsspec>=2025.7.0 - jsonschema[format]>=4.20,<5.0 diff --git a/pyproject.toml b/pyproject.toml index a725c40..0e9fd31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "numpy>=2.0,<3.0", "pyarrow>=21.0,<24.0", "py7zr>=1.0,<2.0", + "inflate64>=1.0,<2.0", "multivolumefile>=0.2.3,<1.0", "fsspec>=2025.7.0", "jsonschema[format]>=4.20,<5.0", @@ -72,6 +73,7 @@ shapely = ">=2.1,<3.0" numpy = ">=2.0,<3.0" pyarrow = ">=21.0,<24.0" py7zr = ">=1.0,<2.0" +inflate64 = ">=1.0,<2.0" multivolumefile = ">=0.2.3,<1.0" unrar = ">=0.4" diff --git a/tests/test_deflate64.py b/tests/test_deflate64.py new file mode 100644 index 0000000..78c3085 --- /dev/null +++ b/tests/test_deflate64.py @@ -0,0 +1,68 @@ +import random +import struct +import zipfile +import zlib + +import inflate64 +from pytest import mark, raises + +from vecorel_cli.conversion.base import BaseConverter +from vecorel_cli.conversion.deflate64 import ZIP_DEFLATED64, deflate64_zip + +# Compressible content fits one read; random bytes need many, which exercises the +# streaming decompressor protocol of ZipExtFile. +CONTENTS = { + "compressible": b"deflate64 " * 20_000, + "incompressible": random.Random(64).randbytes(2_000_000), +} + + +def _write_deflate64_zip(path, data, name="member.txt"): + """A single-member ZIP compressed with Deflate64, which zipfile cannot write.""" + deflater = inflate64.Deflater() + compressed = deflater.deflate(data) + deflater.flush() + crc = zlib.crc32(data) + encoded = name.encode() + sizes = (crc, len(compressed), len(data), len(encoded)) + local = struct.pack("<4s5H3L2H", b"PK\x03\x04", 21, 0, ZIP_DEFLATED64, 0, 0, *sizes, 0) + central = struct.pack( + "<4s6H3L5H2L", b"PK\x01\x02", 21, 21, 0, ZIP_DEFLATED64, 0, 0, *sizes, 0, 0, 0, 0, 0, 0 + ) + offset = len(local) + len(encoded) + len(compressed) + end = struct.pack("<4s4H2LH", b"PK\x05\x06", 0, 0, 1, 1, len(central) + len(encoded), offset, 0) + path.write_bytes(local + encoded + compressed + central + encoded + end) + return path + + +@mark.parametrize("kind", CONTENTS) +def test_zipfile_reads_deflate64_only_inside_the_block(tmp_path, kind): + archive = _write_deflate64_zip(tmp_path / "a.zip", CONTENTS[kind]) + with raises(NotImplementedError): + zipfile.ZipFile(archive).read("member.txt") + with deflate64_zip(): + # read() verifies the CRC, so a truncated or corrupt stream fails here + assert zipfile.ZipFile(archive).read("member.txt") == CONTENTS[kind] + with raises(NotImplementedError): + zipfile.ZipFile(archive).read("member.txt") + + +def test_overlapping_blocks_restore_zipfile_whatever_order_they_exit_in(): + original = zipfile._get_decompressor + first, second = deflate64_zip(), deflate64_zip() + first.__enter__() + second.__enter__() + first.__exit__(None, None, None) + assert zipfile._get_decompressor is not original # the second block still needs it + second.__exit__(None, None, None) + assert zipfile._get_decompressor is original + + +@mark.parametrize("kind", CONTENTS) +def test_download_files_extracts_deflate64_archives(tmp_path, kind): + archive = _write_deflate64_zip(tmp_path / "source.zip", CONTENTS[kind]) + paths = BaseConverter().download_files( + {str(archive): ["member.txt"]}, cache_folder=str(tmp_path / "cache") + ) + assert len(paths) == 1 + with open(paths[0][0], "rb") as f: + assert f.read() == CONTENTS[kind] diff --git a/vecorel_cli/conversion/base.py b/vecorel_cli/conversion/base.py index 3277278..0e942f6 100644 --- a/vecorel_cli/conversion/base.py +++ b/vecorel_cli/conversion/base.py @@ -34,6 +34,7 @@ from ..vecorel.schemas import Schemas from ..vecorel.typing import Sources from ..vecorel.util import get_fs, name_from_uri, stream_file, suffix_duplicate_ids +from .deflate64 import deflate64_zip from .flatdict import FlatDict # a source column named `id` moves here when the converter generates its own id, @@ -399,16 +400,8 @@ def download_files(self, uris, cache_folder=None, **kwargs): if must_extract: if zipfile.is_zipfile(cache_file): - try: - with zipfile.ZipFile(cache_file, "r") as zip_file: - zip_file.extractall(zip_folder) - except NotImplementedError as e: - if str(e) != "That compression method is not supported": - raise e - import zipfile_deflate64 - - with zipfile_deflate64.ZipFile(cache_file, "r") as zip_file: - zip_file.extractall(zip_folder) + with deflate64_zip(), zipfile.ZipFile(cache_file, "r") as zip_file: + zip_file.extractall(zip_folder) elif py7zr.is_7zfile(cache_file): with py7zr.SevenZipFile(cache_file, "r") as sz_file: sz_file.extractall(zip_folder) diff --git a/vecorel_cli/conversion/deflate64.py b/vecorel_cli/conversion/deflate64.py new file mode 100644 index 0000000..3cd7b56 --- /dev/null +++ b/vecorel_cli/conversion/deflate64.py @@ -0,0 +1,54 @@ +import threading +import zipfile +from contextlib import contextmanager + +import inflate64 + +# Deflate64 ("enhanced deflate", ZIP method 9) is what Windows writes for large archives, +# and python's zipfile cannot read it. inflate64, a dependency of py7zr, decodes it. +ZIP_DEFLATED64 = 9 + +_lock = threading.Lock() +_users = 0 +_original_get_decompressor = zipfile._get_decompressor + + +class _Deflate64Decompressor: + """The interface zipfile.ZipExtFile expects of a decompressor other than zlib's + (Python 3.11 to 3.14): decompress(data) and eof.""" + + def __init__(self): + self._inflater = inflate64.Inflater() + + @property + def eof(self): + return self._inflater.eof + + def decompress(self, data): + return self._inflater.inflate(data) + + +def _get_decompressor(compress_type): + if compress_type == ZIP_DEFLATED64: + return _Deflate64Decompressor() + return _original_get_decompressor(compress_type) + + +@contextmanager +def deflate64_zip(): + """Let zipfile read Deflate64 members for the duration of the block. + + The hook is process-global, so it is counted: installed by the first block to enter + and restored by the last to exit, whatever order overlapping blocks exit in.""" + global _users + with _lock: + if _users == 0: + zipfile._get_decompressor = _get_decompressor + _users += 1 + try: + yield + finally: + with _lock: + _users -= 1 + if _users == 0: + zipfile._get_decompressor = _original_get_decompressor