Skip to content

Support zip64 archives larger than 4 GiB #352

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
slimbuck merged 2 commits into from
Oct 7, 2026
Merged

Support zip64 archives larger than 4 GiB #352

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
47e5215
latest
slimbuck Oct 7, 2026
efc7cb2
latest
slimbuck Oct 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 58 additions & 8 deletions src/lib/io/read/zip-file-system.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,8 +224,10 @@ class ZipReadFileSystem implements ReadFileSystem {
throw new Error('Cannot read zip from source with unknown size');
}

// Read the last 65KB to find the End of Central Directory record
const eocdSearchSize = Math.min(65536 + 22, size);
// Read the last 65KB to find the End of Central Directory record (plus
// 20 bytes so the zip64 locator ahead of it is in reach even behind a
// maximum-length comment)
const eocdSearchSize = Math.min(65536 + 22 + 20, size);
const eocdStream = this.source.read(size - eocdSearchSize, size);
const eocdData = await eocdStream.readAll();
eocdStream.close();
Expand All @@ -249,9 +251,32 @@ class ZipReadFileSystem implements ReadFileSystem {
}

const eocdView = new DataView(eocdData.buffer, eocdData.byteOffset + eocdOffset, 22);
const entryCount = eocdView.getUint16(10, true);
const cdSize = eocdView.getUint32(12, true);
const cdOffset = eocdView.getUint32(16, true);
let entryCount = eocdView.getUint16(10, true);
let cdSize = eocdView.getUint32(12, true);
let cdOffset = eocdView.getUint32(16, true);

// A maxed-out field means the real value is in the zip64 end of
// central directory record, found via the locator just ahead of the
// EOCD. Without a locator the classic values stand.
if (entryCount === 0xffff || cdSize === 0xffffffff || cdOffset === 0xffffffff) {
const locatorOffset = eocdOffset - 20;
const locatorView =
locatorOffset >= 0 ? new DataView(eocdData.buffer, eocdData.byteOffset + locatorOffset, 20) : null;
if (locatorView && locatorView.getUint32(0, true) === 0x07064b50) {
const zip64EocdOffset = Number(locatorView.getBigUint64(8, true));
const zip64Stream = this.source.read(zip64EocdOffset, zip64EocdOffset + 56);
const zip64Data = await zip64Stream.readAll();
zip64Stream.close();

const zip64View = new DataView(zip64Data.buffer, zip64Data.byteOffset, 56);
if (zip64View.getUint32(0, true) !== 0x06064b50) {
throw new Error('Invalid zip64 end of central directory signature');
}
entryCount = Number(zip64View.getBigUint64(32, true));
cdSize = Number(zip64View.getBigUint64(40, true));
cdOffset = Number(zip64View.getBigUint64(48, true));
}
}

// Read central directory
const cdStream = this.source.read(cdOffset, cdOffset + cdSize);
Expand All @@ -275,12 +300,37 @@ class ZipReadFileSystem implements ReadFileSystem {

const gpFlags = cdView.getUint16(8, true);
const method = cdView.getUint16(10, true);
const compressedSize = cdView.getUint32(20, true);
const uncompressedSize = cdView.getUint32(24, true);
let compressedSize = cdView.getUint32(20, true);
let uncompressedSize = cdView.getUint32(24, true);
const nameLen = cdView.getUint16(28, true);
const extraLen = cdView.getUint16(30, true);
const commentLen = cdView.getUint16(32, true);
const localHeaderOffset = cdView.getUint32(42, true);
let localHeaderOffset = cdView.getUint32(42, true);

// zip64: maxed-out fields are carried, in fixed order, by the
// zip64 extended information extra field (header id 0x0001)
if (uncompressedSize === 0xffffffff || compressedSize === 0xffffffff || localHeaderOffset === 0xffffffff) {
const extraEnd = 46 + nameLen + extraLen;
let p = 46 + nameLen;
while (p + 4 <= extraEnd && cdView.getUint16(p, true) !== 0x0001) {
p += 4 + cdView.getUint16(p + 2, true);
}
if (p + 4 > extraEnd) {
throw new Error('Zip64 extra field not found in central directory entry');
}
p += 4;
if (uncompressedSize === 0xffffffff) {
uncompressedSize = Number(cdView.getBigUint64(p, true));
p += 8;
}
if (compressedSize === 0xffffffff) {
compressedSize = Number(cdView.getBigUint64(p, true));
p += 8;
}
if (localHeaderOffset === 0xffffffff) {
localHeaderOffset = Number(cdView.getBigUint64(p, true));
}
}

const nameBytes = cdData.subarray(offset + 46, offset + 46 + nameLen);
const utf8 = (gpFlags & 0x800) !== 0;
Expand Down
115 changes: 91 additions & 24 deletions src/lib/io/write/zip-file-system.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { Crc } from './crc';
import type { FileSystem, Writer } from './file-system';

// https://gist.github.com/rvaiya/4a2192df729056880a027789ae3cd4b7
// zip64: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT (4.3.9, 4.3.14-4.3.16, 4.5.3)

type ZipEntry = {
filename: Uint8Array;
Expand Down Expand Up @@ -47,7 +48,9 @@ class ZipEntryWriter implements Writer {
* A file system that writes files into a ZIP archive.
*
* Creates a ZIP file containing all written files. Used internally
* for bundled output formats like .sog files.
* for bundled output formats like .sog files. Archives past the classic
* 4 GiB / 65535-entry limits are written with zip64 records; smaller
* archives use the classic layout only.
*
* @example
* ```ts
Expand All @@ -64,11 +67,21 @@ class ZipEntryWriter implements Writer {
* ```
*/
class ZipFileSystem implements FileSystem {
/**
* Sizes and offsets at or above this value are written in zip64 form
* (0xFFFFFFFF in the u32 field, real value in a zip64 record). Tests lower
* it to exercise zip64 without writing 4 GiB.
*
* @ignore
*/
static zip64Limit = 0xffffffff;

close: () => Promise<void>;
createWriter: (filename: string) => Promise<Writer>;
mkdir: (path: string) => Promise<void>;

constructor(writer: Writer) {
const zip64Limit = ZipFileSystem.zip64Limit;
const textEncoder = new TextEncoder();
const files: ZipEntry[] = [];
let activeEntry: ZipEntry | null = null;
Expand Down Expand Up @@ -107,21 +120,24 @@ class ZipFileSystem implements FileSystem {

const writeEntryFooter = async (entry: ZipEntry) => {
const { crc, sizeBytes } = entry;
offset += sizeBytes + 16;
// Plain (non-zip64) zip stores every size/offset as u32. Fail loud
// when the archive crosses 4 GiB rather than silently writing
// wrapped offsets (an unreadable file).
if (offset >= 0xffffffff) {
throw new Error(
'Zip output exceeds 4 GiB and zip64 is not supported — write unbundled output instead.'
);
}
const data = new Uint8Array(16);
// The local header is written before the size is known, so an
// entry that outgrows u32 gets the zip64 (8-byte size) data
// descriptor and its central directory record carries the zip64
// sizes - the same scheme as Go's archive/zip and Java's
// ZipOutputStream.
const zip64 = sizeBytes >= zip64Limit;
const data = new Uint8Array(zip64 ? 24 : 16);
const view = new DataView(data.buffer);
view.setUint32(0, 0x08074b50, true);
view.setUint32(4, crc.value(), true);
view.setUint32(8, sizeBytes, true);
view.setUint32(12, sizeBytes, true);
if (zip64) {
view.setBigUint64(8, BigInt(sizeBytes), true);
view.setBigUint64(16, BigInt(sizeBytes), true);
} else {
view.setUint32(8, sizeBytes, true);
view.setUint32(12, sizeBytes, true);
}
offset += sizeBytes + data.length;
await writer.write(data);
};

Expand Down Expand Up @@ -150,40 +166,91 @@ class ZipFileSystem implements FileSystem {
activeEntry = null;
}

// central directory starts where entry data ended
const cdOffset = offset;

// Write central directory records
for (const file of files) {
const { filename, crc, sizeBytes, headerOffset } = file;
const nameLen = filename.length;

const cdr = new Uint8Array(46 + nameLen);
// zip64 extended information extra field: holds only the
// overflowed fields, in fixed order (sizes, then header offset)
const sizeZip64 = sizeBytes >= zip64Limit;
const offsetZip64 = headerOffset >= zip64Limit;
const zip64Len = (sizeZip64 ? 16 : 0) + (offsetZip64 ? 8 : 0);
const extraLen = zip64Len ? 4 + zip64Len : 0;
const version = zip64Len ? 45 : 20;

const cdr = new Uint8Array(46 + nameLen + extraLen);
const view = new DataView(cdr.buffer);
view.setUint32(0, 0x02014b50, true);
view.setUint16(4, 20, true);
view.setUint16(6, 20, true);
view.setUint16(4, version, true);
view.setUint16(6, version, true);
view.setUint16(8, 0x8 | 0x800, true);
view.setUint16(10, 0, true);
view.setUint16(12, dosTime, true);
view.setUint16(14, dosDate, true);
view.setUint32(16, crc.value(), true);
view.setUint32(20, sizeBytes, true);
view.setUint32(24, sizeBytes, true);
view.setUint32(20, sizeZip64 ? 0xffffffff : sizeBytes, true);
view.setUint32(24, sizeZip64 ? 0xffffffff : sizeBytes, true);
view.setUint16(28, nameLen, true);
view.setUint32(42, headerOffset, true);
view.setUint16(30, extraLen, true);
view.setUint32(42, offsetZip64 ? 0xffffffff : headerOffset, true);
cdr.set(filename, 46);

if (zip64Len) {
let p = 46 + nameLen;
view.setUint16(p, 0x0001, true);
view.setUint16(p + 2, zip64Len, true);
p += 4;
if (sizeZip64) {
view.setBigUint64(p, BigInt(sizeBytes), true); // uncompressed
view.setBigUint64(p + 8, BigInt(sizeBytes), true); // compressed
p += 16;
}
if (offsetZip64) {
view.setBigUint64(p, BigInt(headerOffset), true);
}
}

offset += cdr.length;
await writer.write(cdr);
}

const filenameLength = files.reduce((tot, file) => tot + file.filename.length, 0);
const cdSize = offset - cdOffset;
const numFiles = files.length;

// Write zip64 end of central directory record and locator ahead of
// the classic record when any of its fields overflow
if (numFiles >= 0xffff || cdSize >= zip64Limit || cdOffset >= zip64Limit) {
const zip64Eocd = new Uint8Array(56 + 20);
const zip64View = new DataView(zip64Eocd.buffer);
zip64View.setUint32(0, 0x06064b50, true);
zip64View.setBigUint64(4, 44n, true); // size of the rest of the record
zip64View.setUint16(12, 45, true);
zip64View.setUint16(14, 45, true);
zip64View.setBigUint64(24, BigInt(numFiles), true);
zip64View.setBigUint64(32, BigInt(numFiles), true);
zip64View.setBigUint64(40, BigInt(cdSize), true);
zip64View.setBigUint64(48, BigInt(cdOffset), true);

// locator
zip64View.setUint32(56, 0x07064b50, true);
zip64View.setBigUint64(64, BigInt(offset), true); // zip64 record offset
zip64View.setUint32(72, 1, true); // total number of disks

await writer.write(zip64Eocd);
}

// Write end of central directory record
const eocd = new Uint8Array(22);
const eocdView = new DataView(eocd.buffer);
eocdView.setUint32(0, 0x06054b50, true);
eocdView.setUint16(8, files.length, true);
eocdView.setUint16(10, files.length, true);
eocdView.setUint32(12, filenameLength + files.length * 46, true);
eocdView.setUint32(16, offset, true); // central directory starts where entry data ended
eocdView.setUint16(8, Math.min(numFiles, 0xffff), true);
eocdView.setUint16(10, Math.min(numFiles, 0xffff), true);
eocdView.setUint32(12, cdSize >= zip64Limit ? 0xffffffff : cdSize, true);
eocdView.setUint32(16, cdOffset >= zip64Limit ? 0xffffffff : cdOffset, true);

await writer.write(eocd);

Expand Down
Loading
Loading