diff --git a/src/lib/io/read/zip-file-system.ts b/src/lib/io/read/zip-file-system.ts index 294382d8..cd1c6282 100644 --- a/src/lib/io/read/zip-file-system.ts +++ b/src/lib/io/read/zip-file-system.ts @@ -224,8 +224,10 @@ class ZipReadFileSystem implements ReadFileSystem { throw new Error('Cannot read zip from source with unknown size'); } - // Read the last 65KB to find the End of Central Directory record - const eocdSearchSize = Math.min(65536 + 22, size); + // Read the last 65KB to find the End of Central Directory record (plus + // 20 bytes so the zip64 locator ahead of it is in reach even behind a + // maximum-length comment) + const eocdSearchSize = Math.min(65536 + 22 + 20, size); const eocdStream = this.source.read(size - eocdSearchSize, size); const eocdData = await eocdStream.readAll(); eocdStream.close(); @@ -249,9 +251,32 @@ class ZipReadFileSystem implements ReadFileSystem { } const eocdView = new DataView(eocdData.buffer, eocdData.byteOffset + eocdOffset, 22); - const entryCount = eocdView.getUint16(10, true); - const cdSize = eocdView.getUint32(12, true); - const cdOffset = eocdView.getUint32(16, true); + let entryCount = eocdView.getUint16(10, true); + let cdSize = eocdView.getUint32(12, true); + let cdOffset = eocdView.getUint32(16, true); + + // A maxed-out field means the real value is in the zip64 end of + // central directory record, found via the locator just ahead of the + // EOCD. Without a locator the classic values stand. + if (entryCount === 0xffff || cdSize === 0xffffffff || cdOffset === 0xffffffff) { + const locatorOffset = eocdOffset - 20; + const locatorView = + locatorOffset >= 0 ? new DataView(eocdData.buffer, eocdData.byteOffset + locatorOffset, 20) : null; + if (locatorView && locatorView.getUint32(0, true) === 0x07064b50) { + const zip64EocdOffset = Number(locatorView.getBigUint64(8, true)); + const zip64Stream = this.source.read(zip64EocdOffset, zip64EocdOffset + 56); + const zip64Data = await zip64Stream.readAll(); + zip64Stream.close(); + + const zip64View = new DataView(zip64Data.buffer, zip64Data.byteOffset, 56); + if (zip64View.getUint32(0, true) !== 0x06064b50) { + throw new Error('Invalid zip64 end of central directory signature'); + } + entryCount = Number(zip64View.getBigUint64(32, true)); + cdSize = Number(zip64View.getBigUint64(40, true)); + cdOffset = Number(zip64View.getBigUint64(48, true)); + } + } // Read central directory const cdStream = this.source.read(cdOffset, cdOffset + cdSize); @@ -275,12 +300,37 @@ class ZipReadFileSystem implements ReadFileSystem { const gpFlags = cdView.getUint16(8, true); const method = cdView.getUint16(10, true); - const compressedSize = cdView.getUint32(20, true); - const uncompressedSize = cdView.getUint32(24, true); + let compressedSize = cdView.getUint32(20, true); + let uncompressedSize = cdView.getUint32(24, true); const nameLen = cdView.getUint16(28, true); const extraLen = cdView.getUint16(30, true); const commentLen = cdView.getUint16(32, true); - const localHeaderOffset = cdView.getUint32(42, true); + let localHeaderOffset = cdView.getUint32(42, true); + + // zip64: maxed-out fields are carried, in fixed order, by the + // zip64 extended information extra field (header id 0x0001) + if (uncompressedSize === 0xffffffff || compressedSize === 0xffffffff || localHeaderOffset === 0xffffffff) { + const extraEnd = 46 + nameLen + extraLen; + let p = 46 + nameLen; + while (p + 4 <= extraEnd && cdView.getUint16(p, true) !== 0x0001) { + p += 4 + cdView.getUint16(p + 2, true); + } + if (p + 4 > extraEnd) { + throw new Error('Zip64 extra field not found in central directory entry'); + } + p += 4; + if (uncompressedSize === 0xffffffff) { + uncompressedSize = Number(cdView.getBigUint64(p, true)); + p += 8; + } + if (compressedSize === 0xffffffff) { + compressedSize = Number(cdView.getBigUint64(p, true)); + p += 8; + } + if (localHeaderOffset === 0xffffffff) { + localHeaderOffset = Number(cdView.getBigUint64(p, true)); + } + } const nameBytes = cdData.subarray(offset + 46, offset + 46 + nameLen); const utf8 = (gpFlags & 0x800) !== 0; diff --git a/src/lib/io/write/zip-file-system.ts b/src/lib/io/write/zip-file-system.ts index 45dd76e8..6da260e4 100644 --- a/src/lib/io/write/zip-file-system.ts +++ b/src/lib/io/write/zip-file-system.ts @@ -2,6 +2,7 @@ import { Crc } from './crc'; import type { FileSystem, Writer } from './file-system'; // https://gist.github.com/rvaiya/4a2192df729056880a027789ae3cd4b7 +// zip64: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT (4.3.9, 4.3.14-4.3.16, 4.5.3) type ZipEntry = { filename: Uint8Array; @@ -47,7 +48,9 @@ class ZipEntryWriter implements Writer { * A file system that writes files into a ZIP archive. * * Creates a ZIP file containing all written files. Used internally - * for bundled output formats like .sog files. + * for bundled output formats like .sog files. Archives past the classic + * 4 GiB / 65535-entry limits are written with zip64 records; smaller + * archives use the classic layout only. * * @example * ```ts @@ -64,11 +67,21 @@ class ZipEntryWriter implements Writer { * ``` */ class ZipFileSystem implements FileSystem { + /** + * Sizes and offsets at or above this value are written in zip64 form + * (0xFFFFFFFF in the u32 field, real value in a zip64 record). Tests lower + * it to exercise zip64 without writing 4 GiB. + * + * @ignore + */ + static zip64Limit = 0xffffffff; + close: () => Promise; createWriter: (filename: string) => Promise; mkdir: (path: string) => Promise; constructor(writer: Writer) { + const zip64Limit = ZipFileSystem.zip64Limit; const textEncoder = new TextEncoder(); const files: ZipEntry[] = []; let activeEntry: ZipEntry | null = null; @@ -107,21 +120,24 @@ class ZipFileSystem implements FileSystem { const writeEntryFooter = async (entry: ZipEntry) => { const { crc, sizeBytes } = entry; - offset += sizeBytes + 16; - // Plain (non-zip64) zip stores every size/offset as u32. Fail loud - // when the archive crosses 4 GiB rather than silently writing - // wrapped offsets (an unreadable file). - if (offset >= 0xffffffff) { - throw new Error( - 'Zip output exceeds 4 GiB and zip64 is not supported — write unbundled output instead.' - ); - } - const data = new Uint8Array(16); + // The local header is written before the size is known, so an + // entry that outgrows u32 gets the zip64 (8-byte size) data + // descriptor and its central directory record carries the zip64 + // sizes - the same scheme as Go's archive/zip and Java's + // ZipOutputStream. + const zip64 = sizeBytes >= zip64Limit; + const data = new Uint8Array(zip64 ? 24 : 16); const view = new DataView(data.buffer); view.setUint32(0, 0x08074b50, true); view.setUint32(4, crc.value(), true); - view.setUint32(8, sizeBytes, true); - view.setUint32(12, sizeBytes, true); + if (zip64) { + view.setBigUint64(8, BigInt(sizeBytes), true); + view.setBigUint64(16, BigInt(sizeBytes), true); + } else { + view.setUint32(8, sizeBytes, true); + view.setUint32(12, sizeBytes, true); + } + offset += sizeBytes + data.length; await writer.write(data); }; @@ -150,40 +166,91 @@ class ZipFileSystem implements FileSystem { activeEntry = null; } + // central directory starts where entry data ended + const cdOffset = offset; + // Write central directory records for (const file of files) { const { filename, crc, sizeBytes, headerOffset } = file; const nameLen = filename.length; - const cdr = new Uint8Array(46 + nameLen); + // zip64 extended information extra field: holds only the + // overflowed fields, in fixed order (sizes, then header offset) + const sizeZip64 = sizeBytes >= zip64Limit; + const offsetZip64 = headerOffset >= zip64Limit; + const zip64Len = (sizeZip64 ? 16 : 0) + (offsetZip64 ? 8 : 0); + const extraLen = zip64Len ? 4 + zip64Len : 0; + const version = zip64Len ? 45 : 20; + + const cdr = new Uint8Array(46 + nameLen + extraLen); const view = new DataView(cdr.buffer); view.setUint32(0, 0x02014b50, true); - view.setUint16(4, 20, true); - view.setUint16(6, 20, true); + view.setUint16(4, version, true); + view.setUint16(6, version, true); view.setUint16(8, 0x8 | 0x800, true); view.setUint16(10, 0, true); view.setUint16(12, dosTime, true); view.setUint16(14, dosDate, true); view.setUint32(16, crc.value(), true); - view.setUint32(20, sizeBytes, true); - view.setUint32(24, sizeBytes, true); + view.setUint32(20, sizeZip64 ? 0xffffffff : sizeBytes, true); + view.setUint32(24, sizeZip64 ? 0xffffffff : sizeBytes, true); view.setUint16(28, nameLen, true); - view.setUint32(42, headerOffset, true); + view.setUint16(30, extraLen, true); + view.setUint32(42, offsetZip64 ? 0xffffffff : headerOffset, true); cdr.set(filename, 46); + if (zip64Len) { + let p = 46 + nameLen; + view.setUint16(p, 0x0001, true); + view.setUint16(p + 2, zip64Len, true); + p += 4; + if (sizeZip64) { + view.setBigUint64(p, BigInt(sizeBytes), true); // uncompressed + view.setBigUint64(p + 8, BigInt(sizeBytes), true); // compressed + p += 16; + } + if (offsetZip64) { + view.setBigUint64(p, BigInt(headerOffset), true); + } + } + + offset += cdr.length; await writer.write(cdr); } - const filenameLength = files.reduce((tot, file) => tot + file.filename.length, 0); + const cdSize = offset - cdOffset; + const numFiles = files.length; + + // Write zip64 end of central directory record and locator ahead of + // the classic record when any of its fields overflow + if (numFiles >= 0xffff || cdSize >= zip64Limit || cdOffset >= zip64Limit) { + const zip64Eocd = new Uint8Array(56 + 20); + const zip64View = new DataView(zip64Eocd.buffer); + zip64View.setUint32(0, 0x06064b50, true); + zip64View.setBigUint64(4, 44n, true); // size of the rest of the record + zip64View.setUint16(12, 45, true); + zip64View.setUint16(14, 45, true); + zip64View.setBigUint64(24, BigInt(numFiles), true); + zip64View.setBigUint64(32, BigInt(numFiles), true); + zip64View.setBigUint64(40, BigInt(cdSize), true); + zip64View.setBigUint64(48, BigInt(cdOffset), true); + + // locator + zip64View.setUint32(56, 0x07064b50, true); + zip64View.setBigUint64(64, BigInt(offset), true); // zip64 record offset + zip64View.setUint32(72, 1, true); // total number of disks + + await writer.write(zip64Eocd); + } // Write end of central directory record const eocd = new Uint8Array(22); const eocdView = new DataView(eocd.buffer); eocdView.setUint32(0, 0x06054b50, true); - eocdView.setUint16(8, files.length, true); - eocdView.setUint16(10, files.length, true); - eocdView.setUint32(12, filenameLength + files.length * 46, true); - eocdView.setUint32(16, offset, true); // central directory starts where entry data ended + eocdView.setUint16(8, Math.min(numFiles, 0xffff), true); + eocdView.setUint16(10, Math.min(numFiles, 0xffff), true); + eocdView.setUint32(12, cdSize >= zip64Limit ? 0xffffffff : cdSize, true); + eocdView.setUint32(16, cdOffset >= zip64Limit ? 0xffffffff : cdOffset, true); await writer.write(eocd); diff --git a/test/zip-file-system.test.mjs b/test/zip-file-system.test.mjs new file mode 100644 index 00000000..2a38c36d --- /dev/null +++ b/test/zip-file-system.test.mjs @@ -0,0 +1,170 @@ +/** + * ZipFileSystem / ZipReadFileSystem zip64 support. + * + * Writing 4 GiB per test is too slow (the CRC runs in JS), so the zip64 cases + * lower ZipFileSystem.zip64Limit: fields at or above it take the zip64 form + * exactly as they would past 4 GiB. The 65535-entry limit is exercised for + * real. + */ + +import assert from 'node:assert'; +import { describe, it } from 'node:test'; + +import { MemoryFileSystem, MemoryReadFileSystem, ZipFileSystem, ZipReadFileSystem } from '../src/lib/index.js'; + +const encoder = new TextEncoder(); + +// entries: [name, parts[]] - each part is a separate write() into the entry +const writeZip = async (entries, zip64Limit = ZipFileSystem.zip64Limit) => { + const memFs = new MemoryFileSystem(); + const writer = await memFs.createWriter('out.zip'); + const defaultLimit = ZipFileSystem.zip64Limit; + ZipFileSystem.zip64Limit = zip64Limit; + let zipFs; + try { + zipFs = new ZipFileSystem(writer); + } finally { + ZipFileSystem.zip64Limit = defaultLimit; + } + for (const [name, parts] of entries) { + const entryWriter = await zipFs.createWriter(name); + for (const part of parts) { + await entryWriter.write(part); + } + await entryWriter.close(); + } + await zipFs.close(); + return memFs.results.get('out.zip'); +}; + +const readZip = async (bytes) => { + const rfs = new MemoryReadFileSystem(); + rfs.set('out.zip', bytes); + return new ZipReadFileSystem(await rfs.createSource('out.zip')); +}; + +const readEntry = async (zip, name) => { + const source = await zip.createSource(name); + const stream = source.read(); + const data = await stream.readAll(); + stream.close(); + return data; +}; + +const concat = (parts) => { + const out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0)); + let o = 0; + for (const p of parts) { + out.set(p, o); + o += p.length; + } + return out; +}; + +const bytesOf = (n, seed) => Uint8Array.from({ length: n }, (_, i) => (i * 31 + seed) & 0xff); + +const tail = (bytes) => { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const eocd = bytes.length - 22; + return { + view, + entryCount: view.getUint16(eocd + 10, true), + cdSize: view.getUint32(eocd + 12, true), + cdOffset: view.getUint32(eocd + 16, true), + hasZip64Locator: bytes.length >= 42 && view.getUint32(eocd - 20, true) === 0x07064b50 + }; +}; + +describe('ZipFileSystem zip64', () => { + it('keeps the classic layout below the limits', async () => { + const entries = [['meta.json', [encoder.encode('{"a":1}')]], ['data.bin', [bytesOf(1000, 1), bytesOf(24, 2)]]]; + const bytes = await writeZip(entries); + + // local header + data + 16-byte descriptor, 46-byte CD records, 22-byte EOCD + const nameBytes = entries.reduce((n, [name]) => n + name.length, 0); + const dataBytes = entries.reduce((n, [, parts]) => n + concat(parts).length, 0); + assert.strictEqual(bytes.length, entries.length * (30 + 16 + 46) + 2 * nameBytes + dataBytes + 22); + + const { view, entryCount, cdSize, cdOffset, hasZip64Locator } = tail(bytes); + assert.strictEqual(hasZip64Locator, false); + assert.strictEqual(entryCount, entries.length); + assert.strictEqual(cdOffset + cdSize, bytes.length - 22); + assert.strictEqual(view.getUint16(cdOffset + 6, true), 20, 'version needed stays 2.0'); + + const zip = await readZip(bytes); + for (const [name, parts] of entries) { + assert.deepStrictEqual(await readEntry(zip, name), concat(parts)); + } + zip.close(); + }); + + it('writes zip64 sizes, offsets and end record only where fields overflow', async () => { + // limit 100: 'a' stays classic, 'b' overflows its size (header at 97), + // 'c' overflows its header offset, and the central directory's offset + // and size both overflow + const a = bytesOf(50, 3); + const b = [bytesOf(90, 4), bytesOf(60, 5)]; + const c = bytesOf(10, 6); + const bytes = await writeZip([['a', [a]], ['b', b], ['c', [c]]], 100); + + const { view, entryCount, cdSize, cdOffset, hasZip64Locator } = tail(bytes); + assert.strictEqual(hasZip64Locator, true); + assert.strictEqual(entryCount, 3); + assert.strictEqual(cdSize, 0xffffffff); + assert.strictEqual(cdOffset, 0xffffffff); + + // 'b' ends in the 24-byte zip64 data descriptor, then 'c' begins + const bDescriptor = 97 + 31 + 150; + assert.strictEqual(view.getUint32(bDescriptor, true), 0x08074b50); + assert.strictEqual(Number(view.getBigUint64(bDescriptor + 8, true)), 150); + assert.strictEqual(view.getUint32(bDescriptor + 24, true), 0x04034b50); + + const zip = await readZip(bytes); + assert.deepStrictEqual(await zip.list(), ['a', 'b', 'c']); + const expected = { a: [0, 50], b: [97, 150], c: [302, 10] }; + for (const [name, [offset, size]] of Object.entries(expected)) { + const entry = await zip.getEntry(name); + assert.strictEqual(entry.offset, offset, `${name} header offset`); + assert.strictEqual(entry.uncompressedSize, size, `${name} size`); + assert.strictEqual(entry.compressedSize, size, `${name} compressed size`); + } + assert.deepStrictEqual(await readEntry(zip, 'a'), a); + assert.deepStrictEqual(await readEntry(zip, 'b'), concat(b)); + assert.deepStrictEqual(await readEntry(zip, 'c'), c); + zip.close(); + }); + + it('finds the zip64 locator behind a maximum-length archive comment', async () => { + const a = bytesOf(50, 3); + const b = bytesOf(150, 4); + const bytes = await writeZip([['a', [a]], ['b', [b]]], 100); + + // a 65535-byte comment puts the locator 65557 bytes from the end + const commented = new Uint8Array(bytes.length + 0xffff); + commented.set(bytes); + new DataView(commented.buffer).setUint16(bytes.length - 2, 0xffff, true); + + const zip = await readZip(commented); + assert.deepStrictEqual(await zip.list(), ['a', 'b']); + assert.deepStrictEqual(await readEntry(zip, 'a'), a); + assert.deepStrictEqual(await readEntry(zip, 'b'), b); + zip.close(); + }); + + it('writes the zip64 end record past 65534 entries', async () => { + const entries = Array.from({ length: 0xffff }, (_, i) => [`f${i}`, [bytesOf(i % 7, i)]]); + const bytes = await writeZip(entries); + + const { entryCount, cdOffset, hasZip64Locator } = tail(bytes); + assert.strictEqual(hasZip64Locator, true); + assert.strictEqual(entryCount, 0xffff); + assert.notStrictEqual(cdOffset, 0xffffffff, 'offset still fits, so stays classic'); + + const zip = await readZip(bytes); + const names = await zip.list(); + assert.strictEqual(names.length, 0xffff); + assert.strictEqual(names[0xfffe], 'f65534'); + assert.deepStrictEqual(await readEntry(zip, 'f65534'), bytesOf(65534 % 7, 65534)); + zip.close(); + }); +});