diff --git a/src/http/routes/tus/lifecycle.ts b/src/http/routes/tus/lifecycle.ts index a5a928459..b32a0bc34 100644 --- a/src/http/routes/tus/lifecycle.ts +++ b/src/http/routes/tus/lifecycle.ts @@ -6,6 +6,7 @@ import { UploadId } from '@storage/protocols/tus' import { Storage } from '@storage/storage' import { Uploader } from '@storage/uploader' import { validateContentEncoding } from '@storage/validators/content-encoding' +import { detectMimeFromStoredObject, mimeFamiliesMatch } from '@storage/validators/file-signature' import { validateMimeType } from '@storage/validators/mime-type' import { DataStore, Metadata, Upload } from '@tus/server' import { randomUUID } from 'crypto' @@ -311,6 +312,83 @@ export async function onCreate( return { metadata } } +/** + * Peek the first bytes of the stored object and reject if the detected MIME + * is not consistent with the bucket's allowlist (and, when declared, the + * client-provided contentType). No-op when: + * - the storage layer has no `asSuperUser` or `backend.getObject` method + * (minimal mocks in unit tests exercise the no-policy fast path), + * - the bucket has no `allowed_mime_types` policy to enforce. + * + * On rejection, best-effort deletes the stored bytes before re-throwing. + * Any error during detection also triggers cleanup, so a non-verified + * object cannot persist on a MIME-restricted bucket. + */ +async function verifyUploadedMimeAgainstDeclared( + req: MultiPartRequest, + upload: Upload, + resourceId: UploadId, + s3Key: string +) { + const storage = req.upload.storage + if ( + typeof storage.asSuperUser !== 'function' || + typeof storage.backend?.getObject !== 'function' + ) { + return + } + + let bucket: { allowed_mime_types?: string[] | null } | undefined + try { + bucket = await storage.asSuperUser().findBucket(resourceId.bucket, 'id, allowed_mime_types') + } catch { + // If we cannot even look the bucket up, we cannot verify. Leaving the + // object stored matches pre-magic-byte behaviour; the standard upload + // leg has the same contract. + return + } + + if (!bucket?.allowed_mime_types?.length) { + return + } + + try { + const declaredContentType = upload.metadata?.contentType + const detected = await detectMimeFromStoredObject((range) => + storage.backend.getObject(storageS3Bucket, s3Key, resourceId.version, { range }) + ) + + let rejectWith: Error | undefined + + if (declaredContentType) { + if (detected && !mimeFamiliesMatch(detected, declaredContentType)) { + rejectWith = ERRORS.InvalidMimeType(`${detected} (declared as ${declaredContentType})`) + } + } else { + // No declared contentType on a MIME-restricted bucket: fail closed. + if (!detected) { + rejectWith = ERRORS.InvalidMimeType('unknown (no contentType declared)') + } else if (!bucket.allowed_mime_types.some((m) => mimeFamiliesMatch(detected, m))) { + rejectWith = ERRORS.InvalidMimeType(detected) + } + } + + if (rejectWith) throw rejectWith + } catch (err) { + // Best-effort cleanup: delete the stored bytes before re-throwing so a + // non-verified object cannot persist on a MIME-restricted bucket. + if (typeof storage.backend.deleteObject === 'function') { + await storage.backend.deleteObject(storageS3Bucket, s3Key, resourceId.version).catch(() => { + req.log?.warn( + { s3Key, version: resourceId.version }, + 'tus signature-verify cleanup: backend delete failed' + ) + }) + } + throw err + } +} + /** * Runs when the upload to the underline store is completed */ @@ -331,6 +409,37 @@ export async function onUploadFinish(rawReq: Request, upload: Upload) { resourceId.version ) + // Magic-byte MIME verification for the TUS leg (issue #639). + // + // onCreate above only validates the client-declared contentType against + // bucket.allowed_mime_types. Without this post-upload check, a caller + // can either declare `image/jpeg` on a JPEG-restricted bucket and + // stream GIF bytes through TUS, OR omit `contentType` entirely so the + // onCreate allowlist check is skipped — in both cases, the standard + // upload leg already rejects (uploader.ts), and TUS must mirror it so + // the two legs cannot diverge. + // + // We peek the first SIGNATURE_PEEK_BYTES of the already-stored object + // via a ranged read (no scan of the full upload), detect its MIME, and + // reject when: + // (a) the client declared a contentType but the detected family does + // not match it, OR + // (b) the client omitted contentType and the detected MIME is not in + // the bucket allowlist (fail-closed), OR + // (c) the client omitted contentType and the signature is unknown + // (fail-closed: no declaration to fall back on). + // + // On rejection we also delete the stored bytes so the bucket does not + // retain the policy-violating object. + // + // The whole block is a no-op when the storage layer lacks + // `asSuperUser` or `backend.getObject` — the two paths we need to run + // the signature check. The existing lifecycle unit tests use minimal + // storage mocks that omit both, and the pre-magic-byte onUploadFinish + // worked without them; keeping the no-op protects those tests from a + // regression caused by this PR's new dependency. + await verifyUploadedMimeAgainstDeclared(req, upload, resourceId, s3Key) + const uploader = new Uploader( req.upload.storage.backend, req.upload.storage.db, diff --git a/src/storage/uploader.ts b/src/storage/uploader.ts index 624205e32..b308b4b17 100644 --- a/src/storage/uploader.ts +++ b/src/storage/uploader.ts @@ -19,6 +19,7 @@ import { } from './events' import { getFileSizeLimit, isEmptyFolder } from './limits' import { metadataSize } from './schemas' +import { mimeFamiliesMatch, wrapWithSignatureDetection } from './validators/file-signature' import { validateMimeType } from './validators/mime-type' import { validateXRobotsTag } from './validators/x-robots-tag' @@ -687,6 +688,36 @@ export async function fileUploadFromRequest( throw ERRORS.NoContentProvided(new Error('Request stream closed before upload could begin')) } + // Magic-byte MIME verification (issue #639). + // + // `validateMimeType` above only checks what the client *claimed* in the + // Content-Type header / multipart field. A renamed `.gif` served as + // `image/jpeg` passes that check. When the bucket actually restricts + // mimetypes, we also verify the first bytes of the stream match the + // declared family. Buckets without restrictions stay on the fast path. + if ( + options.allowedMimeTypes && + options.allowedMimeTypes.length > 0 && + !isEmptyFolder(options.objectName) + ) { + // The sync `validate` closure runs the moment the MIME is known — the + // wrapper either passes bytes through unchanged (match or unknown) or + // destroys the output stream before releasing the final chunk / EOF + // downstream (mismatch). This closes the race the async `detected.then` + // pattern had on files smaller than SIGNATURE_PEEK_BYTES, where the + // detection would resolve only in the source `end` handler and the + // backend could commit before the destroy callback fired. Thanks + // @depthfirst-app for the small-file race catch on PR #1499. + const { stream: inspectedBody } = wrapWithSignatureDetection(body, { + validate: (actualMime) => { + if (!actualMime) return undefined // unknown signature: trust declared + if (mimeFamiliesMatch(actualMime, mimeType)) return undefined + return ERRORS.InvalidMimeType(`${actualMime} (declared as ${mimeType})`) + }, + }) + body = inspectedBody + } + return { body, mimeType, diff --git a/src/storage/validators/file-signature.test.ts b/src/storage/validators/file-signature.test.ts new file mode 100644 index 000000000..7cc3f1665 --- /dev/null +++ b/src/storage/validators/file-signature.test.ts @@ -0,0 +1,408 @@ +import { Readable } from 'stream' +import { + detectMimeFromBuffer, + detectMimeFromStoredObject, + mimeFamiliesMatch, + SIGNATURE_PEEK_BYTES, + wrapWithSignatureDetection, +} from './file-signature' + +const hex = (s: string): Buffer => Buffer.from(s.replace(/\s+/g, ''), 'hex') + +// Minimal real headers for the detectors we support. Each buffer is just the +// signature prefix padded with a few extra NULs so the detector sees "enough". +const SAMPLES = { + jpeg: hex('ffd8ffe000104a46494600'), + png: hex('89504e470d0a1a0a0000000d49484452'), + gif87a: Buffer.from('GIF87a\x00\x00\x00\x00', 'binary'), + gif89a: Buffer.from('GIF89a\x00\x00\x00\x00', 'binary'), + webp: Buffer.concat([Buffer.from('RIFF'), hex('18000000'), Buffer.from('WEBP')]), + bmp: Buffer.from('BM' + '\x00'.repeat(10), 'binary'), + tiffLE: hex('49492a00'), + tiffBE: hex('4d4d002a'), + pdfBuf: Buffer.from('%PDF-1.4\n%EOF', 'binary'), + mp4: Buffer.concat([ + Buffer.from([0, 0, 0, 0x20]), // box size + Buffer.from('ftypmp42'), // ftyp + brand + Buffer.from([0, 0, 0, 0]), // minor + Buffer.from('isommp42'), // compat brands + ]), + heic: Buffer.concat([Buffer.from([0, 0, 0, 0x20]), Buffer.from('ftypheic'), Buffer.alloc(16)]), + avif: Buffer.concat([Buffer.from([0, 0, 0, 0x20]), Buffer.from('ftypavif'), Buffer.alloc(16)]), + mp3: Buffer.concat([Buffer.from('ID3'), Buffer.alloc(10)]), + wav: Buffer.concat([Buffer.from('RIFF'), hex('18000000'), Buffer.from('WAVE')]), + ogg: Buffer.from('OggS' + '\x00'.repeat(10), 'binary'), + flac: Buffer.from('fLaC' + '\x00'.repeat(10), 'binary'), + // EBML header + DocType element: 0x4282 0x84 "webm" places this sample inside + // the WebM subset of EBML containers rather than the general Matroska space. + webm: Buffer.concat([hex('1a45dfa3'), Buffer.alloc(10), hex('428284'), Buffer.from('webm')]), + // Matroska EBML container that is NOT WebM — must NOT be labelled video/webm. + matroska: Buffer.concat([ + hex('1a45dfa3'), + Buffer.alloc(10), + hex('428288'), + Buffer.from('matroska'), + ]), + // Pure EBML header with no DocType — must NOT be labelled video/webm. + ebmlBareHeader: Buffer.concat([hex('1a45dfa3'), Buffer.alloc(60)]), + zip: Buffer.concat([hex('504b0304'), Buffer.alloc(4)]), + rar: Buffer.concat([hex('526172211a0700'), Buffer.alloc(2)]), + gzip: Buffer.concat([hex('1f8b0800'), Buffer.alloc(4)]), + exe: Buffer.from('MZ' + '\x00'.repeat(60), 'binary'), + elf: Buffer.concat([hex('7f454c46'), Buffer.alloc(4)]), + ole: Buffer.concat([hex('d0cf11e0a1b11ae1'), Buffer.alloc(8)]), +} + +describe('detectMimeFromBuffer', () => { + it.each([ + [SAMPLES.jpeg, 'image/jpeg'], + [SAMPLES.png, 'image/png'], + [SAMPLES.gif87a, 'image/gif'], + [SAMPLES.gif89a, 'image/gif'], + [SAMPLES.webp, 'image/webp'], + [SAMPLES.bmp, 'image/bmp'], + [SAMPLES.tiffLE, 'image/tiff'], + [SAMPLES.tiffBE, 'image/tiff'], + [SAMPLES.pdfBuf, 'application/pdf'], + [SAMPLES.mp4, 'video/mp4'], + [SAMPLES.heic, 'image/heic'], + [SAMPLES.avif, 'image/avif'], + [SAMPLES.mp3, 'audio/mpeg'], + [SAMPLES.wav, 'audio/wav'], + [SAMPLES.ogg, 'audio/ogg'], + [SAMPLES.flac, 'audio/flac'], + [SAMPLES.webm, 'video/webm'], + [SAMPLES.zip, 'application/zip'], + [SAMPLES.rar, 'application/x-rar-compressed'], + [SAMPLES.gzip, 'application/gzip'], + [SAMPLES.exe, 'application/vnd.microsoft.portable-executable'], + [SAMPLES.elf, 'application/x-elf'], + [SAMPLES.ole, 'application/x-cfb'], + ])('detects %#', (buf, expected) => { + expect(detectMimeFromBuffer(buf as Buffer)).toBe(expected) + }) + + it('returns undefined for empty or tiny buffers', () => { + expect(detectMimeFromBuffer(Buffer.alloc(0))).toBeUndefined() + expect(detectMimeFromBuffer(Buffer.from([0xff]))).toBeUndefined() + }) + + it('returns undefined for unknown signatures', () => { + // Plain ASCII text has no magic — detector must NOT guess. + expect(detectMimeFromBuffer(Buffer.from('hello world'))).toBeUndefined() + expect(detectMimeFromBuffer(hex('deadbeefcafe'))).toBeUndefined() + }) + + it('distinguishes RIFF/WEBP from RIFF/WAVE (same prefix)', () => { + expect(detectMimeFromBuffer(SAMPLES.webp)).toBe('image/webp') + expect(detectMimeFromBuffer(SAMPLES.wav)).toBe('audio/wav') + }) + + it('distinguishes WebM from Matroska (shared EBML header)', () => { + // Only a DocType of exactly "webm" should be labelled video/webm — the + // same EBML magic can equally start a Matroska file. + expect(detectMimeFromBuffer(SAMPLES.webm)).toBe('video/webm') + expect(detectMimeFromBuffer(SAMPLES.matroska)).toBeUndefined() + expect(detectMimeFromBuffer(SAMPLES.ebmlBareHeader)).toBeUndefined() + }) + + it('does NOT match RIFF with an unknown form chunk (defense against partial-prefix tricks)', () => { + // RIFF without WEBP or WAVE suffix — may still match x-cfb if that entry has no refine? No, + // x-cfb has a distinct 8-byte prefix. So the correct expectation is undefined. + const riffUnknown = Buffer.concat([Buffer.from('RIFF'), hex('18000000'), Buffer.from('XXXX')]) + const detected = detectMimeFromBuffer(riffUnknown) + expect(detected).toBeUndefined() + }) + + it('does NOT match ftyp with an unknown brand', () => { + const ftypUnknown = Buffer.concat([ + Buffer.from([0, 0, 0, 0x20]), + Buffer.from('ftypZZZZ'), + Buffer.alloc(16), + ]) + expect(detectMimeFromBuffer(ftypUnknown)).toBeUndefined() + }) + + it('SIGNATURE_PEEK_BYTES is at least 32 bytes (enough for every entry)', () => { + expect(SIGNATURE_PEEK_BYTES).toBeGreaterThanOrEqual(32) + }) +}) + +describe('mimeFamiliesMatch', () => { + it('accepts identical mimes (case + whitespace tolerant)', () => { + expect(mimeFamiliesMatch('image/png', 'image/png')).toBe(true) + expect(mimeFamiliesMatch('Image/PNG', 'image/png')).toBe(true) + expect(mimeFamiliesMatch('image/jpeg; charset=binary', 'image/jpeg')).toBe(true) + }) + + it('treats image/jpg as an alias of image/jpeg', () => { + expect(mimeFamiliesMatch('image/jpeg', 'image/jpg')).toBe(true) + expect(mimeFamiliesMatch('image/jpg', 'image/jpeg')).toBe(true) + expect(mimeFamiliesMatch('image/pjpeg', 'image/jpeg')).toBe(true) + }) + + it('accepts zip signature for OOXML office MIMEs (docx/xlsx/pptx)', () => { + expect( + mimeFamiliesMatch( + 'application/zip', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + ) + ).toBe(true) + expect( + mimeFamiliesMatch( + 'application/zip', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + ) + ).toBe(true) + }) + + it('accepts x-cfb (OLE2) for legacy office MIMEs (doc/xls/ppt)', () => { + expect(mimeFamiliesMatch('application/x-cfb', 'application/msword')).toBe(true) + expect(mimeFamiliesMatch('application/x-cfb', 'application/vnd.ms-excel')).toBe(true) + expect(mimeFamiliesMatch('application/x-cfb', 'application/vnd.ms-powerpoint')).toBe(true) + }) + + // Regression for depthfirst-app's catch on #1499: a client cannot declare a + // legacy OLE2 MIME (.doc/.xls/.ppt) and smuggle an arbitrary ZIP through + // the magic-byte check. Only OOXML (`application/vnd.openxmlformats-...`) + // is physically ZIP. + it.each([ + 'application/vnd.ms-excel', + 'application/vnd.ms-powerpoint', + 'application/msword', + 'application/vnd.ms-outlook', + ])('rejects zip signature declared as legacy OLE2 MIME %s', (declared) => { + expect(mimeFamiliesMatch('application/zip', declared)).toBe(false) + }) + + // Regression for depthfirst-app's low-severity catch on #1499: an + // allowlist that said "any vnd.ms-* is OLE2" would let an OLE2 payload + // be stored under a non-OLE2 MIME (fontobject, cab-compressed, xpsdocument). + // These are explicitly NOT in the OLE2 whitelist. + it.each([ + 'application/vnd.ms-fontobject', + 'application/vnd.ms-cab-compressed', + 'application/vnd.ms-xpsdocument', + 'application/vnd.ms-htmlhelp', + ])('rejects OLE2 payload declared as non-OLE2 vnd.ms-* subtype %s', (declared) => { + expect(mimeFamiliesMatch('application/x-cfb', declared)).toBe(false) + }) + + // Guard the ISO BMFF ftyp brand lookup — the big audio/video switch is a + // high-value surface for coverage because a wrong brand would let an + // attacker's payload wear a mismatched media MIME. + it.each([ + { brand: 'mp42', expected: 'video/mp4' }, + { brand: 'isom', expected: 'video/mp4' }, + { brand: 'avc1', expected: 'video/mp4' }, + { brand: 'M4V ', expected: 'video/x-m4v' }, + { brand: 'M4A ', expected: 'audio/mp4' }, + { brand: 'qt ', expected: 'video/quicktime' }, + { brand: '3gp4', expected: 'video/3gpp' }, + { brand: 'heic', expected: 'image/heic' }, + { brand: 'hevm', expected: 'image/heif' }, + { brand: 'avif', expected: 'image/avif' }, + ])('maps ISO BMFF ftyp brand $brand to $expected', ({ brand, expected }) => { + const buf = Buffer.from([ + 0x00, + 0x00, + 0x00, + 0x20, + 0x66, + 0x74, + 0x79, + 0x70, + ...Buffer.from(brand, 'ascii'), + 0x00, + 0x00, + 0x00, + 0x00, + ]) + expect(detectMimeFromBuffer(buf)).toBe(expected) + }) + + it('returns undefined for an unknown ISO BMFF ftyp brand', () => { + const buf = Buffer.from([ + 0x00, + 0x00, + 0x00, + 0x20, + 0x66, + 0x74, + 0x79, + 0x70, + ...Buffer.from('xxxx', 'ascii'), + 0x00, + 0x00, + 0x00, + 0x00, + ]) + expect(detectMimeFromBuffer(buf)).toBeUndefined() + }) + + it('WAV requires the "WAVE" tag at bytes 8..12 (RIFF alone is not enough)', () => { + const good = Buffer.from([ + 0x52, 0x49, 0x46, 0x46, 0x00, 0x00, 0x00, 0x00, 0x57, 0x41, 0x56, 0x45, + ]) + expect(detectMimeFromBuffer(good)).toBe('audio/wav') + const bad = Buffer.from([ + 0x52, 0x49, 0x46, 0x46, 0x00, 0x00, 0x00, 0x00, 0x41, 0x56, 0x49, 0x20, + ]) + expect(detectMimeFromBuffer(bad)).toBeUndefined() + }) + + // The security-critical cases: a mismatch here means the uploader will reject. + it.each([ + ['image/gif', 'image/jpeg'], // renamed .gif → .jpg (exactly the #639 report) + ['application/pdf', 'image/png'], + ['application/vnd.microsoft.portable-executable', 'application/pdf'], + ['image/jpeg', 'text/html'], // polyglot JPG-HTML: declared text, actually JPG + ['application/x-elf', 'image/jpeg'], + ['application/zip', 'image/png'], // zip claiming to be an image + ['application/gzip', 'image/jpeg'], + ['video/mp4', 'image/jpeg'], + ])('rejects mismatch detected=%s declared=%s', (detected, declared) => { + expect(mimeFamiliesMatch(detected, declared)).toBe(false) + }) +}) + +describe('wrapWithSignatureDetection sync validate (closes small-file race)', () => { + // The async `detected.then` pattern previously used in uploader.ts had a + // race on files under SIGNATURE_PEEK_BYTES: the detector resolved only in + // the source `end` handler, at which point out.end() had already fired to + // the backend. The sync `validate` closure runs the moment the MIME is + // known and either lets bytes through or destroys the output stream + // before EOF reaches the backend. + + const wrap = wrapWithSignatureDetection + + function collect(stream: NodeJS.ReadableStream): Promise<{ error?: Error; bytes: Buffer }> { + return new Promise((resolve) => { + const parts: Buffer[] = [] + stream.on('data', (c: Buffer) => parts.push(c)) + stream.on('end', () => resolve({ bytes: Buffer.concat(parts) })) + stream.on('error', (error) => resolve({ error, bytes: Buffer.concat(parts) })) + }) + } + + it('destroys the output stream synchronously when validate returns an error on a small file', async () => { + const gif = Buffer.concat([Buffer.from('GIF89a', 'ascii'), Buffer.alloc(32)]) + const source = Readable.from([gif]) + const { stream } = wrap(source, { + validate: (mime) => + mime === 'image/gif' ? new Error('mismatch: gif declared as jpeg') : undefined, + }) + const result = await collect(stream) + expect(result.error).toBeDefined() + expect(result.error?.message).toContain('mismatch') + }) + + it('passes the stream through unchanged when validate returns undefined', async () => { + const pngHeader = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) + const payload = Buffer.concat([pngHeader, Buffer.alloc(64, 0xaa)]) + const source = Readable.from([payload]) + const { stream } = wrap(source, { validate: () => undefined }) + const result = await collect(stream) + expect(result.error).toBeUndefined() + expect(result.bytes.length).toBe(payload.length) + expect(result.bytes.subarray(0, 8).equals(pngHeader)).toBe(true) + }) + + it('does not call validate when no options are passed (backwards compat)', async () => { + const payload = Buffer.from('any 12 bytes') + const source = Readable.from([payload]) + const { stream, detected } = wrap(source) + const [result, mime] = await Promise.all([collect(stream), detected]) + expect(result.error).toBeUndefined() + expect(result.bytes.length).toBe(payload.length) + expect(mime).toBeUndefined() // unknown signature + }) +}) + +describe('detectMimeFromStoredObject (TUS leg)', () => { + // The helper is called post-upload with a `getRange(range)` closure that + // normally forwards to the backend's `getObject`. These tests stub that + // closure so we can assert the range request, the stop-early behaviour, + // and the handling of an unknown-signature body — the three things the + // TUS-leg caller in `src/http/routes/tus/lifecycle.ts` relies on. + + function streamFromBuffer(buf: Buffer): Readable { + return Readable.from([buf]) + } + + it('requests a bytes=0-N range covering SIGNATURE_PEEK_BYTES', async () => { + let seen: string | undefined + const pngHeader = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) + await detectMimeFromStoredObject(async (range) => { + seen = range + return { body: streamFromBuffer(pngHeader) } + }) + expect(seen).toBe(`bytes=0-${SIGNATURE_PEEK_BYTES - 1}`) + }) + + it('detects the MIME of a PNG stored object', async () => { + const pngHeader = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) + const detected = await detectMimeFromStoredObject(async () => ({ + body: streamFromBuffer(pngHeader), + })) + expect(detected).toBe('image/png') + }) + + it('catches a GIF stored under a JPEG-declared TUS upload (the #639 case)', async () => { + const gifHeader = Buffer.concat([Buffer.from('GIF89a', 'ascii'), Buffer.alloc(64)]) + const detected = await detectMimeFromStoredObject(async () => ({ + body: streamFromBuffer(gifHeader), + })) + expect(detected).toBe('image/gif') + // The TUS caller gates on mimeFamiliesMatch; verify it would reject. + expect(mimeFamiliesMatch(detected!, 'image/jpeg')).toBe(false) + }) + + it('returns undefined when the body is empty', async () => { + const detected = await detectMimeFromStoredObject(async () => ({ + body: streamFromBuffer(Buffer.alloc(0)), + })) + expect(detected).toBeUndefined() + }) + + it('returns undefined when the signature is unknown', async () => { + const unknown = Buffer.alloc(16, 0x00) + const detected = await detectMimeFromStoredObject(async () => ({ + body: streamFromBuffer(unknown), + })) + expect(detected).toBeUndefined() + }) + + it('accepts Uint8Array chunks (S3 adapter emits them)', async () => { + const pngHeader = Uint8Array.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) + const stream = Readable.from([pngHeader]) + const detected = await detectMimeFromStoredObject(async () => ({ body: stream })) + expect(detected).toBe('image/png') + }) + + it('accepts string chunks (adapters that decode UTF-8 by default)', async () => { + // Buffer.from(string, 'binary') round-trips byte-for-byte through utf-8 + // only for ASCII, so we build a chunk whose magic bytes happen to live + // in the ASCII range — the WAV RIFF/WAVE signature. + const wavHeader = 'RIFF\x00\x00\x00\x00WAVE' + const stream = Readable.from([wavHeader]) + const detected = await detectMimeFromStoredObject(async () => ({ body: stream })) + expect(detected).toBe('audio/wav') + }) + + it('stops reading once SIGNATURE_PEEK_BYTES have arrived (bounded cost)', async () => { + // A single chunk larger than the peek window would still work because the + // ranged read is the real guard — but we exercise the loop's break by + // feeding many small chunks and checking the function completes without + // consuming them all. + const pngHeader = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) + const stuffer = Buffer.alloc(SIGNATURE_PEEK_BYTES) + const chunks = [pngHeader, stuffer, stuffer, stuffer, stuffer] + const stream = Readable.from(chunks) + const detected = await detectMimeFromStoredObject(async () => ({ body: stream })) + expect(detected).toBe('image/png') + // We cannot easily assert "only 2 chunks consumed" because Readable.from + // buffers aggressively; the behavioural contract is the detection result, + // and the ranged read on the real backend keeps bytes-transferred bounded. + }) +}) diff --git a/src/storage/validators/file-signature.ts b/src/storage/validators/file-signature.ts new file mode 100644 index 000000000..be8099de5 --- /dev/null +++ b/src/storage/validators/file-signature.ts @@ -0,0 +1,453 @@ +import { PassThrough, Readable } from 'stream' + +// Magic-byte (file-signature) detection for upload MIME validation. +// +// Why this file exists +// -------------------- +// `src/storage/validators/mime-type.ts` only compares the client-declared MIME +// string against the bucket's `allowed_mime_types` list. A caller that renames +// a `.gif` to `.jpg` and sends `Content-Type: image/jpeg` passes that check +// even though the actual bytes are a GIF (issue #639, user flogesell, 2024). +// +// This helper peeks the first chunk of the upload stream and derives a MIME +// from the signature bytes. The uploader can then reject uploads whose actual +// content does not match the restriction, while keeping buckets with no +// restrictions on the fast path. +// +// Design notes +// ------------ +// 1. Pure helper — no imports from logger / config / database. Keeps it reusable +// from any upload code path (standard / S3 / resumable) without pulling in +// circular dependencies. +// 2. Returns `undefined` for signatures it cannot identify. The uploader +// treats that as "not enough evidence" and does NOT reject on it; otherwise +// legitimate novel formats would start failing. We only refuse when we are +// certain the actual MIME is in a different family than the declared one. +// 3. 4 KiB peek is enough for every entry in SIGNATURES below while still +// being cheap to buffer in a PassThrough. + +/** Minimum bytes we want before attempting detection. */ +export const SIGNATURE_PEEK_BYTES = 4096 + +type SignatureEntry = { + mime: string + /** Either a prefix (array of ints), or a `{ offset, bytes }` for formats with a prefix such as ISO BMFF. */ + signatures: Array + /** + * Optional post-match check. Used for container formats (e.g. ISO BMFF) + * where the magic prefix is shared across many sub-types. + */ + refine?: (buf: Buffer) => string | undefined +} + +/** + * ISO Base Media File Format major-brand lookup. Called when the ftyp box is + * detected (`...ftyp` at offset 4). The brand lives at bytes 8..12. + */ +function refineIsoBmff(buf: Buffer): string | undefined { + if (buf.length < 12) return undefined + const brand = buf.subarray(8, 12).toString('ascii') + // Covers the brands bucket owners actually restrict on. + switch (brand) { + case 'mp42': + case 'mp41': + case 'isom': + case 'avc1': + case 'iso2': + case 'iso4': + case 'iso5': + case 'iso6': + case 'dash': + return 'video/mp4' + case 'M4V ': + return 'video/x-m4v' + case 'M4A ': + return 'audio/mp4' + case 'qt ': + return 'video/quicktime' + case '3gp4': + case '3gp5': + case '3gp6': + return 'video/3gpp' + case 'heic': + case 'heix': + case 'hevc': + return 'image/heic' + case 'heis': + case 'hevm': + case 'hevs': + case 'mif1': + case 'msf1': + return 'image/heif' + case 'avif': + case 'avis': + return 'image/avif' + default: + return undefined + } +} + +/** + * Ordered list — first match wins. Order matters only for ambiguous short prefixes + * (none at the moment), but keeping it stable makes diffs easier to review. + */ +const SIGNATURES: SignatureEntry[] = [ + // ─── Images ───────────────────────────────────────────────────────── + { mime: 'image/jpeg', signatures: [[0xff, 0xd8, 0xff]] }, + { mime: 'image/png', signatures: [[0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]] }, + // GIF87a and GIF89a + { + mime: 'image/gif', + signatures: [ + [0x47, 0x49, 0x46, 0x38, 0x37, 0x61], + [0x47, 0x49, 0x46, 0x38, 0x39, 0x61], + ], + }, + // WebP: "RIFF????WEBP" + { + mime: 'image/webp', + signatures: [[0x52, 0x49, 0x46, 0x46]], + refine: (buf) => + buf.length >= 12 && buf.subarray(8, 12).toString('ascii') === 'WEBP' + ? 'image/webp' + : undefined, + }, + { mime: 'image/bmp', signatures: [[0x42, 0x4d]] }, + // TIFF little-endian + big-endian + { + mime: 'image/tiff', + signatures: [ + [0x49, 0x49, 0x2a, 0x00], + [0x4d, 0x4d, 0x00, 0x2a], + ], + }, + { mime: 'image/x-icon', signatures: [[0x00, 0x00, 0x01, 0x00]] }, + + // ─── Documents ────────────────────────────────────────────────────── + { mime: 'application/pdf', signatures: [[0x25, 0x50, 0x44, 0x46, 0x2d]] }, + // Office legacy OLE2 (.doc, .xls, .ppt) + { mime: 'application/x-cfb', signatures: [[0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]] }, + + // ─── Archives ─────────────────────────────────────────────────────── + // ZIP (also covers .docx / .xlsx / .zip / .jar / .apk — detected as zip here; + // callers restricting a specific office format should include the "application/zip" + // family in `allowedMimeTypes` too). + { + mime: 'application/zip', + signatures: [ + [0x50, 0x4b, 0x03, 0x04], + [0x50, 0x4b, 0x05, 0x06], + [0x50, 0x4b, 0x07, 0x08], + ], + }, + { + mime: 'application/x-rar-compressed', + signatures: [ + [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00], + [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00], + ], + }, + { mime: 'application/x-7z-compressed', signatures: [[0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]] }, + { mime: 'application/gzip', signatures: [[0x1f, 0x8b]] }, + + // ─── Audio / Video ────────────────────────────────────────────────── + // ISO BMFF: ftyp box at bytes 4..8 + { + mime: 'video/mp4', + signatures: [{ offset: 4, bytes: [0x66, 0x74, 0x79, 0x70] }], + refine: refineIsoBmff, + }, + // MP3 ID3v2 header + { mime: 'audio/mpeg', signatures: [[0x49, 0x44, 0x33]] }, + // WAV: "RIFF????WAVE" + { + mime: 'audio/wav', + signatures: [[0x52, 0x49, 0x46, 0x46]], + refine: (buf) => + buf.length >= 12 && buf.subarray(8, 12).toString('ascii') === 'WAVE' + ? 'audio/wav' + : undefined, + }, + // OGG container + { mime: 'audio/ogg', signatures: [[0x4f, 0x67, 0x67, 0x53]] }, + // FLAC + { mime: 'audio/flac', signatures: [[0x66, 0x4c, 0x61, 0x43]] }, + // WebM — EBML header is shared with Matroska; DocType element distinguishes. + // Without this refine, any .mkv could pass as video/webm by magic bytes alone. + { + mime: 'video/webm', + signatures: [[0x1a, 0x45, 0xdf, 0xa3]], + refine: (buf) => { + // DocType element: 0x42 0x82 + VInt-length (0x84 = len 4) + "webm" (4 bytes). + // Scan a bounded window at the head of the file (DocType lives inside the + // EBML header, typically within the first 32-64 bytes). + const window = Math.min(buf.length, 128) + const webmTag = Buffer.from([0x42, 0x82, 0x84, 0x77, 0x65, 0x62, 0x6d]) + for (let i = 4; i <= window - webmTag.length; i++) { + if (buf.subarray(i, i + webmTag.length).equals(webmTag)) return 'video/webm' + } + // No "webm" DocType in header — reject rather than mislabel Matroska/malformed. + return undefined + }, + }, + + // ─── Executables (security-relevant: callers restricting images must reject these) ─ + // PE / EXE + { mime: 'application/vnd.microsoft.portable-executable', signatures: [[0x4d, 0x5a]] }, + // ELF + { mime: 'application/x-elf', signatures: [[0x7f, 0x45, 0x4c, 0x46]] }, + // Mach-O (both endian + fat) + { + mime: 'application/x-mach-binary', + signatures: [ + [0xfe, 0xed, 0xfa, 0xce], + [0xfe, 0xed, 0xfa, 0xcf], + [0xce, 0xfa, 0xed, 0xfe], + [0xcf, 0xfa, 0xed, 0xfe], + [0xca, 0xfe, 0xba, 0xbe], + ], + }, +] + +function matchesSignature( + buf: Buffer, + sig: number[] | { offset: number; bytes: number[] } +): boolean { + if (Array.isArray(sig)) { + if (buf.length < sig.length) return false + for (let i = 0; i < sig.length; i++) if (buf[i] !== sig[i]) return false + return true + } + if (buf.length < sig.offset + sig.bytes.length) return false + for (let i = 0; i < sig.bytes.length; i++) { + if (buf[sig.offset + i] !== sig.bytes[i]) return false + } + return true +} + +/** + * Detect a MIME type from the first bytes of a file. + * Returns `undefined` when no signature in our list matches. + */ +export function detectMimeFromBuffer(buf: Buffer): string | undefined { + if (!buf || buf.length < 2) return undefined + for (const entry of SIGNATURES) { + for (const sig of entry.signatures) { + if (!matchesSignature(buf, sig)) continue + if (entry.refine) { + // Refine is authoritative for container formats (RIFF/WEBP vs RIFF/WAVE, + // ISO BMFF ftyp brands, …). If it cannot confirm the sub-type, we do NOT + // fall back to the entry's default MIME — otherwise RIFF/XXXX would be + // misreported as WEBP, defeating the signature check entirely. + const refined = entry.refine(buf) + if (refined) return refined + continue + } + return entry.mime + } + } + return undefined +} + +/** + * Returns true when the detected and declared MIME belong to the same high-level + * family. We keep the comparison loose on purpose: + * * `image/jpeg` === `image/jpg` (common client typo) + * * `application/zip` is accepted ONLY for OOXML Office types (`.docx`, + * `.xlsx`, `.pptx` — Office Open XML files which are physically ZIP). + * Legacy Office (`.doc`, `.xls`, `.ppt`) are OLE2/CFB containers, NOT + * ZIP, so accepting ZIP for them would let a caller declare + * `application/vnd.ms-excel` and upload an arbitrary archive — thanks + * @depthfirst-app for the catch on PR #1499. + * The uploader uses this to decide whether a magic-byte mismatch is worth + * refusing the upload for. + */ +export function mimeFamiliesMatch(detected: string, declared: string): boolean { + const d = normalizeMime(detected) + const c = normalizeMime(declared) + if (d === c) return true + + // Treat 'image/jpg' as an alias of 'image/jpeg'. + const JPEG = new Set(['image/jpeg', 'image/jpg', 'image/pjpeg']) + if (JPEG.has(d) && JPEG.has(c)) return true + + // OOXML office documents are physically ZIP; accept zip signature only for + // those specific subtypes. See the maintainer note above for why the legacy + // `application/vnd.ms-*` and `application/msword` types are intentionally + // excluded from this list — those are OLE2/CFB, not ZIP, and are handled by + // the `application/x-cfb` branch below. + if (d === 'application/zip' && c.startsWith('application/vnd.openxmlformats-officedocument')) { + return true + } + + // Legacy OLE2 formats (.doc, .xls, .ppt, Visio, Project, Outlook .msg) + // share the x-cfb signature. The list is explicit so unrelated `vnd.ms-*` + // subtypes like `vnd.ms-fontobject`, `vnd.ms-cab-compressed` and + // `vnd.ms-xpsdocument` — which are NOT OLE2 — do not match. Thanks + // @depthfirst-app for the second catch on PR #1499. + if (d === 'application/x-cfb' && OLE2_CFB_MIME_TYPES.has(c)) { + return true + } + + return false +} + +const OLE2_CFB_MIME_TYPES = new Set([ + 'application/msword', + 'application/vnd.ms-excel', + 'application/vnd.ms-powerpoint', + 'application/vnd.ms-project', + 'application/vnd.ms-outlook', + 'application/vnd.ms-works', + 'application/visio', + 'application/vnd.visio', + 'application/x-ole-storage', +]) + +function normalizeMime(s: string): string { + return (s || '').trim().toLowerCase().split(';')[0].trim() +} + +/** + * Peek the first {@link SIGNATURE_PEEK_BYTES} of an already-stored object and + * derive a MIME from the signature bytes. Used by the resumable (TUS) upload + * leg at commit time — unlike {@link wrapWithSignatureDetection}, which runs + * on the incoming stream for the standard upload leg. + * + * Returns `undefined` when the signature is unknown (same contract as + * {@link detectMimeFromBuffer}) — the caller keeps the declared MIME in that + * case. + * + * The ranged read is driven by the caller: pass a `getRange` closure that + * calls the backend's `getObject` with a `bytes=0-N` Range header. We stop + * reading once we have {@link SIGNATURE_PEEK_BYTES}, regardless of final + * object size. + */ +// Supertype of every body shape returned by StorageBackendAdapter.getObject +// (Buffer, Blob, Readable, ReadableStream) plus the stream-only shape used +// in tests. We type `getRange`'s result loosely so the TUS caller can pass +// the backend result directly without re-shaping it; the body-type branches +// below pick the right read path. +export async function detectMimeFromStoredObject( + getRange: (range: string) => Promise<{ + body?: unknown + }> +): Promise { + const resp = await getRange(`bytes=0-${SIGNATURE_PEEK_BYTES - 1}`) + const body = resp.body + if (body == null) return undefined + + // Fast path for the Buffer and Blob return shapes used by backend + // adapters that already fully-buffer the object (file backend in dev, + // test backend). No streaming needed. + if (Buffer.isBuffer(body)) { + return body.length ? detectMimeFromBuffer(body) : undefined + } + if (typeof Blob !== 'undefined' && body instanceof Blob) { + const buf = Buffer.from(await body.arrayBuffer()) + return buf.length ? detectMimeFromBuffer(buf) : undefined + } + + // Stream path: S3-shaped adapters emit a Readable or Web ReadableStream. + // Iterate chunks and stop once we have SIGNATURE_PEEK_BYTES. The + // AsyncIterable cast is safe for both Node Readable (iterator built-in) + // and Web ReadableStream (iterator in Node 18+ via Symbol.asyncIterator). + const chunks: Buffer[] = [] + let total = 0 + const iterable = body as AsyncIterable + for await (const chunk of iterable) { + const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk) + chunks.push(buf) + total += buf.length + if (total >= SIGNATURE_PEEK_BYTES) break + } + if (!chunks.length) return undefined + return detectMimeFromBuffer(Buffer.concat(chunks, total)) +} + +/** + * Wrap an upload stream so the first chunk is held long enough to detect the + * actual MIME, then released unchanged. The caller gets: + * - `stream`: a drop-in replacement for the original body stream + * - `detected`: a promise that resolves to the detected MIME (or undefined) + * + * We buffer up to SIGNATURE_PEEK_BYTES, run the detector once, then flush and + * stop buffering. If the source emits less than that before 'end', we still + * detect on whatever we have. No chunks are swallowed. + * + * Optional `options.validate(mime)` is run synchronously the moment the MIME + * is known. If it returns an Error, the wrapper destroys the output stream + * before releasing the final chunk / EOF downstream, so the backend cannot + * commit a mismatched object even on files smaller than SIGNATURE_PEEK_BYTES + * where detection resolves only in the source `end` handler. The async + * `detected` promise is still fulfilled for observers that want to log the + * detection outcome, but validation enforcement must go through + * `options.validate` to be race-safe. Thanks @depthfirst-app for the + * small-file race catch on PR #1499. + */ +export function wrapWithSignatureDetection( + source: Readable, + options?: { validate?: (mime: string | undefined) => Error | undefined } +): { + stream: PassThrough + detected: Promise +} { + const out = new PassThrough() + let peek: Buffer[] = [] + let peekLen = 0 + let flushed = false + let terminalError: Error | undefined + let detectedResolve!: (v: string | undefined) => void + const detected = new Promise((r) => { + detectedResolve = r + }) + + const flush = (): string | undefined => { + if (flushed) return undefined + flushed = true + const buf = peek.length === 1 ? peek[0] : Buffer.concat(peek, peekLen) + const mime = detectMimeFromBuffer(buf) + detectedResolve(mime) + // We did NOT consume the bytes — they are already written to `out` below + // when the detector resolves mid-stream (peek >= SIGNATURE_PEEK_BYTES). + peek = [] + if (options?.validate) { + const err = options.validate(mime) + if (err) terminalError = err + } + return mime + } + + source.on('data', (chunk: Buffer) => { + if (terminalError) return + if (!flushed) { + peek.push(chunk) + peekLen += chunk.length + if (peekLen >= SIGNATURE_PEEK_BYTES) flush() + } + if (terminalError) { + out.destroy(terminalError) + source.destroy() + return + } + if (!out.write(chunk)) source.pause() + }) + out.on('drain', () => source.resume()) + source.on('end', () => { + if (!flushed) flush() + if (terminalError) { + out.destroy(terminalError) + } else { + out.end() + } + }) + source.on('error', (err) => { + if (!flushed) { + flushed = true + detectedResolve(undefined) + } + out.destroy(err) + }) + + return { stream: out, detected } +}