From 8f4a4f90c429af4af1fd784ec84c02775f752f54 Mon Sep 17 00:00:00 2001 From: Philipp Dunkel Date: Fri, 16 Oct 2026 21:32:52 +0200 Subject: [PATCH] zlib: add raw content read methods to ZipEntry zipEntry.rawContent was a getter that exposed the retained buffer of an in-memory entry and returned null for a file-backed or streaming one, so the still-compressed bytes of a member read through ZipFile could not be obtained at all. Replace it with rawContent(), rawContentSync() and rawContentIterator(), which mirror content(), contentSync() and contentIterator() but return the bytes as stored, without decompressing or verifying them. They work for every entry that has readable content, return caller-owned buffers from the buffered variants, and accept a maxSize bound on the compressed size. Signed-off-by: Philipp Dunkel --- doc/api/zlib.md | 59 +++++++++-- lib/internal/zip/entry.js | 54 +++++++++- test/parallel/test-zlib-zip-raw-content.js | 114 +++++++++++++++++++++ test/parallel/test-zlib-zip-vfs.js | 2 +- test/parallel/test-zlib-zip-writable.js | 7 +- test/parallel/test-zlib-zip.js | 2 +- 6 files changed, 221 insertions(+), 17 deletions(-) create mode 100644 test/parallel/test-zlib-zip-raw-content.js diff --git a/doc/api/zlib.md b/doc/api/zlib.md index 44caec51b3eb..7093c21abefe 100644 --- a/doc/api/zlib.md +++ b/doc/api/zlib.md @@ -1525,8 +1525,7 @@ which streaming never makes. Output is still bounded per chunk to the declared uncompressed size; pass `maxSize` to impose an explicit ceiling. For an in-memory entry stored without compression, the yielded chunks are -zero-copy views of the entry's retained content (see -[`zipEntry.rawContent`][]); do not mutate them. +zero-copy views of the entry's retained content; do not mutate them. The yielded chunks are **provisional until the iterator completes**. CRC-32 verification (and the final declared-size check) can only run once every byte @@ -1669,20 +1668,56 @@ The entry's raw name bytes, before any character decoding. Useful when the archive's names are in an encoding other than UTF-8 or CP437 and the caller wants to decode them itself. -### `zipEntry.rawContent` +### `zipEntry.rawContent([options])` -* Type: {Buffer|null} +* `options` {Object} + * `maxSize` {number} Reject content with more than this many compressed + bytes, before allocating anything. **Default:** + [`zlib.getMaxZipContentSize()`][]. +* Returns: {Promise} Fulfilled with a {Buffer} containing the entry's raw + content: the bytes as stored in the archive, still compressed if + [`zipEntry.compressed`][] is `true`. The buffer is a fresh copy that shares + no memory with the archive or with data the entry was created from. + +The content is neither decompressed nor checked against +[`zipEntry.crc32`][], which covers the uncompressed bytes. + +Throws an [`ERR_ZIP_ENTRY_TOO_LARGE`][] error if the entry's compressed size +exceeds `maxSize`, and an [`ERR_INVALID_STATE`][] error for a streaming entry +([`zlib.ZipEntry.createStream()`][]) whose content is not yet available. + +### `zipEntry.rawContentSync([options])` + + + +* `options` {Object} See [`zipEntry.rawContent()`][]. +* Returns: {Buffer} The entry's raw content. -The entry's raw (still compressed, if applicable) content when it is held in -memory, or `null` when there is no in-memory buffer to expose - for an entry -created with [`zlib.ZipEntry.createStream()`][], or a file-backed entry -returned by [`zipFile.get()`][], whose bytes are read from disk on demand -rather than retained. Use [`zipEntry.content()`][] or -[`zipEntry.contentIterator()`][] to read a file-backed entry. +The synchronous version of [`zipEntry.rawContent()`][]. + +### `zipEntry.rawContentIterator([options])` + + + +* `options` {Object} + * `maxSize` {number} Reject content with more than this many compressed + bytes, before reading anything. **Default:** no limit. +* Returns: {AsyncIterator} of {Buffer} chunks of the entry's raw content. + +Unlike [`zipEntry.rawContent()`][], this does not buffer the whole member in +memory. For a file-backed entry (one returned by [`zipFile.get()`][]) the +bytes are read from disk as the iterator is consumed and nothing is retained; +the entry is valid only while its `ZipFile` is open. For an in-memory entry, +the yielded chunk is a zero-copy view of the entry's retained content; do not +mutate it. ### `zipEntry.size` @@ -3425,15 +3460,17 @@ Create a Zstandard decompression transform. [`zipBuffer.comment`]: #zipbuffercomment [`zipBuffer.toBuffer()`]: #zipbuffertobufferoptions [`zipBuffer.toBufferSync()`]: #zipbuffertobuffersyncoptions +[`zipEntry.compressed`]: #zipentrycompressed [`zipEntry.content()`]: #zipentrycontentoptions [`zipEntry.contentIterator()`]: #zipentrycontentiteratoroptions [`zipEntry.contentSync()`]: #zipentrycontentsyncoptions +[`zipEntry.crc32`]: #zipentrycrc32 [`zipEntry.isDirectory`]: #zipentryisdirectory [`zipEntry.isSymlink`]: #zipentryissymlink [`zipEntry.modified`]: #zipentrymodified [`zipEntry.nameBuffer`]: #zipentrynamebuffer [`zipEntry.name`]: #zipentryname -[`zipEntry.rawContent`]: #zipentryrawcontent +[`zipEntry.rawContent()`]: #zipentryrawcontentoptions [`zipFile.add()`]: #zipfileaddfilename-data-options [`zipFile.addEntry()`]: #zipfileaddentryentry [`zipFile.close()`]: #zipfileclose diff --git a/lib/internal/zip/entry.js b/lib/internal/zip/entry.js index cfe54903a126..64337d3e57b9 100644 --- a/lib/internal/zip/entry.js +++ b/lib/internal/zip/entry.js @@ -222,7 +222,6 @@ class ZipEntry { // Whether the content is stored in compressed form, whatever the method. get compressed() { return this.method !== METHOD_STORE; } - get rawContent() { return this.#content; } get method() { return this.#meta ? this.#meta.method : this.#central.compressionMethod; } @@ -660,6 +659,59 @@ class ZipEntry { }, options); } + #assertRawWithin(maxSize) { + const declared = this.compressedSize; + if (declared > maxSize) { + throw new ERR_ZIP_ENTRY_TOO_LARGE( + `entry ${JSONStringify(this.name)} declares ${declared} compressed ` + + `bytes, exceeding the ${maxSize} byte limit`); + } + } + + /** + * Reads the entry's raw (still-compressed) content into a single `Buffer`, + * without decompressing or CRC-32-verifying it. + * @param {{ maxSize?: number }} [options] + * @returns {Promise} + */ + async rawContent(options) { + this.#assertRawWithin(options?.maxSize ?? getMaxZipContentSize()); + const handle = this.#fd; + if (handle === null) return Buffer.from(this.#inMemoryCompressed()); + if (handle.closing) { + throw new ERR_INVALID_STATE( + 'cannot read a ZipEntry after its backing ZipFile has been closed'); + } + handle.reads++; + try { + return await this.#compressedBytes(); + } finally { + endHandleRead(handle); + } + } + + /** + * The synchronous counterpart of `rawContent()`. + * @param {{ maxSize?: number }} [options] + * @returns {Buffer} + */ + rawContentSync(options) { + this.#assertRawWithin(options?.maxSize ?? getMaxZipContentSize()); + const compressed = this.#compressedBytesSync(); + return this.#fd === null ? Buffer.from(compressed) : compressed; + } + + /** + * Yields the entry's raw (still-compressed) content as a bounded-memory + * async iterator of `Buffer` chunks, without decompressing or verifying it. + * @param {{ maxSize?: number }} [options] + * @returns {AsyncGenerator} + */ + rawContentIterator(options) { + if (options?.maxSize !== undefined) this.#assertRawWithin(options.maxSize); + return this.#rawSource(); + } + // Synchronous serialization: emit the local file header (sec. 4.3.7) // followed by the raw (already-compressed) content bytes. Streaming // (source-backed) entries cannot be serialized this way. diff --git a/test/parallel/test-zlib-zip-raw-content.js b/test/parallel/test-zlib-zip-raw-content.js new file mode 100644 index 000000000000..f40c99198596 --- /dev/null +++ b/test/parallel/test-zlib-zip-raw-content.js @@ -0,0 +1,114 @@ +'use strict'; + +require('../common'); + +const assert = require('node:assert'); +const zlib = require('node:zlib'); +const fs = require('node:fs/promises'); +const path = require('node:path'); +const tmpdir = require('../common/tmpdir'); +const { test } = require('node:test'); + +tmpdir.refresh(); + +async function buildArchive(entries) { + const chunks = []; + for await (const chunk of zlib.createZipArchive(entries)) chunks.push(chunk); + return Buffer.concat(chunks); +} + +async function collect(iterator) { + const chunks = []; + for await (const chunk of iterator) chunks.push(chunk); + return Buffer.concat(chunks); +} + +const text = Buffer.from('raw content '.repeat(500)); + +function decode(entry, raw) { + switch (entry.method) { + case 0: return raw; + case 8: return zlib.inflateRawSync(raw); + case 93: return zlib.zstdDecompressSync(raw); + } + assert.fail(`unexpected method ${entry.method}`); +} + +async function assertRawReads(entry, expected) { + const raw = await entry.rawContent(); + assert.strictEqual(raw.length, entry.compressedSize); + assert.deepStrictEqual(decode(entry, raw), expected); + assert.deepStrictEqual(entry.rawContentSync(), raw); + assert.deepStrictEqual(await collect(entry.rawContentIterator()), raw); +} + +test('in-memory entries expose their raw bytes for every method', async () => { + for (const method of ['deflate', 'store', 'zstd']) { + const entry = await zlib.ZipEntry.create(`${method}.txt`, text, { method }); + assert.strictEqual(entry.compressed, method !== 'store'); + await assertRawReads(entry, text); + } +}); + +test('parsed and file-backed entries expose their raw bytes', async () => { + const archive = await buildArchive([ + await zlib.ZipEntry.create('d.txt', text), + await zlib.ZipEntry.create('s.txt', text, { method: 'store' }), + await zlib.ZipEntry.create('z.txt', text, { method: 'zstd' }), + await zlib.ZipEntry.create('empty.txt', Buffer.alloc(0)), + ]); + for (const entry of zlib.ZipEntry.read(archive)) { + await assertRawReads(entry, entry.name === 'empty.txt' ? Buffer.alloc(0) : text); + } + + const dir = await fs.mkdtemp(path.join(tmpdir.path, 'zlib-zip-raw-')); + const filePath = path.join(dir, 'archive.zip'); + await fs.writeFile(filePath, archive); + const zip = await zlib.ZipFile.open(filePath); + let entry; + try { + for (const name of zip.keys()) { + entry = await zip.get(name); + await assertRawReads(entry, name === 'empty.txt' ? Buffer.alloc(0) : text); + } + } finally { + await zip.close(); + } + await assert.rejects(entry.rawContent(), { code: 'ERR_INVALID_STATE' }); + assert.throws(() => entry.rawContentSync(), { code: 'ERR_INVALID_STATE' }); + await assert.rejects(collect(entry.rawContentIterator()), { code: 'ERR_INVALID_STATE' }); +}); + +test('rawContent() and rawContentSync() return caller-owned copies', async () => { + const data = Buffer.from('stored bytes'); + const entry = await zlib.ZipEntry.create('s.txt', data, { method: 'store' }); + const raw = await entry.rawContent(); + raw.fill(0); + const rawSync = entry.rawContentSync(); + assert.deepStrictEqual(rawSync, data); + rawSync.fill(0); + assert.deepStrictEqual(await entry.content(), data); + assert.deepStrictEqual(data, Buffer.from('stored bytes')); +}); + +test('maxSize bounds the compressed size', async () => { + const entry = await zlib.ZipEntry.create('s.txt', text, { method: 'store' }); + const limit = { maxSize: text.length - 1 }; + await assert.rejects(entry.rawContent(limit), { code: 'ERR_ZIP_ENTRY_TOO_LARGE' }); + assert.throws(() => entry.rawContentSync(limit), { code: 'ERR_ZIP_ENTRY_TOO_LARGE' }); + assert.throws(() => entry.rawContentIterator(limit), { code: 'ERR_ZIP_ENTRY_TOO_LARGE' }); + + const exact = { maxSize: text.length }; + assert.deepStrictEqual(await entry.rawContent(exact), text); + assert.deepStrictEqual(entry.rawContentSync(exact), text); + assert.deepStrictEqual(await collect(entry.rawContentIterator(exact)), text); +}); + +test('a pending streaming entry has no raw content yet', async () => { + async function* source() { yield text; } + const entry = zlib.ZipEntry.createStream('s.txt', source()); + await assert.rejects(entry.rawContent(), { code: 'ERR_INVALID_STATE' }); + assert.throws(() => entry.rawContentSync(), { code: 'ERR_INVALID_STATE' }); + assert.throws(() => entry.rawContentIterator(), { code: 'ERR_INVALID_STATE' }); + await entry[Symbol.asyncDispose](); +}); diff --git a/test/parallel/test-zlib-zip-vfs.js b/test/parallel/test-zlib-zip-vfs.js index dd85e7251b43..e9fae65e719f 100644 --- a/test/parallel/test-zlib-zip-vfs.js +++ b/test/parallel/test-zlib-zip-vfs.js @@ -49,7 +49,7 @@ test('ZipFile.open() reads a VFS-backed archive through an async virtual fd', as // A lazy, file-backed entry: it retains no content and reads straight // from the virtual fd on demand. const a = await zf.get('a.txt'); - assert.strictEqual(a.rawContent, null); + assert.strictEqual((await a.rawContent()).length, a.compressedSize); assert.strictEqual((await a.content()).toString(), 'hello vfs'); assert.strictEqual(await zf.get('a.txt'), a); // Cached handle identity diff --git a/test/parallel/test-zlib-zip-writable.js b/test/parallel/test-zlib-zip-writable.js index 9e036f62de31..5beb4751ad7b 100644 --- a/test/parallel/test-zlib-zip-writable.js +++ b/test/parallel/test-zlib-zip-writable.js @@ -150,6 +150,7 @@ test('ZipFile addEntry() promotes a spent streaming entry into a readable file-b ]); try { const zip = await zlib.ZipFile.open(filePath, { writable: true }); + let mem; try { const payload = 'streamed payload'.repeat(64); async function* source() { @@ -177,13 +178,13 @@ test('ZipFile addEntry() promotes a spent streaming entry into a readable file-b assert.strictEqual(copy.get('s.txt').crc32 >>> 0, streamEntry.crc32 >>> 0); // An in-memory entry added alongside keeps its own buffer (not promoted). - const mem = await zlib.ZipEntry.create('m.txt', Buffer.from('in memory')); + mem = await zlib.ZipEntry.create('m.txt', Buffer.from('in memory')); await zip.addEntry(mem); - assert.deepStrictEqual(mem.rawContent, mem.rawContent); // still a Buffer - assert.notStrictEqual(mem.rawContent, null); } finally { await zip.close(); } + // Still readable once the ZipFile is closed, so it was not promoted. + assert.strictEqual((await mem.content()).toString(), 'in memory'); // The bytes persisted correctly and re-open sees the streamed member. const reread = await zlib.ZipFile.open(filePath); diff --git a/test/parallel/test-zlib-zip.js b/test/parallel/test-zlib-zip.js index 910c17122d8f..a83159a72b83 100644 --- a/test/parallel/test-zlib-zip.js +++ b/test/parallel/test-zlib-zip.js @@ -83,7 +83,7 @@ test('explicit store option is honored even for compressible data', async () => const data = Buffer.from('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'); const entry = await zlib.ZipEntry.create('stored.txt', data, { method: 'store' }); assert.strictEqual(entry.method, 0); - assert.deepStrictEqual(entry.rawContent, data); + assert.deepStrictEqual(await entry.rawContent(), data); }); test('the zstd method compresses and round-trips through an archive', async () => {