diff --git a/apps/nest-node-probe/correctness/zlib-streams.ts b/apps/nest-node-probe/correctness/zlib-streams.ts new file mode 100644 index 0000000..5323a9e --- /dev/null +++ b/apps/nest-node-probe/correctness/zlib-streams.ts @@ -0,0 +1,136 @@ +import { Buffer } from 'node:buffer' +import { PassThrough, type Transform } from 'node:stream' +import { + brotliDecompressSync, + createBrotliDecompress, + createDeflate, + createDeflateRaw, + createGunzip, + createGzip, + createInflate, + createInflateRaw, + createUnzip, + gunzipSync +} from 'node:zlib' + +const log = (line: string): void => { + console.log(line) +} + +const TEXT = 'The quick brown fox jumps over the lazy dog. '.repeat(40) +const GZIP = '1f8b08000000000000130bc94855282ccd4cce56482aca2fcf5348cbaf50c82acd2d2856c82f4b2d5228c94855c849acaa5448c94fd7530819553caa7854f1a8e251c5a38a87976200e6c3956408070000' +const DEFLATE = '789c0bc94855282ccd4cce56482aca2fcf5348cbaf50c82acd2d2856c82f4b2d5228c94855c849acaa5448c94fd7530819553caa7854f1a8e251c5a38a87976200880a8637' +const DEFLATE_RAW = '0bc94855282ccd4cce56482aca2fcf5348cbaf50c82acd2d2856c82f4b2d5228c94855c849acaa5448c94fd7530819553caa7854f1a8e251c5a38a87976200' +const BROTLI = '1b0707882c0e78d3d0955d9710bb172ba9cad092cc8cad415ce6f236c8199e9e0a7b830d387048206f24bf41a715ce1c1e27aa2938c2a9749f0603' +const GZIP_MEMBERS = '1f8b08000000000000134bcb2c2a2e510000fc7af11c060000001f8b08000000000000132b4e4dcecf4b010069111fb606000000' + +interface Case { + name: string + decoder: string + input: Buffer + expected: string +} + +function decoder(kind: string): Transform { + switch (kind) { + case 'gunzip': + return createGunzip() + case 'inflate': + return createInflate() + case 'inflateRaw': + return createInflateRaw() + case 'unzip': + return createUnzip() + case 'brotli': + return createBrotliDecompress() + default: + throw new Error(`unknown decoder ${kind}`) + } +} + +const cases: Case[] = [ + { name: 'gunzip', decoder: 'gunzip', input: Buffer.from(GZIP, 'hex'), expected: TEXT }, + { name: 'inflate', decoder: 'inflate', input: Buffer.from(DEFLATE, 'hex'), expected: TEXT }, + { name: 'inflateRaw', decoder: 'inflateRaw', input: Buffer.from(DEFLATE_RAW, 'hex'), expected: TEXT }, + { name: 'unzip gzip', decoder: 'unzip', input: Buffer.from(GZIP, 'hex'), expected: TEXT }, + { name: 'unzip zlib', decoder: 'unzip', input: Buffer.from(DEFLATE, 'hex'), expected: TEXT }, + { name: 'brotli', decoder: 'brotli', input: Buffer.from(BROTLI, 'hex'), expected: TEXT }, + { name: 'gunzip members', decoder: 'gunzip', input: Buffer.from(GZIP_MEMBERS, 'hex'), expected: 'first second' }, + { name: 'gunzip garbage', decoder: 'gunzip', input: Buffer.from('0102030405', 'hex'), expected: '' }, + { name: 'gunzip truncated', decoder: 'gunzip', input: Buffer.from(GZIP, 'hex').subarray(0, 40), expected: '' }, + { name: 'inflate garbage', decoder: 'inflate', input: Buffer.from('0102030405', 'hex'), expected: '' }, + { name: 'brotli truncated', decoder: 'brotli', input: Buffer.from(BROTLI, 'hex').subarray(0, 40), expected: '' }, + { name: 'brotli corrupt', decoder: 'brotli', input: Buffer.from('ff', 'hex'), expected: '' }, + { name: 'brotli empty', decoder: 'brotli', input: Buffer.from('3b', 'hex'), expected: '' } +] + +let at = 0 + +function runDecoder(item: Case, done: () => void): void { + const source = new PassThrough() + const stream = decoder(item.decoder) + const chunks: Buffer[] = [] + stream.on('data', (chunk: Buffer) => { + chunks.push(chunk) + }) + stream.on('end', () => { + const output = Buffer.concat(chunks) + log(`${item.name}: end ${output.length} ${output.toString() === item.expected ? 'match' : 'MISMATCH'}`) + done() + }) + stream.on('error', (error: Error) => { + log(`${item.name}: error ${error.message}`) + done() + }) + source.pipe(stream) + const third = Math.max(1, Math.floor(item.input.length / 3)) + source.write(item.input.subarray(0, third)) + source.write(item.input.subarray(third, 2 * third)) + source.end(item.input.subarray(2 * third)) +} + +function roundTrip(name: string, encoder: Transform, decode: Transform, done: () => void): void { + const chunks: Buffer[] = [] + encoder.pipe(decode) + decode.on('data', (chunk: Buffer) => { + chunks.push(chunk) + }) + decode.on('end', () => { + const output = Buffer.concat(chunks) + log(`${name}: end ${output.length} ${output.toString() === TEXT ? 'match' : 'MISMATCH'}`) + done() + }) + encoder.write(Buffer.from(TEXT.slice(0, 600))) + encoder.write(Buffer.from(TEXT.slice(600, 1200))) + encoder.end(Buffer.from(TEXT.slice(1200))) +} + +function next(): void { + if (at < cases.length) { + const item = cases[at] + at++ + runDecoder(item, next) + return + } + if (at === cases.length) { + at++ + roundTrip('gzip round trip', createGzip(), createGunzip(), next) + return + } + if (at === cases.length + 1) { + at++ + roundTrip('deflate round trip', createDeflate({ level: 9 }), createInflate(), next) + return + } + if (at === cases.length + 2) { + at++ + roundTrip('deflateRaw round trip', createDeflateRaw(), createInflateRaw(), next) + return + } + log(`brotliDecompressSync: ${brotliDecompressSync(Buffer.from(BROTLI, 'hex')).toString() === TEXT ? 'match' : 'MISMATCH'}`) + log(`gunzipSync: ${gunzipSync(Buffer.from(GZIP, 'hex')).toString() === TEXT ? 'match' : 'MISMATCH'}`) + log('done') +} + +log('start') +next() diff --git a/plugin/index.mjs b/plugin/index.mjs index 46183df..642f9f2 100644 --- a/plugin/index.mjs +++ b/plugin/index.mjs @@ -288,7 +288,8 @@ const nodeOnlyHostFunctions = new Map([ ['__gea_node_crypto_timing_safe_equal', 'gea::node::crypto::timingSafeEqual'], ['__gea_node_crypto_get_fips', 'gea::node::crypto::getFips'], ['__gea_node_zlib_inflate', 'gea::node::zlib::inflate'], - ['__gea_node_zlib_deflate', 'gea::node::zlib::deflate'] + ['__gea_node_zlib_deflate', 'gea::node::zlib::deflate'], + ['__gea_node_brotli_decompress', 'gea::node::brotli::decompress'] ]) /** @@ -442,7 +443,9 @@ const hostPreambles = () => ? [runtimeHeader, '#include "gea_node_crypto.hpp"'] : spelling.startsWith('gea::node::zlib::') ? [runtimeHeader, '#include "gea_node_zlib.hpp"'] - : [runtimeHeader] + : spelling.startsWith('gea::node::brotli::') + ? [runtimeHeader, '#include "gea_node_brotli.hpp"'] + : [runtimeHeader] ]) ) diff --git a/runtime/node/gea_node_brotli.hpp b/runtime/node/gea_node_brotli.hpp new file mode 100644 index 0000000..22c9751 --- /dev/null +++ b/runtime/node/gea_node_brotli.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include "gea_runtime.h" + +#include + +#include +#include +#include +#include + +namespace gea::node::brotli { +using Bytes = gea::TypedArray; + +inline std::vector decompress(const Bytes& bytes) { + BrotliDecoderState* state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); + if (state == nullptr) gea::host::throwRuntimeError("Error", "Cannot allocate memory"); + static const std::uint8_t empty = 0; + const std::uint8_t* nextIn = bytes.size() == 0 ? &empty : reinterpret_cast(bytes.data()); + std::size_t availableIn = bytes.size(); + std::vector output; + constexpr std::size_t chunk = 64 * 1024; + BrotliDecoderResult result = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT; + while (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { + const std::size_t used = output.size(); + output.resize(used + chunk); + std::uint8_t* nextOut = output.data() + used; + std::size_t availableOut = chunk; + result = BrotliDecoderDecompressStream(state, &availableIn, &nextIn, &availableOut, &nextOut, nullptr); + output.resize(used + (chunk - availableOut)); + } + if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { + BrotliDecoderDestroyInstance(state); + gea::host::throwRuntimeError("Error", "unexpected end of file"); + } + if (result != BROTLI_DECODER_RESULT_SUCCESS) { + BrotliDecoderDestroyInstance(state); + gea::host::throwRuntimeError("Error", "Decompression failed"); + } + BrotliDecoderDestroyInstance(state); + return output; +} + +} diff --git a/runtime/node/generated/facades/zlib.ts b/runtime/node/generated/facades/zlib.ts index e8e4775..02cf683 100644 --- a/runtime/node/generated/facades/zlib.ts +++ b/runtime/node/generated/facades/zlib.ts @@ -1,7 +1,7 @@ // Generated from the pinned Node 24 declaration inventory. Do not edit. // Canonical builtin: node:zlib import { nodeNotImplemented } from "../../not-implemented" -export { constants, deflate, deflateRawSync, deflateSync, gunzip, gunzipSync, gzip, gzipSync, inflate, inflateRawSync, inflateSync, unzipSync } from "../../zlib" +export { brotliDecompress, BrotliDecompress, brotliDecompressSync, constants, createBrotliDecompress, createDeflate, createDeflateRaw, createGunzip, createGzip, createInflate, createInflateRaw, createUnzip, deflate, Deflate, DeflateRaw, deflateRawSync, deflateSync, gunzip, Gunzip, gunzipSync, gzip, Gzip, gzipSync, inflate, Inflate, InflateRaw, inflateRawSync, inflateSync, Unzip, unzipSync } from "../../zlib" export function brotliCompress(...args: unknown[]): never { void args @@ -20,23 +20,6 @@ export function brotliCompressSync(...args: unknown[]): never { return nodeNotImplemented("node:zlib", "brotliCompressSync") } -export function brotliDecompress(...args: unknown[]): never { - void args - return nodeNotImplemented("node:zlib", "brotliDecompress") -} - -export class BrotliDecompress { - constructor(...args: unknown[]) { - void args - nodeNotImplemented("node:zlib", "BrotliDecompress") - } -} - -export function brotliDecompressSync(...args: unknown[]): never { - void args - return nodeNotImplemented("node:zlib", "brotliDecompressSync") -} - export function crc32(...args: unknown[]): never { void args return nodeNotImplemented("node:zlib", "crc32") @@ -47,46 +30,6 @@ export function createBrotliCompress(...args: unknown[]): never { return nodeNotImplemented("node:zlib", "createBrotliCompress") } -export function createBrotliDecompress(...args: unknown[]): never { - void args - return nodeNotImplemented("node:zlib", "createBrotliDecompress") -} - -export function createDeflate(...args: unknown[]): never { - void args - return nodeNotImplemented("node:zlib", "createDeflate") -} - -export function createDeflateRaw(...args: unknown[]): never { - void args - return nodeNotImplemented("node:zlib", "createDeflateRaw") -} - -export function createGunzip(...args: unknown[]): never { - void args - return nodeNotImplemented("node:zlib", "createGunzip") -} - -export function createGzip(...args: unknown[]): never { - void args - return nodeNotImplemented("node:zlib", "createGzip") -} - -export function createInflate(...args: unknown[]): never { - void args - return nodeNotImplemented("node:zlib", "createInflate") -} - -export function createInflateRaw(...args: unknown[]): never { - void args - return nodeNotImplemented("node:zlib", "createInflateRaw") -} - -export function createUnzip(...args: unknown[]): never { - void args - return nodeNotImplemented("node:zlib", "createUnzip") -} - export function createZstdCompress(...args: unknown[]): never { void args return nodeNotImplemented("node:zlib", "createZstdCompress") @@ -97,70 +40,21 @@ export function createZstdDecompress(...args: unknown[]): never { return nodeNotImplemented("node:zlib", "createZstdDecompress") } -export class Deflate { - constructor(...args: unknown[]) { - void args - nodeNotImplemented("node:zlib", "Deflate") - } -} - export function deflateRaw(...args: unknown[]): never { void args return nodeNotImplemented("node:zlib", "deflateRaw") } -export class DeflateRaw { - constructor(...args: unknown[]) { - void args - nodeNotImplemented("node:zlib", "DeflateRaw") - } -} - -export class Gunzip { - constructor(...args: unknown[]) { - void args - nodeNotImplemented("node:zlib", "Gunzip") - } -} - -export class Gzip { - constructor(...args: unknown[]) { - void args - nodeNotImplemented("node:zlib", "Gzip") - } -} - -export class Inflate { - constructor(...args: unknown[]) { - void args - nodeNotImplemented("node:zlib", "Inflate") - } -} - export function inflateRaw(...args: unknown[]): never { void args return nodeNotImplemented("node:zlib", "inflateRaw") } -export class InflateRaw { - constructor(...args: unknown[]) { - void args - nodeNotImplemented("node:zlib", "InflateRaw") - } -} - export function unzip(...args: unknown[]): never { void args return nodeNotImplemented("node:zlib", "unzip") } -export class Unzip { - constructor(...args: unknown[]) { - void args - nodeNotImplemented("node:zlib", "Unzip") - } -} - export const Z_ASCII: unknown = undefined export const Z_BEST_COMPRESSION: unknown = undefined diff --git a/runtime/node/generated/node24-compatibility.json b/runtime/node/generated/node24-compatibility.json index 22cc2ae..5cfbcd4 100644 --- a/runtime/node/generated/node24-compatibility.json +++ b/runtime/node/generated/node24-compatibility.json @@ -20918,25 +20918,38 @@ { "name": "brotliDecompress", "kind": "function", - "status": "stubbed" + "status": "partial", + "overlay": "zlib.ts", + "supportedCallSignatures": [ + "(input: InputType, callback: ZlibCallback): void" + ] }, { "name": "BrotliDecompress", "kind": "class", - "status": "stubbed", + "status": "partial", + "overlay": "zlib.ts", + "supportedConstructSignatures": [ + "(): BrotliDecompress" + ], "members": [ { "name": "", "kind": "Constructor", "scope": "instance", - "status": "stubbed" + "status": "partial", + "overlayKind": "Constructor" } ] }, { "name": "brotliDecompressSync", "kind": "function", - "status": "stubbed" + "status": "partial", + "overlay": "zlib.ts", + "supportedCallSignatures": [ + "(input: InputType): Buffer" + ] }, { "name": "constants", @@ -20957,42 +20970,74 @@ { "name": "createBrotliDecompress", "kind": "function", - "status": "stubbed" + "status": "partial", + "overlay": "zlib.ts", + "supportedCallSignatures": [ + "(): BrotliDecompress" + ] }, { "name": "createDeflate", "kind": "function", - "status": "stubbed" + "status": "partial", + "overlay": "zlib.ts", + "supportedCallSignatures": [ + "(options?: ZlibOptions): Deflate" + ] }, { "name": "createDeflateRaw", "kind": "function", - "status": "stubbed" + "status": "partial", + "overlay": "zlib.ts", + "supportedCallSignatures": [ + "(options?: ZlibOptions): DeflateRaw" + ] }, { "name": "createGunzip", "kind": "function", - "status": "stubbed" + "status": "partial", + "overlay": "zlib.ts", + "supportedCallSignatures": [ + "(): Gunzip" + ] }, { "name": "createGzip", "kind": "function", - "status": "stubbed" + "status": "partial", + "overlay": "zlib.ts", + "supportedCallSignatures": [ + "(options?: ZlibOptions): Gzip" + ] }, { "name": "createInflate", "kind": "function", - "status": "stubbed" + "status": "partial", + "overlay": "zlib.ts", + "supportedCallSignatures": [ + "(): Inflate" + ] }, { "name": "createInflateRaw", "kind": "function", - "status": "stubbed" + "status": "partial", + "overlay": "zlib.ts", + "supportedCallSignatures": [ + "(): InflateRaw" + ] }, { "name": "createUnzip", "kind": "function", - "status": "stubbed" + "status": "partial", + "overlay": "zlib.ts", + "supportedCallSignatures": [ + "(): Unzip" + ] }, { "name": "createZstdCompress", @@ -21016,13 +21061,18 @@ { "name": "Deflate", "kind": "class", - "status": "stubbed", + "status": "partial", + "overlay": "zlib.ts", + "supportedConstructSignatures": [ + "(options?: ZlibOptions): Deflate" + ], "members": [ { "name": "", "kind": "Constructor", "scope": "instance", - "status": "stubbed" + "status": "partial", + "overlayKind": "Constructor" } ] }, @@ -21034,13 +21084,18 @@ { "name": "DeflateRaw", "kind": "class", - "status": "stubbed", + "status": "partial", + "overlay": "zlib.ts", + "supportedConstructSignatures": [ + "(options?: ZlibOptions): DeflateRaw" + ], "members": [ { "name": "", "kind": "Constructor", "scope": "instance", - "status": "stubbed" + "status": "partial", + "overlayKind": "Constructor" } ] }, @@ -21074,13 +21129,18 @@ { "name": "Gunzip", "kind": "class", - "status": "stubbed", + "status": "partial", + "overlay": "zlib.ts", + "supportedConstructSignatures": [ + "(): Gunzip" + ], "members": [ { "name": "", "kind": "Constructor", "scope": "instance", - "status": "stubbed" + "status": "partial", + "overlayKind": "Constructor" } ] }, @@ -21105,13 +21165,18 @@ { "name": "Gzip", "kind": "class", - "status": "stubbed", + "status": "partial", + "overlay": "zlib.ts", + "supportedConstructSignatures": [ + "(options?: ZlibOptions): Gzip" + ], "members": [ { "name": "", "kind": "Constructor", "scope": "instance", - "status": "stubbed" + "status": "partial", + "overlayKind": "Constructor" } ] }, @@ -21136,13 +21201,18 @@ { "name": "Inflate", "kind": "class", - "status": "stubbed", + "status": "partial", + "overlay": "zlib.ts", + "supportedConstructSignatures": [ + "(): Inflate" + ], "members": [ { "name": "", "kind": "Constructor", "scope": "instance", - "status": "stubbed" + "status": "partial", + "overlayKind": "Constructor" } ] }, @@ -21154,13 +21224,18 @@ { "name": "InflateRaw", "kind": "class", - "status": "stubbed", + "status": "partial", + "overlay": "zlib.ts", + "supportedConstructSignatures": [ + "(): InflateRaw" + ], "members": [ { "name": "", "kind": "Constructor", "scope": "instance", - "status": "stubbed" + "status": "partial", + "overlayKind": "Constructor" } ] }, @@ -21190,13 +21265,18 @@ { "name": "Unzip", "kind": "class", - "status": "stubbed", + "status": "partial", + "overlay": "zlib.ts", + "supportedConstructSignatures": [ + "(): Unzip" + ], "members": [ { "name": "", "kind": "Constructor", "scope": "instance", - "status": "stubbed" + "status": "partial", + "overlayKind": "Constructor" } ] }, diff --git a/runtime/node/zlib.ts b/runtime/node/zlib.ts index 390178d..6d7c102 100644 --- a/runtime/node/zlib.ts +++ b/runtime/node/zlib.ts @@ -7,6 +7,7 @@ // tables both hand over a whole Buffer and take a whole Buffer back. import { Buffer } from './buffer' +import { Transform, type TransformCallback } from './stream' // The driver always supplies Buffer here. Keeping the alias exact to that // source use prevents the Buffer | Uint8Array API union from degrading the @@ -23,6 +24,8 @@ export type ZlibCallback = (error: Error | null, result: Buffer) => void declare function __gea_node_zlib_inflate(input: Buffer, windowBits: number): Buffer /** @gea-host-inert */ declare function __gea_node_zlib_deflate(input: Buffer, windowBits: number, level: number): Buffer +/** @gea-host-inert */ +declare function __gea_node_brotli_decompress(input: Buffer): Buffer // zlib's `windowBits` for each of node's framings. const ZLIB = 15 @@ -95,3 +98,130 @@ export function gunzip(input: InputType, callback: ZlibCallback): void { export function gzip(input: InputType, options: ZlibOptions, callback: ZlibCallback): void { settle(callback, () => gzipSync(input, options)) } + +export function brotliDecompressSync(input: InputType): Buffer { + return __gea_node_brotli_decompress(input) +} + +export function brotliDecompress(input: InputType, callback: ZlibCallback): void { + settle(callback, () => brotliDecompressSync(input)) +} + +const MODE_INFLATE = 0 +const MODE_DEFLATE = 1 +const MODE_BROTLI_DECOMPRESS = 2 + +class ZlibBase extends Transform { + private chunks_: Buffer[] + private mode_: number + private windowBits_: number + private level_: number + + constructor(mode: number, windowBits: number, level: number) { + super() + this.chunks_ = [] + this.mode_ = mode + this.windowBits_ = windowBits + this.level_ = level + } + + override _transform(chunk: Buffer, _encoding: unknown, callback: TransformCallback): void { + this.chunks_.push(chunk) + callback(null) + } + + override _flush(callback: TransformCallback): void { + const input = Buffer.concat(this.chunks_) + this.chunks_ = [] + let output: Buffer + try { + if (this.mode_ === MODE_BROTLI_DECOMPRESS) output = __gea_node_brotli_decompress(input) + else if (this.mode_ === MODE_DEFLATE) output = __gea_node_zlib_deflate(input, this.windowBits_, this.level_) + else output = __gea_node_zlib_inflate(input, this.windowBits_) + } catch (error) { + callback(error instanceof Error ? error : new Error(String(error))) + return + } + callback(null, output) + } +} + +export class Gunzip extends ZlibBase { + constructor() { + super(MODE_INFLATE, GZIP, 0) + } +} + +export class Inflate extends ZlibBase { + constructor() { + super(MODE_INFLATE, ZLIB, 0) + } +} + +export class InflateRaw extends ZlibBase { + constructor() { + super(MODE_INFLATE, RAW, 0) + } +} + +export class Unzip extends ZlibBase { + constructor() { + super(MODE_INFLATE, DETECT, 0) + } +} + +export class BrotliDecompress extends ZlibBase { + constructor() { + super(MODE_BROTLI_DECOMPRESS, 0, 0) + } +} + +export class Gzip extends ZlibBase { + constructor(options?: ZlibOptions) { + super(MODE_DEFLATE, GZIP, levelOf(options)) + } +} + +export class Deflate extends ZlibBase { + constructor(options?: ZlibOptions) { + super(MODE_DEFLATE, ZLIB, levelOf(options)) + } +} + +export class DeflateRaw extends ZlibBase { + constructor(options?: ZlibOptions) { + super(MODE_DEFLATE, RAW, levelOf(options)) + } +} + +export function createGunzip(): Gunzip { + return new Gunzip() +} + +export function createInflate(): Inflate { + return new Inflate() +} + +export function createInflateRaw(): InflateRaw { + return new InflateRaw() +} + +export function createUnzip(): Unzip { + return new Unzip() +} + +export function createBrotliDecompress(): BrotliDecompress { + return new BrotliDecompress() +} + +export function createGzip(options?: ZlibOptions): Gzip { + return new Gzip(options) +} + +export function createDeflate(options?: ZlibOptions): Deflate { + return new Deflate(options) +} + +export function createDeflateRaw(options?: ZlibOptions): DeflateRaw { + return new DeflateRaw(options) +} diff --git a/scripts/build.mjs b/scripts/build.mjs index 00771eb..29e1b63 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -43,7 +43,7 @@ // leaves a record. Exit codes: 1 nothing emitted, 2 link failed. import fs from 'node:fs' import path from 'node:path' -import { spawn, spawnSync } from 'node:child_process' +import { execFileSync, spawn, spawnSync } from 'node:child_process' import crypto from 'node:crypto' import os from 'node:os' import { fileURLToPath } from 'node:url' @@ -896,7 +896,7 @@ await buildExecutable(emittedUnits, displayPath) // or worse silently mis-read, by a unit compiled with a different language // dialect, optimization level or visibility -- with `-x c++-header` in place of // `-c`. `pch` on a compile phase is that PCH's `{ file, codegen }`. -function cxxFlags({ phase, cxxIsClang, inputs, output, needsCrypto, needsZlib, pch = null }) { +function cxxFlags({ phase, cxxIsClang, inputs, output, needsCrypto, needsZlib, needsBrotli, pch = null }) { const compiling = phase !== 'link' const linking = phase === 'all' || phase === 'link' const runtimeDir = path.join(repo, 'runtime') @@ -1014,6 +1014,7 @@ function cxxFlags({ phase, cxxIsClang, inputs, output, needsCrypto, needsZlib, p // the compiler and node-compat runtime roots. ...(compiling ? [`-I${compilerRuntimeDir}`, `-I${runtimeDir}`, `-I${path.join(runtimeDir, 'node')}`, `-I${outDir}`] : []), ...(compiling && needsZlib && windowsLibraries() ? [`-I${path.join(windowsLibraries(), 'include')}`] : []), + ...(compiling && needsBrotli ? brotliIncludeFlags() : []), // Escape hatch for one-off builds that need a flag the pipeline does not // model -- `-g` for a profile or an allocation census, `-fsanitize=...` for // a leak hunt. Whitespace-separated, appended last so it can override an @@ -1079,6 +1080,7 @@ function cxxFlags({ phase, cxxIsClang, inputs, output, needsCrypto, needsZlib, p ...(process.platform === 'linux' ? ['-pthread'] : []), ...(linking && process.platform === 'linux' && needsCrypto ? ['-lcrypto'] : []), ...(linking && needsZlib ? zlibLinkFlags() : []), + ...(linking && needsBrotli ? brotliLinkFlags() : []), '-o', output ] @@ -1185,6 +1187,39 @@ function zlibLinkFlags() { return windows ? [`-L${path.join(windows, 'lib')}`, '-lzs'] : ['-lz'] } +function brotliPrefix() { + if (brotliPrefix.cache !== undefined) return brotliPrefix.cache + const candidates = [] + if (process.env.BROTLI_DIR) candidates.push(path.resolve(process.env.BROTLI_DIR)) + else if (process.platform === 'win32') { + if (windowsLibraries()) candidates.push(windowsLibraries()) + } else if (process.platform === 'darwin') { + try { + candidates.push(execFileSync('brew', ['--prefix', 'brotli'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim()) + } catch {} + candidates.push('/opt/homebrew/opt/brotli', '/usr/local/opt/brotli') + } + brotliPrefix.cache = candidates.find((prefix) => fs.existsSync(path.join(prefix, 'include', 'brotli', 'decode.h'))) ?? null + if (brotliPrefix.cache === null && process.platform !== 'linux') { + throw new Error( + `brotli (libbrotlidec) not found: node:zlib's brotli decoder needs its headers and library. ${ + process.platform === 'darwin' ? 'Install with `brew install brotli`' : 'Install with `vcpkg install brotli --triplet x64-windows-static`' + }, or set BROTLI_DIR to a prefix holding include/brotli/decode.h and lib/.` + ) + } + return brotliPrefix.cache +} + +function brotliIncludeFlags() { + const prefix = brotliPrefix() + return prefix === null ? [] : [`-I${path.join(prefix, 'include')}`] +} + +function brotliLinkFlags() { + const prefix = brotliPrefix() + return [...(prefix === null ? [] : [`-L${path.join(prefix, 'lib')}`]), '-lbrotlidec', '-lbrotlicommon'] +} + // Full-LTO link flags for clang: the linker that reads LLVM bitcode -- gold's // plugin on Linux (see `cxxFlags`), lld-link on Windows, ld64 itself on macOS. function fullLtoLinkFlags() { @@ -1705,12 +1740,13 @@ async function linkUnits(units, displayPath, exePath) { // zlib the same way, on every platform: node:zlib's codecs are the system // libz, which macOS ships too but does not link by default. const needsZlib = units.some((unit) => unit.source.includes('#include "gea_node_zlib.hpp"')) + const needsBrotli = units.some((unit) => unit.source.includes('#include "gea_node_brotli.hpp"')) // The C++ compiler is the caller's; the default is clang++, which is what the // published benchmarks were built with (clang 18 on the Ubuntu bench host; // g++ 13 there is 4-10% slower on the same emitted source). const cxx = process.env.CXX ?? 'clang++' const cxxIsClang = /clang/.test(path.basename(cxx)) || (!/g\+\+|gcc/.test(path.basename(cxx)) && process.platform === 'darwin') - const common = { cxxIsClang, needsCrypto, needsZlib } + const common = { cxxIsClang, needsCrypto, needsZlib, needsBrotli } const linkStarted = process.hrtime.bigint() const fail = (stderr, fields = {}) => { diff --git a/scripts/test-native-brotli.mjs b/scripts/test-native-brotli.mjs new file mode 100644 index 0000000..38d4a7d --- /dev/null +++ b/scripts/test-native-brotli.mjs @@ -0,0 +1,81 @@ +import assert from 'node:assert/strict' +import { brotliCompressSync } from 'node:zlib' +import { execFileSync } from 'node:child_process' +import { existsSync, mkdirSync } from 'node:fs' +import { dirname, resolve } from 'node:path' + +import { compilerRuntimeInclude } from './resolve-compiler.mjs' + +const root = resolve(import.meta.dirname, '..') +const executable = resolve(root, 'apps/nest-node-probe/dist/native-brotli') + (process.platform === 'win32' ? '.exe' : '') +mkdirSync(dirname(executable), { recursive: true }) +const bytes = (value) => `bytes("${value.toString('hex')}")` +const inputs = [ + Buffer.alloc(0), + Buffer.from('abc'), + Buffer.from(Array.from({ length: 300000 }, (_, i) => (i * 37) & 255)), + Buffer.from('{"hello":"nest","n":1} '.repeat(20000)) +] +const operations = inputs.map((input) => `std::cout << hex(gea::node::brotli::decompress(${bytes(brotliCompressSync(input))})) << '\\n';`) +const trailing = Buffer.concat([brotliCompressSync(inputs[1]), Buffer.from('xyz')]) +operations.push(`std::cout << hex(gea::node::brotli::decompress(${bytes(trailing)})) << '\\n';`) +const truncated = brotliCompressSync(inputs[3]).subarray(0, 40) +const prefixes = [] +if (process.env.BROTLI_DIR) prefixes.push(resolve(process.env.BROTLI_DIR)) +else if (process.platform === 'win32') prefixes.push(...[process.env.VCPKG_ROOT, 'C:\\vcpkg'].filter(Boolean).map((r) => resolve(r, 'installed', 'x64-windows-static'))) +else if (process.platform === 'darwin') { + try { + prefixes.push(execFileSync('brew', ['--prefix', 'brotli'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim()) + } catch {} + prefixes.push('/opt/homebrew/opt/brotli', '/usr/local/opt/brotli') +} +const prefix = prefixes.find((p) => existsSync(resolve(p, 'include', 'brotli', 'decode.h'))) +const brotliFlags = [...(prefix ? [`-I${resolve(prefix, 'include')}`, `-L${resolve(prefix, 'lib')}`] : []), '-lbrotlidec', '-lbrotlicommon'] +const code = String.raw` +#include "gea_node_buffer.hpp" +#include "gea_node_brotli.hpp" +#include +#include +gea::TypedArray bytes(const std::string& hex) { + const auto source = gea::node::buffer::from(hex, "hex"); + gea::TypedArray result(source->size()); + std::copy(source->data(), source->data() + source->size(), result.data()); + return result; +} +std::string hex(const std::vector& source) { + gea::TypedArray value(source.size()); + std::copy(source.begin(), source.end(), value.data()); + return gea::node::buffer::toString(value, "hex"); +} +template void throws(F action, const std::string& expected) { + bool caught = false; + try { action(); } catch (const gea::Value& error) { + caught = true; + assert(gea::host::instanceOfError(error, "Error")); + std::cout << "throws " << expected << '\n'; + } + assert(caught); +} +int main() { + ${operations.join('\n ')} + throws([] { gea::node::brotli::decompress(bytes("0102030405")); }, "unexpected end of file"); + throws([] { gea::node::brotli::decompress(${bytes(truncated)}); }, "unexpected end of file"); + throws([] { gea::node::brotli::decompress(bytes("ff")); }, "Decompression failed"); +} +` +execFileSync( + process.env.CXX ?? 'clang++', + ['-std=c++20', '-fsanitize=undefined,address', `-I${compilerRuntimeInclude()}`, `-I${resolve(root, 'runtime/node')}`, '-x', 'c++', '-', ...brotliFlags, '-o', executable], + { input: code, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], maxBuffer: 256 * 1024 * 1024 } +) +const lines = execFileSync(executable, { encoding: 'utf8', timeout: 60000, maxBuffer: 256 * 1024 * 1024 }) + .trimEnd() + .split(/\r?\n/) +let at = 0 +for (const input of inputs) assert.equal(lines[at++], input.toString('hex'), 'decompress') +assert.equal(lines[at++], inputs[1].toString('hex'), 'trailing bytes ignored') +assert.equal(lines[at++], 'throws unexpected end of file') +assert.equal(lines[at++], 'throws unexpected end of file') +assert.equal(lines[at++], 'throws Decompression failed') +assert.equal(at, lines.length) +console.log(`NATIVE_BROTLI_OK:${lines.length}`) diff --git a/scripts/test-zlib-streams.mjs b/scripts/test-zlib-streams.mjs new file mode 100644 index 0000000..beeb3ca --- /dev/null +++ b/scripts/test-zlib-streams.mjs @@ -0,0 +1,26 @@ +import assert from 'node:assert/strict' +import { execFileSync, spawnSync } from 'node:child_process' +import { mkdirSync } from 'node:fs' +import { resolve } from 'node:path' + +const root = resolve(import.meta.dirname, '..') +const output = resolve(root, 'apps/nest-node-probe/dist') +const source = resolve(root, 'apps/nest-node-probe/correctness/zlib-streams.ts') +const executable = resolve(output, 'zlib-streams') +const options = { cwd: root, encoding: 'utf8', env: { ...process.env, TMPDIR: output } } +mkdirSync(output, { recursive: true }) + +execFileSync(process.execPath, [resolve(root, 'scripts/build.mjs'), source, '--out', output, '--exe', executable], { + ...options, + stdio: 'inherit' +}) +const expected = spawnSync(process.execPath, [source], { ...options, timeout: 15000 }) +const actual = spawnSync(executable, [], { ...options, timeout: 15000 }) +assert.equal(expected.status, 0, `node status\n${expected.stderr}`) +assert.equal(actual.status, 0, `compiled status\n${actual.stderr}`) +const expectedLines = expected.stdout.trim().split('\n') +const actualLines = actual.stdout.trim().split('\n') +assert.equal(actualLines.length, expectedLines.length, `line count\n--- node\n${expected.stdout}\n--- compiled\n${actual.stdout}`) +for (let i = 0; i < expectedLines.length; i++) assert.equal(actualLines[i], expectedLines[i], `line ${i + 1}`) +assert.ok(expectedLines.includes('brotli: end 1800 match') && expectedLines.includes('gunzip garbage: error incorrect header check'), 'the oracle covered the decoders') +console.log(`ZLIB_STREAMS_PARITY_OK:${expectedLines.length}`)