From 175904b57eec66b25410184b801075dbacc73312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Haylee=20Sch=C3=A4fer?= Date: Sat, 17 Oct 2026 12:48:49 +0200 Subject: [PATCH 1/5] Profile world placement stages and section rebuild ownership --- apps/web/benchmarks/README.md | 36 ++++ apps/web/benchmarks/placement-profile.ts | 220 +++++++++++++++++++++++ apps/web/benchmarks/world.ts | 7 +- apps/web/scripts/benchmark-world.mjs | 19 +- apps/web/scripts/placement-profile.mjs | 118 ++++++++++++ 5 files changed, 398 insertions(+), 2 deletions(-) create mode 100644 apps/web/benchmarks/placement-profile.ts create mode 100644 apps/web/scripts/placement-profile.mjs diff --git a/apps/web/benchmarks/README.md b/apps/web/benchmarks/README.md index 0915aa1c..7bd141e7 100644 --- a/apps/web/benchmarks/README.md +++ b/apps/web/benchmarks/README.md @@ -18,6 +18,42 @@ The default load radius is 1, unload radius is 2, viewport is 1280×720 at pixel Chrome uses its selected GPU by default. `--software-rendering` requests SwiftShader for validation on machines without usable GPU rendering; those results describe software rendering. `--headed` shows the browser. The report records the browser, actual WebGL renderer, operating system, revision, settings, and hashes of every world file requested. Missing files are listed separately. Benchmark results and save files do not belong in Git. +## Profile placement + +Add `--profile-placement` to record placement stages and identify sections built more than once during a stop. This option requires section meshing. For example: + +```sh +yarn workspace @minerender/web benchmark:world \ + --world "SAVE_FOLDER" --version 1.16.2 \ + --route '[[-15,8],[-7,8],[1,8],[9,8]]' \ + --profile-placement --output /tmp/world-placement.json +``` + +The runner instruments four built ESM modules in memory while bundling the benchmark. It leaves library source, package files, and the section worker unchanged. If a method or measured boundary has changed, bundling fails instead of silently omitting that measurement. Without the flag, the runner uses the ordinary benchmark bundle. + +The report's `placementProfile` contains: + +- `events`: timed spans with IDs, explicit parent IDs, pass, phase, and waypoint. Section builds and their child stages include absolute section coordinates, including the vertical section coordinate. State and model preparation events identify their block or model; they have no section assignment. +- `sections`: build counts by pass, waypoint, and section. `repeatedBuilds` counts calls after the first at that stop. Loading an adjacent column can change a section's border, so repeated calls do not by themselves prove unnecessary work. Returning to a previously visited section is a separate stop. +- `stages`: call counts, failures, summed `inclusiveMs`, and `unionMs` by pass and stage, including cleanup. The union counts overlapping intervals of that stage once. Different stages can overlap or contain each other, so neither column can be added across stages to calculate total streaming time. Filter `events` by `phase` to exclude cleanup. +- `issues` and `incompleteSpans`: missing ownership or unfinished instrumentation. The runner exits unsuccessfully if it detects these issues or if the profiler's section-build count differs from the benchmark's count. + +Stage boundaries are: + +| Stage | Includes | +|---|---| +| `state-preparation` | Selecting prepared state/templates and resolving tints, including cache hits and waits | +| `state-resolution` | The state cache's loader: model resolution, model atlases, and rotation data | +| `template-preparation` / `template-build` | Template cache lookup/wait / the cache loader's geometry creation | +| `model-atlas` | Creating a model texture atlas, including awaited texture loads; warm caches can skip this stage | +| `section-rebuild` | A rebuild attempt, including neighborhood preparation, mesh replacement, and attempts that need no build | +| `section-build` / `section-input` | A mesh build / conversion of its entries and templates to worker input | +| `worker-roundtrip` | Worker dispatch through reply, including queueing, message cloning, and main-thread scheduling; this is not worker CPU time | +| `synchronous-geometry` | Geometry built on the main thread when no worker is available or a worker rejects | +| `mesh-assembly` / `atlas-paint` | Page canvases, geometry, materials, bounds, and subscriptions / its nested canvas allocation and image painting | + +Profiling adds clock reads, span records, promise continuations, and retained JavaScript memory. Run at least three ordinary and three profiled invocations under matching conditions to assess its overhead. Use ordinary runs for performance comparisons. Profiled times describe the instrumented run, and its heap includes profiler records. The worker hash stays unchanged; the instrumented main script has a different hash. Both are recorded in the report. + ## Read the report Each invocation has two passes. The first starts with fresh browser asset storage and a fresh `AnvilWorldSource`; the second keeps loaded assets and the renderer but creates a fresh world and source. Shared GPU resources and shader programs can also remain warm. Within each pass, returning to earlier stops reloads columns while raw region files can remain cached. The runner serves save files with `Cache-Control: no-store`. diff --git a/apps/web/benchmarks/placement-profile.ts b/apps/web/benchmarks/placement-profile.ts new file mode 100644 index 00000000..73939e8d --- /dev/null +++ b/apps/web/benchmarks/placement-profile.ts @@ -0,0 +1,220 @@ +export interface PlacementProfileContext { + pass: "cold" | "warm"; + phase: "outbound" | "return" | "cleanup"; + waypoint: number | null; +} + +export type PlacementProfileKind = "section-rebuild" | "section-build" | "section-input" | "worker-roundtrip" + | "synchronous-geometry" | "mesh-assembly" | "atlas-paint" | "model-atlas" | "state-resolution" + | "state-preparation" | "template-preparation" | "template-build"; + +export interface PlacementProfileMetadata { + [key: string]: string | number | boolean | null | undefined; + x?: number; + y?: number; + z?: number; +} + +export interface PlacementProfileEvent extends PlacementProfileContext, PlacementProfileMetadata { + spanId: number; + parentSpanId?: number; + kind: PlacementProfileKind; + startMs: number; + durationMs: number; + success: boolean; + error?: string; +} + +interface PendingSpan { + spanId: number; + parentSpanId?: number; + kind: PlacementProfileKind; + context: PlacementProfileContext; + metadata: PlacementProfileMetadata; + startMs: number; +} + +export interface PlacementProfileHooks { + begin(kind: PlacementProfileKind, metadata?: PlacementProfileMetadata, parentId?: number): number; + end(spanId: number, success: boolean, error?: unknown, metadata?: PlacementProfileMetadata): void; + measure(kind: PlacementProfileKind, metadata: PlacementProfileMetadata, parentId: number | undefined, action: (spanId: number) => T): T; + measureAsync(kind: PlacementProfileKind, metadata: PlacementProfileMetadata, parentId: number | undefined, action: (spanId: number) => Promise): Promise; +} + +interface StageSummary { + pass: PlacementProfileContext["pass"]; + kind: PlacementProfileKind; + count: number; + succeeded: number; + failed: number; + inclusiveMs: number; + unionMs: number; +} + +interface SectionSummary extends PlacementProfileContext { + x: number; + y: number; + z: number; + builds: number; + succeeded: number; + failed: number; + repeatedBuilds: number; + inclusiveMs: number; + unionMs: number; +} + +const GLOBAL_KEY = "__minerenderPlacementProfile"; +const errorMessage = (error: unknown): string => error instanceof Error ? error.message : String(error); + +function unionDuration(events: readonly PlacementProfileEvent[]): number { + const ordered = events.slice().sort((a, b) => a.startMs - b.startMs); + let total = 0, start = ordered[0]?.startMs ?? 0, end = start; + for (const event of ordered) { + const nextEnd = event.startMs + event.durationMs; + if (event.startMs > end) { + total += end - start; + start = event.startMs; + end = nextEnd; + } else end = Math.max(end, nextEnd); + } + return total + end - start; +} + +/** Installs hooks only for the benchmark bundle. Explicit parent IDs preserve ownership across awaits. */ +export function installPlacementProfiler(started: number, getContext: () => PlacementProfileContext) { + if (!Number.isFinite(started)) throw new RangeError("The placement profiler start time must be finite"); + if (globalThis.__minerenderPlacementProfile !== undefined) throw new Error("A placement profiler is already installed"); + const descriptor = Object.getOwnPropertyDescriptor(globalThis, GLOBAL_KEY); + const events: PlacementProfileEvent[] = []; + const spans = new Map(); + const active = new Set(); + const issues: string[] = []; + let nextId = 0, restored = false; + + const metadataCopy = (metadata: PlacementProfileMetadata): PlacementProfileMetadata => { + const copy: PlacementProfileMetadata = {}; + for (const [key, value] of Object.entries(metadata)) { + if (value === undefined) continue; + if (value === null || typeof value === "string" || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value)) copy[key] = value; + else issues.push(`Ignored invalid ${key} metadata`); + } + return copy; + }; + + const hooks: PlacementProfileHooks = { + begin(kind, metadata = {}, parentId) { + if (restored) return 0; + const parent = parentId === undefined ? undefined : spans.get(parentId); + if (parentId !== undefined && !parent) issues.push(`Missing parent span ${parentId} for ${kind}`); + const spanId = ++nextId; + spans.set(spanId, { + spanId, parentSpanId: parentId, kind, context: { ...(parent?.context ?? getContext()) }, + metadata: { ...parent?.metadata, ...metadataCopy(metadata) }, startMs: performance.now() - started + }); + active.add(spanId); + return spanId; + }, + end(spanId, success, error, metadata = {}) { + if (!spanId || restored) return; + const span = spans.get(spanId); + if (!span || !active.delete(spanId)) { + issues.push(`Unknown or completed span ${spanId}`); + return; + } + const durationMs = Math.max(0, performance.now() - started - span.startMs); + events.push({ + ...span.metadata, ...metadataCopy(metadata), ...span.context, + spanId, parentSpanId: span.parentSpanId, kind: span.kind, startMs: span.startMs, durationMs, success, + ...(!success && { error: errorMessage(error) }) + }); + }, + measure(kind, metadata, parentId, action) { + const spanId = hooks.begin(kind, metadata, parentId); + let success = false, error: unknown; + try { + const result = action(spanId); + success = true; + return result; + } catch (caught) { + error = caught; + throw caught; + } finally { + hooks.end(spanId, success, error); + } + }, + async measureAsync(kind, metadata, parentId, action) { + const spanId = hooks.begin(kind, metadata, parentId); + let success = false, error: unknown; + try { + const result = await action(spanId); + success = true; + return result; + } catch (caught) { + error = caught; + throw caught; + } finally { + hooks.end(spanId, success, error); + } + } + }; + Object.defineProperty(globalThis, GLOBAL_KEY, { configurable: true, writable: true, value: hooks }); + + return { + ...hooks, + report() { + const stageEvents = new Map(); + const sectionEvents = new Map(); + const reportIssues = issues.slice(); + for (const spanId of active) reportIssues.push(`Span ${spanId} is still open`); + for (const event of events) { + const stageKey = JSON.stringify([event.pass, event.kind]); + let stage = stageEvents.get(stageKey); + if (!stage) stageEvents.set(stageKey, stage = []); + stage.push(event); + if (event.kind !== "section-build") continue; + if (![event.x, event.y, event.z].every(value => Number.isInteger(value))) { + reportIssues.push(`Section build ${event.spanId} has no integer section coordinates`); + continue; + } + const sectionKey = JSON.stringify([event.pass, event.phase, event.waypoint, event.x, event.y, event.z]); + let section = sectionEvents.get(sectionKey); + if (!section) sectionEvents.set(sectionKey, section = []); + section.push(event); + } + const stages: StageSummary[] = [...stageEvents.values()].map(group => ({ + pass: group[0].pass, kind: group[0].kind, count: group.length, + succeeded: group.filter(event => event.success).length, failed: group.filter(event => !event.success).length, + inclusiveMs: group.reduce((total, event) => total + event.durationMs, 0), unionMs: unionDuration(group) + })); + const sections: SectionSummary[] = [...sectionEvents.values()].map(group => ({ + pass: group[0].pass, phase: group[0].phase, waypoint: group[0].waypoint, + x: group[0].x!, y: group[0].y!, z: group[0].z!, builds: group.length, + succeeded: group.filter(event => event.success).length, failed: group.filter(event => !event.success).length, + repeatedBuilds: Math.max(0, group.length - 1), + inclusiveMs: group.reduce((total, event) => total + event.durationMs, 0), unionMs: unionDuration(group) + })); + return { + events: events.slice().sort((a, b) => a.spanId - b.spanId), stages, sections, + incompleteSpans: [...active].map(spanId => { + const span = spans.get(spanId)!; + return { ...span.metadata, ...span.context, spanId, parentSpanId: span.parentSpanId, kind: span.kind, startMs: span.startMs }; + }), + issues: reportIssues, + notes: [ + "Inclusive stage durations overlap; interval unions count each stage's covered wall time once per pass.", + "Repeated section builds count extra calls at the same section coordinates within one waypoint; they do not prove redundant work." + ] + }; + }, + restore() { + if (restored) return; + restored = true; + if (descriptor) Object.defineProperty(globalThis, GLOBAL_KEY, descriptor); + else delete globalThis.__minerenderPlacementProfile; + } + }; +} + +declare global { + var __minerenderPlacementProfile: PlacementProfileHooks | undefined; +} diff --git a/apps/web/benchmarks/world.ts b/apps/web/benchmarks/world.ts index 6b160f69..c608824f 100644 --- a/apps/web/benchmarks/world.ts +++ b/apps/web/benchmarks/world.ts @@ -2,6 +2,7 @@ import { AnvilParser, AnvilWorldSource, AssetLoader, Env, MineRenderWorld, Renderer, SectionMesh, SectionWorker, WorldStreamer } from "minerender"; +import { installPlacementProfiler } from "./placement-profile"; export interface WorldBenchmarkConfig { version: string; @@ -15,6 +16,7 @@ export interface WorldBenchmarkConfig { biomeBlendRadius: number; renderEntities: boolean; maxCachedRegions: number; + profilePlacement: boolean; } const CAMERA_HEIGHT = 160; @@ -114,6 +116,7 @@ async function runWorldBenchmark(config: WorldBenchmarkConfig) { const operations: Operation[] = []; const observedDataVersions = new Set(); let context: Pick = { pass: "cold", phase: "outbound", waypoint: null }; + const placementProfile = config.profilePlacement ? installPlacementProfiler(started, () => context) : undefined; let frames: ReturnType | undefined; const worker = { created: 0, buildRequests: 0, buildReplies: 0, buildFailures: 0, synchronousBuilds: 0 }; const workerStats = () => ({ ...worker, fallbackBuilds: worker.buildFailures + worker.synchronousBuilds }); @@ -316,7 +319,8 @@ async function runWorldBenchmark(config: WorldBenchmarkConfig) { runs.push({ pass, cacheStatus: `fresh source; ${pass === "cold" ? "cold" : "warm"} assets`, waypoints, cleanupMs, heapBytesAfterCleanupGc }); } status.textContent = "Benchmark complete"; - return { settings, environment, observedDataVersions: [...observedDataVersions].sort((a, b) => a - b), runs, operations, worker: workerStats() }; + return { settings, environment, observedDataVersions: [...observedDataVersions].sort((a, b) => a - b), runs, operations, + worker: workerStats(), placementProfile: placementProfile?.report() }; } catch (error) { status.textContent = `Benchmark failed: ${errorMessage(error)}`; throw error; @@ -332,6 +336,7 @@ async function runWorldBenchmark(config: WorldBenchmarkConfig) { SectionWorker.prototype.build = original.buildWorker; if (createWorkerDescriptor) Object.defineProperty(provider, "createWorker", createWorkerDescriptor); else delete provider.createWorker; + placementProfile?.restore(); running = false; } } diff --git a/apps/web/scripts/benchmark-world.mjs b/apps/web/scripts/benchmark-world.mjs index fe5ef97e..7a067660 100644 --- a/apps/web/scripts/benchmark-world.mjs +++ b/apps/web/scripts/benchmark-world.mjs @@ -10,6 +10,7 @@ import { parseArgs } from "node:util"; import * as esbuild from "esbuild"; import { polyfillNode } from "esbuild-plugin-polyfill-node"; import puppeteer from "puppeteer-core"; +import { placementProfilePlugin } from "./placement-profile.mjs"; const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const repository = path.resolve(root, "../.."); @@ -27,6 +28,7 @@ const { values: args } = parseArgs({ options: { "saved-lighting": { type: "boolean", default: false }, "ambient-occlusion": { type: "boolean", default: false }, "render-entities": { type: "boolean", default: false }, + "profile-placement": { type: "boolean", default: false }, "software-rendering": { type: "boolean", default: false }, headed: { type: "boolean", default: false }, "timeout-ms": { type: "string", default: "1800000" }, @@ -43,6 +45,7 @@ if (args.help) { --saved-lighting --ambient-occlusion --biome-blend-radius N --render-entities --no-section-meshing --max-cached-regions N --software-rendering Use SwiftShader instead of Chrome's selected GPU +--profile-placement Record section rebuilds and placement stages (adds timing overhead) --headed --timeout-ms N --output REPORT.json Set CHROME_PATH for a Chrome binary outside the default macOS installation. @@ -152,10 +155,12 @@ try { maxCachedRegions: integer("max-cached-regions", 0), biomeBlendRadius: integer("biome-blend-radius", 0, 7), sectionMeshing: !args["no-section-meshing"], savedLighting: args["saved-lighting"], ambientOcclusion: args["ambient-occlusion"], renderEntities: args["render-entities"], + profilePlacement: args["profile-placement"], }; if (!config.sectionMeshing && (config.savedLighting || config.ambientOcclusion || config.biomeBlendRadius)) { throw new Error("Saved lighting, ambient occlusion, and biome blending require section meshing"); } + if (config.profilePlacement && !config.sectionMeshing) throw new Error("Placement profiling requires section meshing"); const timeout = integer("timeout-ms", 1000); const chrome = process.env.CHROME_PATH ?? "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"; await stat(chrome); @@ -163,7 +168,7 @@ try { const bundle = await esbuild.build({ entryPoints: [path.join(root, "benchmarks/world.ts")], bundle: true, write: false, format: "esm", platform: "browser", target: "es2020", logLevel: "warning", - plugins: [polyfillNode({ polyfills: { zlib: true } })], + plugins: [...(config.profilePlacement ? [placementProfilePlugin()] : []), polyfillNode({ polyfills: { zlib: true } })], }); const requests = new Map(); const html = 'World streaming benchmark
Starting benchmark'; @@ -218,6 +223,14 @@ try { }), ]); } finally { clearTimeout(timeoutTimer); } + if (config.profilePlacement) { + if (!result.placementProfile) throw new Error("The browser returned no placement profile"); + problems.push(...result.placementProfile.issues.map(issue => `Placement profile: ${issue}`)); + const builds = result.operations.filter(operation => operation.kind === "section-build").length; + if (result.placementProfile.events.filter(event => event.kind === "section-build").length !== builds) { + problems.push("Placement profile: section build count does not match benchmark operations"); + } + } const manifest = []; for (const relative of [...requests.keys()].sort()) { const file = files.get(relative); @@ -246,6 +259,10 @@ try { const updateMs = run.waypoints.reduce((sum, point) => sum + point.updateMs, 0); const maxGap = Math.max(...run.waypoints.map(point => point.frames.maxMs ?? 0)); console.log(`${run.pass}: ${updateMs.toFixed(0)} ms streaming, ${maxGap.toFixed(1)} ms maximum frame gap`); + if (report.placementProfile) { + const sections = report.placementProfile.sections.filter(section => section.pass === run.pass); + console.log(` placement profile: ${sections.reduce((sum, section) => sum + section.builds, 0)} section builds, ${sections.reduce((sum, section) => sum + section.repeatedBuilds, 0)} repeated within a waypoint`); + } } console.log(`Report: ${output}`); if (problems.length || report.runs.some(run => run.waypoints.some(point => point.failedChunks > 0))) process.exitCode = 1; diff --git a/apps/web/scripts/placement-profile.mjs b/apps/web/scripts/placement-profile.mjs new file mode 100644 index 00000000..215c9d7a --- /dev/null +++ b/apps/web/scripts/placement-profile.mjs @@ -0,0 +1,118 @@ +import { readFile } from "node:fs/promises"; +import ts from "typescript"; + +const modules = ["Chunk", "SectionMesh", "SectionModels", "UVMapper"]; +const hooks = "globalThis.__minerenderPlacementProfile"; + +/** Instruments the benchmark bundle; the library's source and package output stay unchanged. */ +export function instrumentPlacementModule(contents, name) { + const source = ts.createSourceFile(`${name}.mjs`, contents, ts.ScriptTarget.Latest, true, ts.ScriptKind.JS); + const edits = []; + const fail = detail => { throw new Error(`Placement profiling: ${name}: ${detail}; rebuild the library or update the instrumentation`); }; + function find(root, predicate, description) { + const matches = []; + function visit(node) { + if (predicate(node)) matches.push(node); + ts.forEachChild(node, visit); + } + visit(root); + if (matches.length !== 1) fail(`expected one ${description}, found ${matches.length}`); + return matches[0]; + } + const method = name => find(source, node => ts.isMethodDeclaration(node) && node.name.getText(source) === name, `${name} method`); + const call = (root, name) => find(root, node => ts.isCallExpression(node) && node.expression.getText(source) === name, `${name} call`); + const text = node => node.getText(source); + const measure = (kind, metadata, parent, action, asynchronous = false) => + `${hooks}.${asynchronous ? "measureAsync" : "measure"}(${JSON.stringify(kind)}, ${metadata}, ${parent}, ${action})`; + const wrap = (kind, metadata, body, parent = "undefined") => + `return ${measure(kind, metadata, parent, `async (__placementSpan) => {${body}}`, true)};`; + function body(target, changes = [], wrapper) { + const start = target.body.getStart(source) + 1, end = target.body.end - 1; + let value = contents.slice(start, end); + for (const { node, value: replacement } of changes.sort((a, b) => b.node.getStart(source) - a.node.getStart(source))) { + const from = node.getStart(source) - start, to = node.end - start; + value = value.slice(0, from) + replacement + value.slice(to); + } + edits.push({ start, end, value: wrapper ? wrapper(value) : value }); + } + function parameter(target, expected, value) { + if (target.parameters.length !== expected) fail(`expected ${expected} parameters in ${text(target.name)}`); + const end = target.parameters.at(-1).end; + edits.push({ start: end, end, value: `, ${value}` }); + } + if (name === "Chunk") { + const target = method("rebuildSectionMesh"), build = call(target, "SectionMesh.buildAsync"); + if (build.arguments.length !== 4) fail("expected four section build arguments"); + body(target, [{ node: build, value: `${text(build).slice(0, -1)}, __placementSpan)` }], value => + wrap("section-rebuild", "{x: this.x, y: this.y, z: this.z}", value)); + } else if (name === "SectionMesh") { + const target = method("buildAsync"); + parameter(target, 4, "__placementParent"); + const input = call(target, "toInput"), worker = call(target, "worker.build"), assembly = call(target, "this.fromPages"); + const fallback = []; + function visit(node) { + if (ts.isCallExpression(node) && node.expression.getText(source) === "buildSectionGeometry") fallback.push(node); + ts.forEachChild(node, visit); + } + visit(target); + if (fallback.length !== 2) fail(`expected two geometry fallback calls, found ${fallback.length}`); + body(target, [ + { node: input, value: measure("section-input", "{}", "__placementSpan", `() => ${text(input)}`) }, + { node: worker, value: measure("worker-roundtrip", "{}", "__placementSpan", `() => ${text(worker)}`, true) }, + { node: assembly, value: `${text(assembly).slice(0, -1)}, __placementSpan)` }, + ...fallback.map(node => ({ node, value: measure("synchronous-geometry", "{}", "__placementSpan", `() => ${text(node)}`) })) + ], value => wrap("section-build", "{entries: entries.length, fluids: !!fluids}", value, "__placementParent")); + const assemble = method("fromPages"); + parameter(assemble, 2, "__placementParent"); + const loop = find(assemble, node => ts.isForOfStatement(node) && node.expression.getText(source) === "pages", "page loop"); + const statements = loop.statement.statements; + if (!statements || statements.length < 4 || !text(statements[0]).includes("createCanvas(page.width, page.height)") + || !text(statements[1]).includes('canvas.getContext("2d")') || !ts.isForOfStatement(statements[2]) + || !text(statements[3]).includes("new BufferGeometry()")) fail("atlas painting boundary changed"); + const paintStart = statements[0].getStart(source), paintEnd = statements[2].end; + const originalPaint = contents.slice(paintStart, paintEnd); + const paintNode = { getStart: () => paintStart, end: paintEnd }; + body(assemble, [{ node: paintNode, value: + `const {canvas, context} = ${measure("atlas-paint", "{pixels: page.width * page.height}", "__placementSpan", + `() => {${originalPaint}\nreturn {canvas, context};}`)};` }], value => + `return ${measure("mesh-assembly", "{pages: pages.length, triangles: pages.reduce((sum, page) => sum + page.indices.length / 3, 0)}", + "__placementParent", `(__placementSpan) => {${value}}`)};`); + } else if (name === "SectionModels") { + for (const [name, kind] of [["prepareState", "state-preparation"], ["prepare", "state-resolution"], ["template", "template-preparation"]]) { + const target = method(name); + const changes = []; + if (name === "template") { + const load = call(target, "cached").arguments.at(-1); + if (!ts.isArrowFunction(load) || !ts.isBlock(load.body)) fail("expected a template cache loader"); + changes.push({ node: load.body, value: `{${wrap("template-build", "{model: model.key?.toNamespacedString()}", + contents.slice(load.body.getStart(source) + 1, load.body.end - 1), "__placementSpan")}}` }); + } + body(target, changes, value => wrap(kind, name === "template" ? "{model: model.key?.toNamespacedString()}" + : "{block: blockState.key?.toNamespacedString()}", value)); + } + } else if (name === "UVMapper") { + body(method("createAtlas"), [], value => wrap("model-atlas", "{model: originalModel.key?.toNamespacedString()}", value)); + } else fail("unexpected module"); + for (const edit of edits.sort((a, b) => b.start - a.start)) contents = contents.slice(0, edit.start) + edit.value + contents.slice(edit.end); + return contents; +} + +export function placementProfilePlugin() { + const instrumented = new Set(); + return { + name: "placement-profile", + setup(build) { + build.onLoad({ filter: /\/dist\/browser\/esm\/(?:world\/(?:Chunk|SectionMesh|SectionModels)|UVMapper)\.mjs$/ }, async ({ path }) => { + const name = path.split("/").at(-1).slice(0, -4); + const contents = instrumentPlacementModule(await readFile(path, "utf8"), name); + instrumented.add(name); + return { contents, loader: "js" }; + }); + build.onEnd(result => { + if (result.errors.length) return; + const missing = modules.filter(name => !instrumented.has(name)); + if (missing.length) return { errors: [{ text: `Placement profiling: missing ESM modules: ${missing.join(", ")}` }] }; + }); + } + }; +} From 216addfee5027f44e0ced6d8fa32ed1472581fe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Haylee=20Sch=C3=A4fer?= Date: Sat, 17 Oct 2026 16:58:48 +0200 Subject: [PATCH 2/5] codex round 1: attribute section builds to own and neighbouring column placements, check drift in the web build --- apps/web/benchmarks/README.md | 18 ++-- apps/web/benchmarks/placement-profile.ts | 105 ++++++++++++----------- apps/web/benchmarks/world.ts | 18 +++- apps/web/build.mjs | 6 ++ apps/web/scripts/benchmark-world.mjs | 11 +-- apps/web/scripts/placement-profile.mjs | 27 +++++- 6 files changed, 106 insertions(+), 79 deletions(-) diff --git a/apps/web/benchmarks/README.md b/apps/web/benchmarks/README.md index 7bd141e7..35078cb2 100644 --- a/apps/web/benchmarks/README.md +++ b/apps/web/benchmarks/README.md @@ -20,23 +20,17 @@ Chrome uses its selected GPU by default. `--software-rendering` requests SwiftSh ## Profile placement -Add `--profile-placement` to record placement stages and identify sections built more than once during a stop. This option requires section meshing. For example: - -```sh -yarn workspace @minerender/web benchmark:world \ - --world "SAVE_FOLDER" --version 1.16.2 \ - --route '[[-15,8],[-7,8],[1,8],[9,8]]' \ - --profile-placement --output /tmp/world-placement.json -``` +Add `--profile-placement` to the command above. This option records placement stages and requires section meshing. The runner instruments four built ESM modules in memory while bundling the benchmark. It leaves library source, package files, and the section worker unchanged. If a method or measured boundary has changed, bundling fails instead of silently omitting that measurement. Without the flag, the runner uses the ordinary benchmark bundle. The report's `placementProfile` contains: - `events`: timed spans with IDs, explicit parent IDs, pass, phase, and waypoint. Section builds and their child stages include absolute section coordinates, including the vertical section coordinate. State and model preparation events identify their block or model; they have no section assignment. -- `sections`: build counts by pass, waypoint, and section. `repeatedBuilds` counts calls after the first at that stop. Loading an adjacent column can change a section's border, so repeated calls do not by themselves prove unnecessary work. Returning to a previously visited section is a separate stop. -- `stages`: call counts, failures, summed `inclusiveMs`, and `unionMs` by pass and stage, including cleanup. The union counts overlapping intervals of that stage once. Different stages can overlap or contain each other, so neither column can be added across stages to calculate total streaming time. Filter `events` by `phase` to exclude cleanup. -- `issues` and `incompleteSpans`: missing ownership or unfinished instrumentation. The runner exits unsuccessfully if it detects these issues or if the profiler's section-build count differs from the benchmark's count. +- `summary`: build totals per pass, split into `ownColumnBuilds`, `neighborBuilds`, `unloadBuilds`, and `otherBuilds`, plus `sectionsBuiltTwiceInOneOperation` for sections built at least twice within one operation. +- `sections`: counts by pass and section coordinates (`x`, `y`, `z`), with `builds`, `succeeded`, the same operation split, `maxBuildsPerOperation`, `inclusiveMs`, and `unionMs`. +- `stages`: call counts, successes, summed `inclusiveMs`, and `unionMs` by pass and stage, including cleanup. The union counts overlapping intervals of that stage once. Different stages can overlap or contain each other, so neither column can be added across stages to calculate total streaming time. Filter `events` by `phase` to exclude cleanup. +- `issues` and `incompleteSpans`: missing ownership or unfinished instrumentation. The runner exits unsuccessfully if it detects these issues. Stage boundaries are: @@ -52,7 +46,7 @@ Stage boundaries are: | `synchronous-geometry` | Geometry built on the main thread when no worker is available or a worker rejects | | `mesh-assembly` / `atlas-paint` | Page canvases, geometry, materials, bounds, and subscriptions / its nested canvas allocation and image painting | -Profiling adds clock reads, span records, promise continuations, and retained JavaScript memory. Run at least three ordinary and three profiled invocations under matching conditions to assess its overhead. Use ordinary runs for performance comparisons. Profiled times describe the instrumented run, and its heap includes profiler records. The worker hash stays unchanged; the instrumented main script has a different hash. Both are recorded in the report. +Profiling adds clock reads, span records, promise continuations, and retained JavaScript memory. Use ordinary runs for performance comparisons. Profiled times describe the instrumented run, and its heap includes profiler records. The worker hash stays unchanged; the instrumented main script has a different hash. Both are recorded in the report. ## Read the report diff --git a/apps/web/benchmarks/placement-profile.ts b/apps/web/benchmarks/placement-profile.ts index 73939e8d..148915ea 100644 --- a/apps/web/benchmarks/placement-profile.ts +++ b/apps/web/benchmarks/placement-profile.ts @@ -2,6 +2,7 @@ export interface PlacementProfileContext { pass: "cold" | "warm"; phase: "outbound" | "return" | "cleanup"; waypoint: number | null; + operation: { id: number; kind: "placement" | "unload"; columns: { x: number; z: number }[] } | null; } export type PlacementProfileKind = "section-rebuild" | "section-build" | "section-input" | "worker-roundtrip" @@ -15,7 +16,7 @@ export interface PlacementProfileMetadata { z?: number; } -export interface PlacementProfileEvent extends PlacementProfileContext, PlacementProfileMetadata { +export interface PlacementProfileEvent extends PlacementProfileContext, Pick { spanId: number; parentSpanId?: number; kind: PlacementProfileKind; @@ -36,7 +37,7 @@ interface PendingSpan { export interface PlacementProfileHooks { begin(kind: PlacementProfileKind, metadata?: PlacementProfileMetadata, parentId?: number): number; - end(spanId: number, success: boolean, error?: unknown, metadata?: PlacementProfileMetadata): void; + end(spanId: number, success: boolean, error?: unknown): void; measure(kind: PlacementProfileKind, metadata: PlacementProfileMetadata, parentId: number | undefined, action: (spanId: number) => T): T; measureAsync(kind: PlacementProfileKind, metadata: PlacementProfileMetadata, parentId: number | undefined, action: (spanId: number) => Promise): Promise; } @@ -46,19 +47,22 @@ interface StageSummary { kind: PlacementProfileKind; count: number; succeeded: number; - failed: number; inclusiveMs: number; unionMs: number; } -interface SectionSummary extends PlacementProfileContext { +interface SectionSummary { + pass: PlacementProfileContext["pass"]; x: number; y: number; z: number; builds: number; succeeded: number; - failed: number; - repeatedBuilds: number; + ownColumnBuilds: number; + neighborBuilds: number; + unloadBuilds: number; + otherBuilds: number; + maxBuildsPerOperation: number; inclusiveMs: number; unionMs: number; } @@ -82,28 +86,21 @@ function unionDuration(events: readonly PlacementProfileEvent[]): number { /** Installs hooks only for the benchmark bundle. Explicit parent IDs preserve ownership across awaits. */ export function installPlacementProfiler(started: number, getContext: () => PlacementProfileContext) { - if (!Number.isFinite(started)) throw new RangeError("The placement profiler start time must be finite"); - if (globalThis.__minerenderPlacementProfile !== undefined) throw new Error("A placement profiler is already installed"); - const descriptor = Object.getOwnPropertyDescriptor(globalThis, GLOBAL_KEY); const events: PlacementProfileEvent[] = []; const spans = new Map(); - const active = new Set(); const issues: string[] = []; - let nextId = 0, restored = false; + let nextId = 0; const metadataCopy = (metadata: PlacementProfileMetadata): PlacementProfileMetadata => { const copy: PlacementProfileMetadata = {}; for (const [key, value] of Object.entries(metadata)) { - if (value === undefined) continue; - if (value === null || typeof value === "string" || typeof value === "boolean" || typeof value === "number" && Number.isFinite(value)) copy[key] = value; - else issues.push(`Ignored invalid ${key} metadata`); + if (value !== undefined) copy[key] = value; } return copy; }; const hooks: PlacementProfileHooks = { begin(kind, metadata = {}, parentId) { - if (restored) return 0; const parent = parentId === undefined ? undefined : spans.get(parentId); if (parentId !== undefined && !parent) issues.push(`Missing parent span ${parentId} for ${kind}`); const spanId = ++nextId; @@ -111,19 +108,15 @@ export function installPlacementProfiler(started: number, getContext: () => Plac spanId, parentSpanId: parentId, kind, context: { ...(parent?.context ?? getContext()) }, metadata: { ...parent?.metadata, ...metadataCopy(metadata) }, startMs: performance.now() - started }); - active.add(spanId); return spanId; }, - end(spanId, success, error, metadata = {}) { - if (!spanId || restored) return; + end(spanId, success, error) { const span = spans.get(spanId); - if (!span || !active.delete(spanId)) { - issues.push(`Unknown or completed span ${spanId}`); - return; - } - const durationMs = Math.max(0, performance.now() - started - span.startMs); + if (!span) return; + spans.delete(spanId); + const durationMs = performance.now() - started - span.startMs; events.push({ - ...span.metadata, ...metadataCopy(metadata), ...span.context, + ...span.metadata, ...span.context, spanId, parentSpanId: span.parentSpanId, kind: span.kind, startMs: span.startMs, durationMs, success, ...(!success && { error: errorMessage(error) }) }); @@ -160,12 +153,11 @@ export function installPlacementProfiler(started: number, getContext: () => Plac Object.defineProperty(globalThis, GLOBAL_KEY, { configurable: true, writable: true, value: hooks }); return { - ...hooks, report() { const stageEvents = new Map(); const sectionEvents = new Map(); const reportIssues = issues.slice(); - for (const spanId of active) reportIssues.push(`Span ${spanId} is still open`); + for (const spanId of spans.keys()) reportIssues.push(`Span ${spanId} is still open`); for (const event of events) { const stageKey = JSON.stringify([event.pass, event.kind]); let stage = stageEvents.get(stageKey); @@ -176,45 +168,56 @@ export function installPlacementProfiler(started: number, getContext: () => Plac reportIssues.push(`Section build ${event.spanId} has no integer section coordinates`); continue; } - const sectionKey = JSON.stringify([event.pass, event.phase, event.waypoint, event.x, event.y, event.z]); + const sectionKey = JSON.stringify([event.pass, event.x, event.y, event.z]); let section = sectionEvents.get(sectionKey); if (!section) sectionEvents.set(sectionKey, section = []); section.push(event); } const stages: StageSummary[] = [...stageEvents.values()].map(group => ({ pass: group[0].pass, kind: group[0].kind, count: group.length, - succeeded: group.filter(event => event.success).length, failed: group.filter(event => !event.success).length, - inclusiveMs: group.reduce((total, event) => total + event.durationMs, 0), unionMs: unionDuration(group) - })); - const sections: SectionSummary[] = [...sectionEvents.values()].map(group => ({ - pass: group[0].pass, phase: group[0].phase, waypoint: group[0].waypoint, - x: group[0].x!, y: group[0].y!, z: group[0].z!, builds: group.length, - succeeded: group.filter(event => event.success).length, failed: group.filter(event => !event.success).length, - repeatedBuilds: Math.max(0, group.length - 1), + succeeded: group.filter(event => event.success).length, inclusiveMs: group.reduce((total, event) => total + event.durationMs, 0), unionMs: unionDuration(group) })); + const sections: SectionSummary[] = [...sectionEvents.values()].map(group => { + const operations = new Map(); + let ownColumnBuilds = 0, neighborBuilds = 0, unloadBuilds = 0, otherBuilds = 0; + for (const event of group) { + const operation = event.operation; + if (!operation) otherBuilds++; + else { + operations.set(operation.id, (operations.get(operation.id) ?? 0) + 1); + if (operation.kind === "unload") unloadBuilds++; + else if (operation.columns.some(column => column.x === event.x && column.z === event.z)) ownColumnBuilds++; + else neighborBuilds++; + } + } + return { + pass: group[0].pass, x: group[0].x!, y: group[0].y!, z: group[0].z!, builds: group.length, + succeeded: group.filter(event => event.success).length, + ownColumnBuilds, neighborBuilds, unloadBuilds, otherBuilds, + maxBuildsPerOperation: Math.max(0, ...operations.values()), + inclusiveMs: group.reduce((total, event) => total + event.durationMs, 0), unionMs: unionDuration(group) + }; + }); + const summary = (["cold", "warm"] as const).map(pass => sections.filter(section => section.pass === pass).reduce((total, section) => ({ + pass, builds: total.builds + section.builds, + ownColumnBuilds: total.ownColumnBuilds + section.ownColumnBuilds, + neighborBuilds: total.neighborBuilds + section.neighborBuilds, + unloadBuilds: total.unloadBuilds + section.unloadBuilds, + otherBuilds: total.otherBuilds + section.otherBuilds, + sectionsBuiltTwiceInOneOperation: total.sectionsBuiltTwiceInOneOperation + Number(section.maxBuildsPerOperation >= 2) + }), { pass, builds: 0, ownColumnBuilds: 0, neighborBuilds: 0, unloadBuilds: 0, otherBuilds: 0, sectionsBuiltTwiceInOneOperation: 0 })); return { - events: events.slice().sort((a, b) => a.spanId - b.spanId), stages, sections, - incompleteSpans: [...active].map(spanId => { - const span = spans.get(spanId)!; + events: events.slice().sort((a, b) => a.spanId - b.spanId), stages, sections, summary, + incompleteSpans: [...spans.values()].map(span => { + const { spanId } = span; return { ...span.metadata, ...span.context, spanId, parentSpanId: span.parentSpanId, kind: span.kind, startMs: span.startMs }; }), - issues: reportIssues, - notes: [ - "Inclusive stage durations overlap; interval unions count each stage's covered wall time once per pass.", - "Repeated section builds count extra calls at the same section coordinates within one waypoint; they do not prove redundant work." - ] + issues: reportIssues }; }, restore() { - if (restored) return; - restored = true; - if (descriptor) Object.defineProperty(globalThis, GLOBAL_KEY, descriptor); - else delete globalThis.__minerenderPlacementProfile; + delete (globalThis as typeof globalThis & { [GLOBAL_KEY]?: PlacementProfileHooks })[GLOBAL_KEY]; } }; } - -declare global { - var __minerenderPlacementProfile: PlacementProfileHooks | undefined; -} diff --git a/apps/web/benchmarks/world.ts b/apps/web/benchmarks/world.ts index c608824f..2d3b43e4 100644 --- a/apps/web/benchmarks/world.ts +++ b/apps/web/benchmarks/world.ts @@ -2,7 +2,7 @@ import { AnvilParser, AnvilWorldSource, AssetLoader, Env, MineRenderWorld, Renderer, SectionMesh, SectionWorker, WorldStreamer } from "minerender"; -import { installPlacementProfiler } from "./placement-profile"; +import { installPlacementProfiler, PlacementProfileContext } from "./placement-profile"; export interface WorldBenchmarkConfig { version: string; @@ -116,7 +116,9 @@ async function runWorldBenchmark(config: WorldBenchmarkConfig) { const operations: Operation[] = []; const observedDataVersions = new Set(); let context: Pick = { pass: "cold", phase: "outbound", waypoint: null }; - const placementProfile = config.profilePlacement ? installPlacementProfiler(started, () => context) : undefined; + let operation: PlacementProfileContext["operation"] = null; + let nextOperationId = 0; + const placementProfile = config.profilePlacement ? installPlacementProfiler(started, () => ({ ...context, operation })) : undefined; let frames: ReturnType | undefined; const worker = { created: 0, buildRequests: 0, buildReplies: 0, buildFailures: 0, synchronousBuilds: 0 }; const workerStats = () => ({ ...worker, fallbackBuilds: worker.buildFailures + worker.synchronousBuilds }); @@ -255,10 +257,18 @@ async function runWorldBenchmark(config: WorldBenchmarkConfig) { readEntityRegion: config.renderEntities ? (x, z, signal) => readFile("entity-region", x, z, `/world-files/entities/r.${x}.${z}.mca`, signal) : undefined }); const streamer = new WorldStreamer({ - placeChunk: chunk => timed("placement", { x: chunk.x, z: chunk.z }, () => world.placeChunk(chunk)), + placeChunk: chunk => timed("placement", { x: chunk.x, z: chunk.z }, async () => { + operation = { id: ++nextOperationId, kind: "placement", columns: [{ x: chunk.x, z: chunk.z }] }; + try { await world.placeChunk(chunk); } + finally { operation = null; } + }), unloadChunkColumns: columns => timed("unload", { columnCount: columns.length, columns: columns.map(({ x, z }) => ({ x, z })) - }, () => world.unloadChunkColumns(columns)) + }, async () => { + operation = { id: ++nextOperationId, kind: "unload", columns: columns.map(({ x, z }) => ({ x, z })) }; + try { await world.unloadChunkColumns(columns); } + finally { operation = null; } + }) }, { getChunk: (x, z, signal) => timed("source", { x, z }, () => source.getChunk(x, z, signal)) }, { loadRadius: config.loadRadius, unloadRadius: config.unloadRadius }); diff --git a/apps/web/build.mjs b/apps/web/build.mjs index 0eec1cb6..6852d5c0 100644 --- a/apps/web/build.mjs +++ b/apps/web/build.mjs @@ -4,6 +4,7 @@ import glob from "glob"; import { parseArgs } from "node:util"; import { copyFile, mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { dirname, relative, sep } from "node:path"; +import { instrumentPlacementModule } from "./scripts/placement-profile.mjs"; const { values: args } = parseArgs({ options: { @@ -74,3 +75,8 @@ if (args.serve) { await writeFile(output, html); })); } + +for (const name of ["Chunk", "SectionMesh", "SectionModels", "UVMapper"]) { + const file = `../../packages/minerender/dist/browser/esm/${name === "UVMapper" ? "" : "world/"}${name}.mjs`; + instrumentPlacementModule(await readFile(file, "utf8"), name); +} diff --git a/apps/web/scripts/benchmark-world.mjs b/apps/web/scripts/benchmark-world.mjs index 7a067660..57fcc312 100644 --- a/apps/web/scripts/benchmark-world.mjs +++ b/apps/web/scripts/benchmark-world.mjs @@ -10,7 +10,6 @@ import { parseArgs } from "node:util"; import * as esbuild from "esbuild"; import { polyfillNode } from "esbuild-plugin-polyfill-node"; import puppeteer from "puppeteer-core"; -import { placementProfilePlugin } from "./placement-profile.mjs"; const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const repository = path.resolve(root, "../.."); @@ -168,7 +167,7 @@ try { const bundle = await esbuild.build({ entryPoints: [path.join(root, "benchmarks/world.ts")], bundle: true, write: false, format: "esm", platform: "browser", target: "es2020", logLevel: "warning", - plugins: [...(config.profilePlacement ? [placementProfilePlugin()] : []), polyfillNode({ polyfills: { zlib: true } })], + plugins: [...(config.profilePlacement ? [(await import("./placement-profile.mjs")).placementProfilePlugin()] : []), polyfillNode({ polyfills: { zlib: true } })], }); const requests = new Map(); const html = 'World streaming benchmark
Starting benchmark'; @@ -226,10 +225,6 @@ try { if (config.profilePlacement) { if (!result.placementProfile) throw new Error("The browser returned no placement profile"); problems.push(...result.placementProfile.issues.map(issue => `Placement profile: ${issue}`)); - const builds = result.operations.filter(operation => operation.kind === "section-build").length; - if (result.placementProfile.events.filter(event => event.kind === "section-build").length !== builds) { - problems.push("Placement profile: section build count does not match benchmark operations"); - } } const manifest = []; for (const relative of [...requests.keys()].sort()) { @@ -260,8 +255,8 @@ try { const maxGap = Math.max(...run.waypoints.map(point => point.frames.maxMs ?? 0)); console.log(`${run.pass}: ${updateMs.toFixed(0)} ms streaming, ${maxGap.toFixed(1)} ms maximum frame gap`); if (report.placementProfile) { - const sections = report.placementProfile.sections.filter(section => section.pass === run.pass); - console.log(` placement profile: ${sections.reduce((sum, section) => sum + section.builds, 0)} section builds, ${sections.reduce((sum, section) => sum + section.repeatedBuilds, 0)} repeated within a waypoint`); + const summary = report.placementProfile.summary.find(summary => summary.pass === run.pass); + console.log(` placement profile: ${summary.builds} section builds (own column ${summary.ownColumnBuilds}, neighbouring columns ${summary.neighborBuilds}, unloads ${summary.unloadBuilds})`); } } console.log(`Report: ${output}`); diff --git a/apps/web/scripts/placement-profile.mjs b/apps/web/scripts/placement-profile.mjs index 215c9d7a..8368fce0 100644 --- a/apps/web/scripts/placement-profile.mjs +++ b/apps/web/scripts/placement-profile.mjs @@ -1,5 +1,6 @@ import { readFile } from "node:fs/promises"; import ts from "typescript"; +import path from "node:path"; const modules = ["Chunk", "SectionMesh", "SectionModels", "UVMapper"]; const hooks = "globalThis.__minerenderPlacementProfile"; @@ -20,6 +21,19 @@ export function instrumentPlacementModule(contents, name) { return matches[0]; } const method = name => find(source, node => ts.isMethodDeclaration(node) && node.name.getText(source) === name, `${name} method`); + function bindings(target, names) { + const declared = new Set(); + function binding(name) { + if (ts.isIdentifier(name)) declared.add(name.text); + else for (const element of name.elements) if (ts.isBindingElement(element)) binding(element.name); + } + function visit(node) { + if (ts.isParameter(node) || ts.isVariableDeclaration(node)) binding(node.name); + ts.forEachChild(node, visit); + } + visit(target); + for (const name of names) if (!declared.has(name)) fail(`missing ${name} binding in ${target.name.getText(source)}`); + } const call = (root, name) => find(root, node => ts.isCallExpression(node) && node.expression.getText(source) === name, `${name} call`); const text = node => node.getText(source); const measure = (kind, metadata, parent, action, asynchronous = false) => @@ -47,6 +61,7 @@ export function instrumentPlacementModule(contents, name) { wrap("section-rebuild", "{x: this.x, y: this.y, z: this.z}", value)); } else if (name === "SectionMesh") { const target = method("buildAsync"); + bindings(target, ["entries", "fluids"]); parameter(target, 4, "__placementParent"); const input = call(target, "toInput"), worker = call(target, "worker.build"), assembly = call(target, "this.fromPages"); const fallback = []; @@ -63,6 +78,7 @@ export function instrumentPlacementModule(contents, name) { ...fallback.map(node => ({ node, value: measure("synchronous-geometry", "{}", "__placementSpan", `() => ${text(node)}`) })) ], value => wrap("section-build", "{entries: entries.length, fluids: !!fluids}", value, "__placementParent")); const assemble = method("fromPages"); + bindings(assemble, ["pages", "page", "canvas", "context"]); parameter(assemble, 2, "__placementParent"); const loop = find(assemble, node => ts.isForOfStatement(node) && node.expression.getText(source) === "pages", "page loop"); const statements = loop.statement.statements; @@ -80,6 +96,7 @@ export function instrumentPlacementModule(contents, name) { } else if (name === "SectionModels") { for (const [name, kind] of [["prepareState", "state-preparation"], ["prepare", "state-resolution"], ["template", "template-preparation"]]) { const target = method(name); + bindings(target, [name === "template" ? "model" : "blockState"]); const changes = []; if (name === "template") { const load = call(target, "cached").arguments.at(-1); @@ -91,7 +108,9 @@ export function instrumentPlacementModule(contents, name) { : "{block: blockState.key?.toNamespacedString()}", value)); } } else if (name === "UVMapper") { - body(method("createAtlas"), [], value => wrap("model-atlas", "{model: originalModel.key?.toNamespacedString()}", value)); + const target = method("createAtlas"); + bindings(target, ["originalModel"]); + body(target, [], value => wrap("model-atlas", "{model: originalModel.key?.toNamespacedString()}", value)); } else fail("unexpected module"); for (const edit of edits.sort((a, b) => b.start - a.start)) contents = contents.slice(0, edit.start) + edit.value + contents.slice(edit.end); return contents; @@ -102,9 +121,9 @@ export function placementProfilePlugin() { return { name: "placement-profile", setup(build) { - build.onLoad({ filter: /\/dist\/browser\/esm\/(?:world\/(?:Chunk|SectionMesh|SectionModels)|UVMapper)\.mjs$/ }, async ({ path }) => { - const name = path.split("/").at(-1).slice(0, -4); - const contents = instrumentPlacementModule(await readFile(path, "utf8"), name); + build.onLoad({ filter: /\/dist\/browser\/esm\/(?:world\/(?:Chunk|SectionMesh|SectionModels)|UVMapper)\.mjs$/ }, async ({ path: file }) => { + const name = path.basename(file, ".mjs"); + const contents = instrumentPlacementModule(await readFile(file, "utf8"), name); instrumented.add(name); return { contents, loader: "js" }; }); From de238fa775f0c6bddb4a3f64f93fb1c4e341d946 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Haylee=20Sch=C3=A4fer?= Date: Sat, 17 Oct 2026 14:14:03 +0200 Subject: [PATCH 3/5] Batch ready world columns with deferred neighbor refresh --- AGENTS.md | 4 +- ROADMAP.md | 2 +- apps/web/benchmarks/README.md | 2 +- apps/web/benchmarks/world.ts | 8 + packages/minerender/reference/concepts.md | 8 +- .../minerender/src/world/MineRenderWorld.ts | 89 +++- .../minerender/src/world/WorldStreamer.ts | 114 ++++- packages/minerender/test/World.test.ts | 80 ++++ packages/minerender/test/WorldCulling.test.ts | 101 +++++ .../minerender/test/WorldStreamer.test.ts | 395 +++++++++++++++++- 10 files changed, 788 insertions(+), 15 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9a207615..0275796f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -90,13 +90,13 @@ The private Yarn workspace root contains the public library and its consumers. U **Entities** (`src/entity/`): `Entities.getEntity` loads a versioned `entity-models` file. By default it selects `main` plus the dataset `passes` that have no `when` label or whose label is in `options.when` (`getPassList` lists them); `layer`/`layers` select exactly the named layers in order, without passes. `textures` overrides are keyed like `EntityModel.layers`, where a further draw of the same geometry is `#` from 2. `getLayerList` lists available names. Each draw uses the material for its dataset `render` mode (`Materials.createEntityCanvasMaterial`), cached per texture, mode, and tint; modes that vanilla culls get no inward faces. `EntityObject` `tints` maps pass tint labels to colours. `energy_swirl` and `breeze_wind` materials are owned by their object: `Ticker` advances `EntityObject.age` and the texture offset while the object has a parent; removal pauses it and reattachment resumes it. `EntityObject.getLayerGroup` and layer-scoped `getGroupByName`/`getMeshByName` expose each layer’s visibility and poses. Parts form a nested `Object3D` hierarchy with radian poses, inherited texture dimensions, cube growth, and mirrored box UVs. The entity root applies the dataset's `transform` (vanilla's renderer operations for the default state: living entities face +Z with their feet at the origin); models without one get scale `(-1, -1, 1)`. Nested child nodes hold this transform, one per operation, so every operation order is exact and caller position, rotation, and scale apply on top of it. `flip: true` forces that plain flip and `flip: false` keeps raw model space. Without an explicit texture key, use dataset `textureLocation` first, then the resolver (model path, a same-named texture in its directory, then variant data); `getBlock` and `getBlockList` are deprecated aliases. `src/bedrock/` contains type declarations only. -**World** (`src/world/`): `MineRenderWorld` is an early prototype with sparse chunks keyed by signed coordinates. Cubic 16³ `Chunk`s store `ChunkData` palettes with immutable states and recycled unused entries, 4,096 `Uint16` state indices, and per-cell NBT. Render handles and stable `BlockInfo` views remain separate; `.block` returns detached snapshots. Block coordinates are integers; chunk-local coordinates are 0–15. World and owned-chunk setters refresh `cullface` masks after edits, including across chunk boundaries; only opaque full-cube models occlude neighbors. The versioned `blockCulling` dataset lists vanilla's further face rules, applied in both rendering modes: glass, ice, slime and similar blocks hide faces toward the same block, mangrove roots toward the same block above or below, and connected panes or bars toward each other. Leaves keep their faces, as vanilla's cutout leaves do; data without `blockCulling` applies no such rule. `new MineRenderWorld(scene, { sectionMeshing: true })` merges models of any shape, including cutout, translucent, multipart, and animated models, plus fluid surfaces into per-section meshes with separate solid, translucent, and fluid atlas pages (at most 2048×2048 pixels by default; `maxAtlasSize`), with one mesh per page. Page selection uses texture alpha (`hasTranslucency`), so plain glass and panes use the solid/cutout page where vanilla uses the translucent layer. Translucent quads within a section are not depth-sorted. Per-position flower offsets remain unsupported. Only block entities keep individual objects; only opaque full cubes occlude neighbors. Animated atlases occupy their own pages and redraw on frame changes. Fluid surfaces use an 18×18×18 cell neighborhood and are built in the section worker when available. The section worker entry must stay free of three, DOM, and Node imports, so pure geometry code lives in `SectionGeometry.ts` and `FluidQuads.ts`. Merged `BlockInfo.object` values are `undefined`; edit through world/chunk setters. Replacing, removing, or clearing blocks releases their instance slots. `await world.unloadChunkColumn(x, z)` removes every section at absolute chunk-column coordinates and refreshes neighboring faces and fluids. `await world.unloadChunkColumns([{ x, z }, ...])` validates, copies, and deduplicates a group before removal, then refreshes surviving faces, fluids, lighting, AO, and biome colors once. `await world.setBlockVisibleAt(position, visible)` preserves block data and reveals neighboring faces in both rendering modes; replacement or removal resets visibility. Without an executor, parsed columns copy each section's palette into the chunk, resolve each palette entry once, and fill cells in one synchronous pass; `placeMultiBlock` groups blocks per chunk and resolves each distinct state once per group, with weighted variants chosen per cell and yields after about 8 ms of work. Passing a `BatchedExecutor` keeps the previous per-position batches, order at repeated positions, and wait-for-active-batch failure semantics. Bulk placement applies saved properties before model creation and flushes neighbor culling once, including after partial failure; single-block edits update neighbors immediately. Parsed column placement and unloading mark whole sections and neighboring sections' shared boundary cells dirty; structure placement and single edits queue chunk-local indices, and the culling drain recomputes masks in a flat pass per section and rebuilds each changed section once through `SectionMesh.buildAsync`, which runs the pure `buildSectionGeometry` (`SectionGeometry.ts`, typed arrays in and out) in the browser section worker when `Env.provider.createWorker` returns one and synchronously otherwise; a rebuild superseded by a newer rebuild or `clear()` is disposed. `BatchedExecutor.submit` returns a task promise; `stop()` rejects queued and future tasks while active tasks finish. Idle executors hold no timer; caller-provided executors remain caller-owned. Saved biomes supply grass, foliage, dry-foliage, and per-cell water colors in both rendering modes. Saved lighting, corner ambient occlusion, and biome blending are opt-in; light propagation, exact partial-shape ambient occlusion, and LOD remain unsupported. The demos in `apps/web/` exercise `MineRenderWorld`. +**World** (`src/world/`): `MineRenderWorld` is an early prototype with sparse chunks keyed by signed coordinates. Cubic 16³ `Chunk`s store `ChunkData` palettes with immutable states and recycled unused entries, 4,096 `Uint16` state indices, and per-cell NBT. Render handles and stable `BlockInfo` views remain separate; `.block` returns detached snapshots. Block coordinates are integers; chunk-local coordinates are 0–15. World and owned-chunk setters refresh `cullface` masks after edits, including across chunk boundaries; only opaque full-cube models occlude neighbors. The versioned `blockCulling` dataset lists vanilla's further face rules, applied in both rendering modes: glass, ice, slime and similar blocks hide faces toward the same block, mangrove roots toward the same block above or below, and connected panes or bars toward each other. Leaves keep their faces, as vanilla's cutout leaves do; data without `blockCulling` applies no such rule. `new MineRenderWorld(scene, { sectionMeshing: true })` merges models of any shape, including cutout, translucent, multipart, and animated models, plus fluid surfaces into per-section meshes with separate solid, translucent, and fluid atlas pages (at most 2048×2048 pixels by default; `maxAtlasSize`), with one mesh per page. Page selection uses texture alpha (`hasTranslucency`), so plain glass and panes use the solid/cutout page where vanilla uses the translucent layer. Translucent quads within a section are not depth-sorted. Per-position flower offsets remain unsupported. Only block entities keep individual objects; only opaque full cubes occlude neighbors. Animated atlases occupy their own pages and redraw on frame changes. Fluid surfaces use an 18×18×18 cell neighborhood and are built in the section worker when available. The section worker entry must stay free of three, DOM, and Node imports, so pure geometry code lives in `SectionGeometry.ts` and `FluidQuads.ts`. Merged `BlockInfo.object` values are `undefined`; edit through world/chunk setters. Replacing, removing, or clearing blocks releases their instance slots. `await world.unloadChunkColumn(x, z)` removes every section at absolute chunk-column coordinates and refreshes neighboring faces and fluids. `await world.unloadChunkColumns([{ x, z }, ...])` validates, copies, and deduplicates a group before removal, then refreshes surviving faces, fluids, lighting, AO, and biome colors once. `await world.setBlockVisibleAt(position, visible)` preserves block data and reveals neighboring faces in both rendering modes; replacement or removal resets visibility. Without an executor, parsed columns copy each section's palette into the chunk, resolve each palette entry once, and fill cells in one synchronous pass; `placeMultiBlock` groups blocks per chunk and resolves each distinct state once per group, with weighted variants chosen per cell and yields after about 8 ms of work. `await world.placeChunks(columns)` places parsed columns in input order with one final neighbor refresh, including after partial failure. Passing a `BatchedExecutor` keeps the previous per-position batches, order at repeated positions, and wait-for-active-batch failure semantics. Bulk placement applies saved properties before model creation and flushes neighbor culling once, including after partial failure; single-block edits update neighbors immediately. Parsed column placement and unloading mark whole sections and neighboring sections' shared boundary cells dirty; structure placement and single edits queue chunk-local indices, and the culling drain recomputes masks in a flat pass per section and rebuilds each changed section once through `SectionMesh.buildAsync`, which runs the pure `buildSectionGeometry` (`SectionGeometry.ts`, typed arrays in and out) in the browser section worker when `Env.provider.createWorker` returns one and synchronously otherwise; a rebuild superseded by a newer rebuild or `clear()` is disposed. `BatchedExecutor.submit` returns a task promise; `stop()` rejects queued and future tasks while active tasks finish. Idle executors hold no timer; caller-provided executors remain caller-owned. Saved biomes supply grass, foliage, dry-foliage, and per-cell water colors in both rendering modes. Saved lighting, corner ambient occlusion, and biome blending are opt-in; light propagation, exact partial-shape ambient occlusion, and LOD remain unsupported. The demos in `apps/web/` exercise `MineRenderWorld`. `world.raycast(raycaster)` returns the nearest visible rendered block: integer `position`, detached `block` data, and scene-space `point`, `normal`, and `distance`. It resolves merged geometry through per-quad cell indices and individual geometry through owned model/instance references. Hidden blocks and unrelated scene objects are excluded; transparent texture pixels still participate in geometric picking. `world.raycastEntities(raycaster)` returns the nearest visible rendered mob owned by that world when `renderEntities` is enabled: canonical `type`, copied saved fractional `position` in block units, live `object`, and scene-space `point`, `normal`, and `distance`. Hidden or detached entities are excluded. Compare block and entity hit distances to account for occlusion; the world demo selects the nearer hit and outlines the selected mob's visible bounds. -`WorldStreamer` loads nearby columns in distance order, retains a bounded surrounding area, and serializes placement and unloading. Its world must provide `unloadChunkColumns`, as `MineRenderWorld` does, to group obsolete columns and disposal cleanup. Failed bulk unloads retain ownership for retry and invalidate loaded entries before any replacement loads. `WorldChunkSource.getChunk(x, z, signal?)` takes absolute chunk coordinates; the streamer aborts obsolete source reads and active reads during disposal. `AnvilWorldSource` reads regions on demand with count and byte cache limits; its `AnvilRegionReader(x, z, signal?)` callback takes region coordinates. The optional `readExternalChunk(x, z, signal?)` reads `.mcc` payloads at absolute chunk coordinates in the same dimension without caching them. Optional `readEntityRegion` reads a dimension's separate `entities/r.x.z.mca` files, including entity-only columns, with cache limits and cancellation. All readers must forward the optional signal to cancellable I/O to stop that work. Source failures appear in `failedChunks` without blocking other columns; `retryFailedChunks()` retries them. The world demo opens a local Java save folder or selected `.mca` and `.mcc` files, selects a dimension, and streams around the view center. +`WorldStreamer` loads nearby columns in distance order, retains a bounded surrounding area, and serializes placement and unloading. With `placeChunks`, it keeps at most four pending or decoded source columns and places the ready prefix together; a later unfinished read does not delay ready columns. Custom adapters without this method retain one-at-a-time reads and placement. Its world must provide `unloadChunkColumns`, as `MineRenderWorld` does, to group obsolete columns and disposal cleanup. Failed bulk unloads retain ownership for retry and invalidate loaded entries before any replacement loads. `WorldChunkSource.getChunk(x, z, signal?)` takes absolute chunk coordinates; the streamer aborts obsolete source reads and active reads during disposal. `AnvilWorldSource` reads regions on demand with count and byte cache limits; its `AnvilRegionReader(x, z, signal?)` callback takes region coordinates. The optional `readExternalChunk(x, z, signal?)` reads `.mcc` payloads at absolute chunk coordinates in the same dimension without caching them. Optional `readEntityRegion` reads a dimension's separate `entities/r.x.z.mca` files, including entity-only columns, with cache limits and cancellation. All readers must forward the optional signal to cancellable I/O to stop that work. Source failures appear in `failedChunks` without blocking other columns; `retryFailedChunks()` retries them. The world demo opens a local Java save folder or selected `.mca` and `.mcc` files, selects a dimension, and streams around the view center. **Formats**: `StructureParser` loads `.nbt`; `SchematicParser` converts numeric `.schematic` blocks to modern names; `parse(nbt, customMappings, options)` accepts `id:metadata` overrides and falls back to the selected version's legacy mappings; opt-in `lenient` ignores Materials, tries metadata zero, and skips unmapped blocks. `SpongeSchematicParser.parse(nbt)` loads v2/v3 `.schem` files with local palettes, applies Offset to block/entity positions, and normalizes entity payloads to vanilla NBT; size retains the source dimensions. Biomes, metadata, and global block ID palettes are not imported. `LitematicaParser.parse(nbt)` loads v5–7 `.litematic` files into a combined structure and named `regions`, each usable with `placeMultiBlock`. Region `position` and `signedSize` retain the source selection; block/entity positions use the schematic origin. Combined size encloses all region bounds, and later entries in `regions` replace earlier cells, including with air. Metadata and scheduled ticks are not imported. `AnvilParser` reads numeric and paletted Java `.mca` chunks with gzip, zlib, LZ4, or uncompressed payloads: `getChunkList`/`parseChunk` use region-local coordinates (0–31), while `placeChunk` replaces the absolute chunk column. Pre-1.13 numeric chunks reuse the schematic mappings; parser and `AnvilWorldSource` options accept `legacyMappings` overrides and `lenient` metadata-zero fallback. Neighbor- and block-entity-derived state reconstruction is not performed. For external `.mcc` payloads, pass `region: { x, z }` and `readExternalChunk` in the parser options; the reader receives absolute chunk coordinates. Parsers retain DataVersion and entity NBT without migration. Modern (1.18+) section biome palettes are kept as 64 IDs per section; `placeChunk` copies them into the world for `world.getBiomeAt(x, y, z)`, and uses them for block and water colors. Await `chunk.setBiomes(samples)` to refresh affected blocks and tall-plant halves above the section. With `renderEntities`, supported mobs render at their saved position and yaw; sheep wool colour and shearing and fox, axolotl and parrot variants apply; other appearance state is ignored. The option defaults to false. Column replacement, unloading, and `clear()` dispose the owned entity objects. Unsupported entities remain data only. `AnvilParser.parseEntityChunk` reads separate modern `entities/*.mca` records at region-local coordinates; `AnvilWorldSource` joins them with terrain when `readEntityRegion` is supplied. `NBTHelper.fromBuffer` also preserves NBT format, compression, and byte metadata. diff --git a/ROADMAP.md b/ROADMAP.md index 8aec2fac..896bde6a 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -106,7 +106,7 @@ Immediate fixes: ~~fix `getChunkAt` to use `Map.get(key)`~~; ~~remove the 4×4× ~~Pick rendered mobs through `world.raycastEntities` and inspect their type and saved position in the world demo.~~ -~~Resolve each block state once per placement group and yield to the event loop instead of a timer-driven batch queue.~~ ~~Build section geometry in a browser worker.~~ Region decoding still runs on the main thread; wasm or worker-side chunk parsing remains a scale option once placement stops dominating. +~~Resolve each block state once per placement group and yield to the event loop instead of a timer-driven batch queue.~~ ~~Build section geometry in a browser worker.~~ ~~Group ready streamed columns with a four-column read window and refresh shared borders once per placement group.~~ Frame pacing during grouped placement remains unresolved: fixed-route measurements showed longer frame-callback gaps despite fewer section builds. Region decoding still runs on the main thread; wasm or worker-side chunk parsing remains a scale option once placement stops dominating. ~~Render sloped water/lava surfaces with still/flow textures and waterlogged blocks.~~ ~~Cover intrinsic water in kelp, seagrass, and bubble columns.~~ For remaining fluid parity, obtain vanilla block-state fluid definitions, face-occlusion shapes, and solidity/flow-blocking flags; add water overlays against glass/leaves. Keep 1 block = 16 units. diff --git a/apps/web/benchmarks/README.md b/apps/web/benchmarks/README.md index 35078cb2..79dd87ef 100644 --- a/apps/web/benchmarks/README.md +++ b/apps/web/benchmarks/README.md @@ -66,7 +66,7 @@ The report separates these measurements: Each stop records loaded, pending, missing, and failed columns, drawing calls and triangles, Three.js geometry and texture counts, and evidence of worker replies or synchronous fallback. Cleanup is timed separately. Garbage collection runs after frame sampling, so its pause is excluded from streaming and frame measurements. Heap samples include shared assets and benchmark records that remain after world cleanup. The runner exits unsuccessfully for source failures or browser errors and preserves completed reports for inspection. -Unloading records one operation per group of columns the streamer removes together, with its `columnCount` and `columns`. +Placement and unloading record one operation per group of columns the streamer handles together, with its `columnCount` and `columns`. The streamer keeps at most four source reads or decoded results in its placement window and groups the ready prefix without waiting for a later unfinished read. Compare total columns as well as operation counts. Source reads can overlap; their summed wall times are not total streaming time. ## Compare runs diff --git a/apps/web/benchmarks/world.ts b/apps/web/benchmarks/world.ts index 2d3b43e4..8bfdb589 100644 --- a/apps/web/benchmarks/world.ts +++ b/apps/web/benchmarks/world.ts @@ -262,6 +262,14 @@ async function runWorldBenchmark(config: WorldBenchmarkConfig) { try { await world.placeChunk(chunk); } finally { operation = null; } }), + placeChunks: chunks => { + const columns = chunks.map(({ x, z }) => ({ x, z })); + return timed("placement", { columnCount: chunks.length, columns }, async () => { + operation = { id: ++nextOperationId, kind: "placement", columns }; + try { await world.placeChunks(chunks); } + finally { operation = null; } + }); + }, unloadChunkColumns: columns => timed("unload", { columnCount: columns.length, columns: columns.map(({ x, z }) => ({ x, z })) }, async () => { diff --git a/packages/minerender/reference/concepts.md b/packages/minerender/reference/concepts.md index a315d505..7e2f6617 100644 --- a/packages/minerender/reference/concepts.md +++ b/packages/minerender/reference/concepts.md @@ -193,12 +193,18 @@ For direct decoding, call `AnvilParser.parseEntityChunk(bytes, localX, localZ)`. Call `updatePosition` after the camera or view center moves. It accepts scene units; `update(x, z)` accepts absolute chunk coordinates. -The streamer aborts obsolete reads and active reads during disposal. Source callbacks must forward the optional signal to cancellable I/O to stop that work. +With `MineRenderWorld`, the streamer reads ahead by at most four columns, counting both pending reads and decoded results. It places the nearest ready columns together and refreshes their shared borders once. An unfinished later read does not delay the ready prefix. Culling and mesh-build dispatch yield between sections after about 8 ms of work. World mutations remain sequential. Custom world adapters can provide `placeChunks` to use this path; adapters with only `placeChunk` retain one-at-a-time reads and placement. + +The streamer aborts obsolete reads and active reads during disposal. Source callbacks must forward the optional signal to cancellable I/O to stop that work. Reads that ignore cancellation occupy their window slots until they settle. `dispose()` waits for them; reporting a placement failure does not. `readExternalChunk` handles oversized chunks stored beside their region file. Its coordinates are absolute chunk coordinates, and it reads from the same dimension as the region reader. The source caches region files within its configured limits; external payloads are read on demand without retaining them. A missing external file is a chunk error and can be retried. Source errors appear in `failedChunks` while other columns continue loading. Call `await stream.retryFailedChunks()` to retry them. Placement or unloading failures reject the update. +A failed placement group can have changed any of its columns. The streamer cleans the whole group before loading it again, and retains ownership if cleanup fails. An update to the center can cancel queued reads; an active placement finishes before eviction or disposal. + +Call `await world.placeChunks(columns)` to replace already decoded columns together. The method validates and copies all column and section coordinates before mutation, preserves input order, and refreshes queued biomes, faces, fluids, saved lighting, and ambient occlusion after the group. Duplicate columns remain in input order. If a column fails, later columns are skipped and applied changes refresh before the promise rejects. The first placement error takes precedence over a refresh error. The input's section data remains caller-owned; keep it unchanged until placement finishes. `placeChunk(chunk, executor?)` retains its single-column and executor behavior. + Use `await world.unloadChunkColumns([{ x: -1, z: 0 }, { x: 0, z: 0 }])` to remove several columns and refresh surviving faces, fluids, saved lighting, ambient occlusion, and biome colors together. Coordinates are absolute chunk columns. The method validates and copies the whole input before removal, ignores duplicates and missing columns, and refreshes surviving neighbors once. The streamer groups obsolete columns and disposal cleanup through this method. Its world must provide `unloadChunkColumns`; `MineRenderWorld` provides it. A rejected bulk unload keeps its column ownership for retry and invalidates the group's loaded entries, because some columns may already have been removed. The next update finishes that cleanup before loading the latest requested area. diff --git a/packages/minerender/src/world/MineRenderWorld.ts b/packages/minerender/src/world/MineRenderWorld.ts index cae39bdc..19461c7f 100644 --- a/packages/minerender/src/world/MineRenderWorld.ts +++ b/packages/minerender/src/world/MineRenderWorld.ts @@ -376,6 +376,58 @@ export class MineRenderWorld { /** Replaces a chunk column's blocks, biomes, and saved light and, with `renderEntities`, its supported saved mobs. */ public async placeChunk(chunk: AnvilChunk, executor?: BatchedExecutor): Promise { + return this.placeColumn(chunk, executor); + } + + /** + * Replaces parsed columns in input order, then refreshes neighboring geometry once. + * Duplicate coordinates retain their order. A failed column stops placement; queued refreshes finish before rejection. + */ + public async placeChunks(chunks: readonly AnvilChunk[]): Promise { + const columns: AnvilChunk[] = []; + for (const chunk of chunks) { + const { x, z } = chunk ?? {}; + if (![x, z].every(Number.isSafeInteger)) { + throw new RangeError("Chunk column coordinates must be safe integers"); + } + const sections = Array.from(chunk.sections, section => { + const { y } = section ?? {}; + if (!Number.isSafeInteger(y)) throw new RangeError("Chunk section coordinates must be safe integers"); + return { ...section, y }; + }); + columns.push({ ...chunk, x, z, sections }); + } + if (!columns.length) return; + await this.culling; + let failed = false, failure: unknown, sliceStart = performance.now(); + try { + for (const column of columns) { + await this.placeColumn(column, undefined, true); + if (performance.now() - sliceStart > 8) { + await yieldToEventLoop(); + sliceStart = performance.now(); + } + } + } catch (error) { + failed = true; + failure = error; + } finally { + try { + const failures = await this.refreshBiomes(true); + if (failures.size) log.warn("Failed to refresh biome colors", failures.values().next().value); + } catch (error) { + if (!failed) { failed = true; failure = error; } + } + try { + await this.updateCulling([]); + } catch (error) { + if (!failed) { failed = true; failure = error; } + } + } + if (failed) throw failure; + } + + private async placeColumn(chunk: AnvilChunk, executor?: BatchedExecutor, deferCulling = false): Promise { if (![chunk.x, chunk.z].every(Number.isSafeInteger)) { throw new RangeError("Chunk column coordinates must be safe integers"); } @@ -459,9 +511,12 @@ export class MineRenderWorld { await placeEntities?.(); } finally { this.markBiomesChanged(chunk.x, chunk.z, undefined, false); - const failures = await this.refreshBiomes(); - if (failures.size) log.warn("Failed to refresh biome colors", failures.values().next().value); - await this.updateCulling([...changes.values()]); + if (deferCulling) await this.updateCulling([...changes.values()], true); + else { + const failures = await this.refreshBiomes(); + if (failures.size) log.warn("Failed to refresh biome colors", failures.values().next().value); + await this.updateCulling([...changes.values()]); + } } } @@ -639,15 +694,33 @@ export class MineRenderWorld { for (const [index, mask] of masks) await chunk.setCullMaskIndex(index, mask); await chunk.refreshLight(indices === "all" ? undefined : indices); chunk.refreshAmbientOcclusion(this._ambientOcclusion); - if ((this._ambientOcclusion || this.ambientOcclusionPending) && performance.now() - sliceStart > 8) { + if (performance.now() - sliceStart > 8) { await yieldToEventLoop(); sliceStart = performance.now(); } } - const builds = await Promise.allSettled([...batch.keys()].map(async chunk => { - await chunk.rebuildSectionMesh(); - batch.delete(chunk); - })); + const pending: Promise[] = []; + let dispatchFailed = false, dispatchFailure: unknown; + try { + for (const chunk of [...batch.keys()]) { + const build = (async () => { + await chunk.rebuildSectionMesh(); + batch.delete(chunk); + })(); + // Handle rejections while dispatch yields; all started builds settle before their results are checked. + void build.catch(() => undefined); + pending.push(build); + if (performance.now() - sliceStart > 8) { + await yieldToEventLoop(); + sliceStart = performance.now(); + } + } + } catch (error) { + dispatchFailed = true; + dispatchFailure = error; + } + const builds = await Promise.allSettled(pending); + if (dispatchFailed) throw dispatchFailure; const failure = builds.find(result => result.status === "rejected"); if (failure?.status === "rejected") throw failure.reason; } catch (error) { diff --git a/packages/minerender/src/world/WorldStreamer.ts b/packages/minerender/src/world/WorldStreamer.ts index ef9fa1ff..2081e8bc 100644 --- a/packages/minerender/src/world/WorldStreamer.ts +++ b/packages/minerender/src/world/WorldStreamer.ts @@ -14,6 +14,18 @@ interface ChunkPosition { z: number; } +interface ChunkRead { + position: ChunkPosition; + controller: AbortController; + promise: Promise; + ready: boolean; + chunk?: AnvilChunk; + failed: boolean; + error?: unknown; +} + +const MAX_PREFETCHED_COLUMNS = 4; + /** * Loads nearby chunk columns in distance order and unloads distant columns. * Updates coalesce to the latest center and serialize world mutations. Use a dedicated world: @@ -30,12 +42,14 @@ export class WorldStreamer { private desired: ChunkPosition[] = []; private center?: ChunkPosition; private loading?: { position: ChunkPosition; controller: AbortController }; + private readonly reads = new Map(); private running?: Promise; private disposal?: Promise; private disposed = false; constructor( - private readonly world: Pick, "placeChunk" | "unloadChunkColumns">, + private readonly world: Pick, "placeChunk" | "unloadChunkColumns"> + & Partial, "placeChunks">>, private readonly source: WorldChunkSource, options: WorldStreamerOptions = {} ) { @@ -88,6 +102,12 @@ export class WorldStreamer { if (this.loading && !this.within(this.loading.position, this.loadRadius)) { this.loading.controller.abort(); } + for (const [key, read] of this.reads) { + if (!this.within(read.position, this.loadRadius)) { + read.controller.abort(); + if (read.ready) this.reads.delete(key); + } + } } return this.running ??= Promise.resolve().then(() => this.drain()); } @@ -111,8 +131,11 @@ export class WorldStreamer { this.desired = []; this.center = undefined; this.loading?.controller.abort(); + for (const read of this.reads.values()) read.controller.abort(); return this.disposal ??= Promise.resolve().then(async () => { await this.running?.catch(() => undefined); + await Promise.all([...this.reads.values()].map(read => read.promise)); + this.reads.clear(); try { await this.unloadMany([...this.owned.values()]); } finally { @@ -141,6 +164,10 @@ export class WorldStreamer { await this.unloadMany(obsolete); continue; } + if (this.world.placeChunks) { + if (await this.placeReadyChunks()) continue; + return; + } const position = this.desired.find(position => this.needsLoad(position)); if (!position) return; let chunk: AnvilChunk | undefined; @@ -182,10 +209,95 @@ export class WorldStreamer { } } } finally { + for (const [key, read] of this.reads) { + read.controller.abort(); + if (read.ready) this.reads.delete(key); + } this.running = undefined; } } + private fillReadSlots(): void { + while (!this.disposed && this.reads.size < MAX_PREFETCHED_COLUMNS) { + const position = this.desired.find(position => this.needsLoad(position) && !this.reads.has(this.key(position))); + if (!position) return; + const key = this.key(position); + const read: ChunkRead = { + position, controller: new AbortController(), promise: Promise.resolve(), ready: false, failed: false + }; + this.reads.set(key, read); + read.promise = (async () => { + try { + read.chunk = await this.source.getChunk(position.x, position.z, read.controller.signal); + if (read.chunk) this.validateChunk(read.chunk, position); + } catch (error) { + read.failed = true; + read.error = error; + } finally { + read.ready = true; + if (read.controller.signal.aborted || this.disposed || !this.within(position, this.loadRadius)) { + this.reads.delete(key); + } + } + })(); + } + } + + /** Places only the ready prefix, so later source reads cannot delay the nearest ready column. */ + private async placeReadyChunks(): Promise { + this.fillReadSlots(); + const position = this.desired.find(position => this.needsLoad(position)); + if (!position) { + if (!this.reads.size) return false; + await Promise.all([...this.reads.values()].map(read => read.promise)); + return true; + } + const head = this.reads.get(this.key(position)); + if (!head) { + // Aborted reads still occupy slots until their source work settles. + await Promise.race([...this.reads.values()].map(read => read.promise)); + return true; + } + if (!head.ready) { + await head.promise; + return true; + } + const chunks: AnvilChunk[] = [], positions: ChunkPosition[] = []; + for (const desired of this.desired) { + if (!this.needsLoad(desired)) continue; + const key = this.key(desired), read = this.reads.get(key); + if (!read?.ready) break; + this.reads.delete(key); + if (read.controller.signal.aborted || this.disposed || !this.within(desired, this.loadRadius)) continue; + if (read.failed) this.failed.set(key, { ...desired, error: read.error }); + else if (!read.chunk) this.missing.set(key, desired); + else { + chunks.push(read.chunk); + positions.push(desired); + } + } + if (!chunks.length) return true; + for (const position of positions) this.owned.set(this.key(position), position); + try { + await this.world.placeChunks!(chunks); + for (const position of positions) this.loaded.set(this.key(position), position); + } catch (error) { + try { + await this.unloadMany(positions); + } catch (cleanupError) { + const failure = error instanceof Error ? error : new Error(String(error)); + try { + Object.defineProperty(failure, "cause", { value: cleanupError, configurable: true }); + } catch { + throw Object.assign(new Error(failure.message), { cause: cleanupError }); + } + throw failure; + } + throw error; + } + return true; + } + private needsLoad(position: ChunkPosition): boolean { const key = this.key(position); return !this.loaded.has(key) && !this.missing.has(key) && !this.failed.has(key); diff --git a/packages/minerender/test/World.test.ts b/packages/minerender/test/World.test.ts index f4d4a599..324e2e50 100644 --- a/packages/minerender/test/World.test.ts +++ b/packages/minerender/test/World.test.ts @@ -322,6 +322,86 @@ for (const useExecutor of [false, true]) test.serial(`parsed chunk columns repla if (executor) t.is(await executor.submit(() => "reusable"), "reusable"); }); +test.serial("column batches preserve signed replacements and repeated column order", async t => { + const { world, scene } = fixture(t); + const getIndex = BlockEntities.getIndex; + BlockEntities.getIndex = async () => ({}); + t.teardown(() => { BlockEntities.getIndex = getIndex; }); + await world.setBlockAt(-32, 100, 48, block); + const neighbor = await world.setBlockAt(-16, 100, 48, block); + const first = new ChunkData(), last = new ChunkData(), removed = new ChunkData(); + first.set(0, { ...block, properties: { axis: "x" } }); + last.set(4095, { ...block, properties: { axis: "z" }, nbt: { items: [1] } }); + removed.set(0, block); + const columns = Object.freeze([ + Object.freeze({ x: -2, z: 3, sections: [{ y: 0, data: first }] }), + Object.freeze({ x: 1, z: -1, sections: [{ y: 1, data: removed }] }), + Object.freeze({ x: -2, z: 3, sections: [{ y: -4, data: last }] }), + Object.freeze({ x: 1, z: -1, sections: [] }) + ]); + await world.placeChunks(columns); + t.is(world.getBlockAt(-32, 100, 48), undefined); + t.is(world.getBlockAt(-32, 0, 48), undefined); + t.deepEqual(world.getBlockAt(-17, -49, 63)!.block, last.get(4095)); + t.deepEqual(world.getBlockAt(-17, -49, 63)!.object.getPosition().toArray(), [-272, -784, 1008]); + t.is(world.getBlockAt(16, 16, -16), undefined); + t.is(world.getBlockAt(-16, 100, 48), neighbor); + t.is(scene.stats.instanceCount, 2); + t.deepEqual(first.get(0)!.properties, { axis: "x" }); + scene.dirty = false; + await world.placeChunks([]); + t.false(scene.dirty); + t.is(scene.stats.instanceCount, 2); +}); + +test.serial("column batches validate every location before changing existing columns", async t => { + const { world, scene, loads } = fixture(t); + const existing = await world.setBlockAt(0, 0, 0, block); + const data = new ChunkData(); + data.set(0, block); + loads.length = 0; + for (const value of [0.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1]) { + for (const invalid of [ + { x: value, z: 1, sections: [] }, + { x: 1, z: value, sections: [] }, + { x: 1, z: 1, sections: [{ y: value, data }] } + ]) { + await t.throwsAsync(world.placeChunks([{ x: 0, z: 0, sections: [] }, invalid]), { instanceOf: RangeError }); + t.is(world.getBlockAt(0, 0, 0), existing); + t.is(scene.stats.instanceCount, 1); + t.deepEqual(loads, []); + } + } +}); + +test.serial("column batches snapshot column and section locations before awaiting active culling", async t => { + const { world } = fixture(t); + const getIndex = BlockEntities.getIndex; + BlockEntities.getIndex = async () => ({}); + t.teardown(() => { BlockEntities.getIndex = getIndex; }); + const data = new ChunkData(); + data.set(4095, block); + let release!: () => void; + world["culling"] = new Promise(resolve => { + release = () => { world["culling"] = undefined; resolve(); }; + }); + t.teardown(() => release()); + const section = { y: -1, data }; + const column = { x: -2, z: -1, sections: [section] }; + const columns = [column]; + const placing = world.placeChunks(columns); + column.x = 3; + column.z = 4; + section.y = 2; + column.sections.length = 0; + columns.length = 0; + t.is(world.getBlockAt(-17, -1, -1), undefined); + release(); + await placing; + t.deepEqual(world.getBlockAt(-17, -1, -1)!.block, block); + t.is(world.getBlockAt(63, 47, 79), undefined); +}); + test.serial("parsed columns clear unknown states, place remaining cells and reject with the first error", async t => { const { world, scene, loads } = fixture(t); await world.setBlockAt(0, 16, 0, block); diff --git a/packages/minerender/test/WorldCulling.test.ts b/packages/minerender/test/WorldCulling.test.ts index 09080266..74b7c270 100644 --- a/packages/minerender/test/WorldCulling.test.ts +++ b/packages/minerender/test/WorldCulling.test.ts @@ -1406,6 +1406,107 @@ test.serial("chunk column placement culls across sections and empty replacement t.is(scene.stats.instanceCount, 1); }); +test.serial("column batches match sequential meshes across signed fluid, lighting, AO, and biome boundaries", async t => { + const { world, scene, colorAt } = biomeBlendFixture(t, 1, { savedLighting: true, ambientOcclusion: true }); + const left = new ChunkData(), right = new ChunkData(), north = new ChunkData(), diagonal = new ChunkData(); + left.set(15 + 12 * 16, { type: "test:cube" }); + left.set(15 + 8 * 16, { type: "oak_leaves" }); + left.set(255, { type: "water", properties: { level: "4" } }); + right.set(12 * 16, { type: "test:cube" }); + right.set(12 * 16 + 256, { type: "test:cube" }); + right.set(13 * 16 + 256, { type: "test:cube" }); + right.set(15 * 16, { type: "water" }); + north.set(15, { type: "water", properties: { level: "4" } }); + diagonal.set(0, { type: "water" }); + const columns = [ + { x: -1, z: -1, data: left, biome: "test:red", light: 0xaa }, + { x: 0, z: -1, data: right, biome: "test:blue", light: 0x44 }, + { x: -1, z: 0, data: north, biome: "test:red", light: 0x77 }, + { x: 0, z: 0, data: diagonal, biome: "test:blue", light: 0xee } + ].map(({ x, z, data, biome, light }) => ({ x, z, lightValid: true, sections: [ + { y: 0, data, biomes: Array(64).fill(biome), light: { sky: new Uint8Array(2048).fill(light) } } + ] })); + const meshState = () => scene.children.filter(child => child instanceof SectionMesh).flatMap(section => + section.children.flatMap(child => { + const geometry = (child as Mesh).geometry; + const attributes = ["position", "normal", "color", "shade"].map(name => geometry.getAttribute(name)); + return Array.from({ length: geometry.getAttribute("position").count / 4 }, (_, quad) => + JSON.stringify([section.position.toArray(), ...attributes.map(attribute => + Array.from(attribute.array.slice(quad * 4 * attribute.itemSize, (quad + 1) * 4 * attribute.itemSize)))])); + })).sort(); + let isolatedColor: number | undefined; + for (const [index, column] of columns.entries()) { + await world.placeChunk(column); + if (index === 0) isolatedColor = colorAt([-1, 0, -8]); + } + const expected = meshState(), corners = topCornerColors(world, [-1, 0, -4]); + const fluidPositions: TripleArray[] = [[-1, 0, -1], [0, 0, -1], [-1, 0, 0], [0, 0, 0]]; + const fluids = fluidPositions.map(position => assertFluidPositions(t, world, scene, [position])); + const color = colorAt([-1, 0, -8]); + t.not(color, isolatedColor); + t.true(new Set(corners).size > 1); + t.is(cullMaskAt(world, [-1, 0, -4]), 1); + await world.clear(); + const build = SectionMesh.buildAsync; + let builds = 0; + SectionMesh.buildAsync = async function (...args) { builds++; return build.apply(this, args); }; + t.teardown(() => { SectionMesh.buildAsync = build; }); + await world.placeChunks(columns); + t.is(builds, 4); + t.deepEqual(meshState(), expected); + t.deepEqual(topCornerColors(world, [-1, 0, -4]), corners); + t.is(colorAt([-1, 0, -8]), color); + t.is(cullMaskAt(world, [-1, 0, -4]), 1); + for (const [index, position] of fluidPositions.entries()) { + t.deepEqual(assertFluidPositions(t, world, scene, [position]), fluids[index]); + } + t.deepEqual(world.getLightAt(-1, 0, -4), { block: 0, sky: 10 }); + t.is(world.getBiomeAt(-1, 0, -8), "test:red"); + t.is(world.getBiomeAt(0, 0, -8), "test:blue"); +}); + +for (const failure of [new Error("column asset failed"), undefined, null]) { + test.serial(`column batches flush partial writes before rejecting and leave later columns untouched (${String(failure)})`, async t => { + const { world, scene, place } = fixture(t, { sectionMeshing: true }); + const survivor = await place([15, 0, 0]), untouched = await place([48, 0, 0]); + const first = new ChunkData(), broken = new ChunkData(), later = new ChunkData(); + first.set(0, { type: "test:cube" }); + broken.set(0, { type: "test:failure" }); + broken.set(1, { type: "test:cube" }); + later.set(1, { type: "test:cube" }); + const get = BlockStates.get, build = SectionMesh.buildAsync; + BlockStates.get = async key => { if (key.path === "failure") throw failure; return get(key); }; + if (failure === undefined) SectionMesh.buildAsync = async () => { throw new Error("later section flush failed"); }; + t.teardown(() => { BlockStates.get = get; SectionMesh.buildAsync = build; }); + let rejected = false, caught: unknown; + try { + await world.placeChunks([ + { x: 1, z: 0, sections: [{ y: 0, data: first }] }, + { x: 2, z: 0, sections: [{ y: 0, data: broken }] }, + { x: 3, z: 0, sections: [{ y: 0, data: later }] } + ]); + } catch (error) { rejected = true; caught = error; } + t.true(rejected); + t.is(caught, failure); + SectionMesh.buildAsync = build; + if (failure === undefined) await world.unloadChunkColumns([{ x: 99, z: 99 }]); + t.is(world.getBlockAt(15, 0, 0), survivor); + t.is(world.getBlockAt(48, 0, 0), untouched); + t.is(world.getBlockAt(49, 0, 0), undefined); + t.is(world.getBlockAt(32, 0, 0), undefined); + t.is(world.getBlockAt(33, 0, 0)!.block.type, "test:cube"); + t.is(renderedIndexCountAt(world, [15, 0, 0]), 30); + t.is(renderedIndexCountAt(world, [16, 0, 0]), 30); + t.is(renderedIndexCountAt(world, [33, 0, 0]), 36); + t.is(broken.get(0)!.type, "test:failure"); + await world.unloadChunkColumns([{ x: 1, z: 0 }, { x: 2, z: 0 }]); + t.is(world.getBlockAt(16, 0, 0), undefined); + t.is(world.getBlockAt(33, 0, 0), undefined); + t.is(renderedIndexCountAt(world, [15, 0, 0]), 36); + t.is(scene.children.filter(child => child instanceof SectionMesh).length, 2); + }); +} + test.serial("parsed columns match single edits for section geometry and per-block cull masks", async t => { const { world, scene, states, place } = fixture(t, { sectionMeshing: true }); states.set("test:entity", { key: AssetKey.parse("blockstates", "test:entity"), diff --git a/packages/minerender/test/WorldStreamer.test.ts b/packages/minerender/test/WorldStreamer.test.ts index c2394c07..4e0b0d77 100644 --- a/packages/minerender/test/WorldStreamer.test.ts +++ b/packages/minerender/test/WorldStreamer.test.ts @@ -45,18 +45,26 @@ test("streaming retains biome-only sections until eviction and restores them on function fixture(options: { loadRadius?: number; unloadRadius?: number } = { loadRadius: 0, unloadRadius: 0 }, hooks: { get?: (x: number, z: number, signal?: AbortSignal) => Promise; place?: (value: AnvilChunk) => Promise; + placeChunks?: (chunks: readonly AnvilChunk[]) => Promise; unloadColumns?: (columns: readonly { x: number; z: number }[]) => Promise; } = {}) { const events: string[] = [], resident = new Set(); const bulkCalls: { x: number; z: number }[][] = []; + const placements: { x: number; z: number }[][] = []; let active = 0, peak = 0; + let activeReads = 0, peakReads = 0, activeMutations = 0, peakMutations = 0; async function operation(event: string, action: () => Promise): Promise { events.push(event); peak = Math.max(peak, ++active); + const reading = event.startsWith("get:"); + if (reading) peakReads = Math.max(peakReads, ++activeReads); + else peakMutations = Math.max(peakMutations, ++activeMutations); try { return await action(); } finally { active--; + if (reading) activeReads--; + else activeMutations--; } } const source = { @@ -74,7 +82,17 @@ function fixture(options: { loadRadius?: number; unloadRadius?: number } = { loa for (const { x, z } of requested) resident.delete(key(x, z)); }) }; - return { streamer: new WorldStreamer(world, source, options), events, resident, bulkCalls, peak: () => peak, world, source }; + if (hooks.placeChunks) Object.assign(world, { + placeChunks: (chunks: readonly AnvilChunk[]) => operation("place-many", async () => { + placements.push(chunks.map(({ x, z }) => ({ x, z }))); + for (const { x, z } of chunks) resident.add(key(x, z)); + await hooks.placeChunks!(chunks); + }) + }); + return { + streamer: new WorldStreamer(world, source, options), events, resident, bulkCalls, placements, + peak: () => peak, peakReads: () => peakReads, peakMutations: () => peakMutations, world, source + }; } test("streaming validates bounded radii and floors signed scene coordinates into chunk coordinates", async t => { @@ -830,3 +848,378 @@ test("disposing an empty bulk-capable streamer does not call world unload method t.deepEqual(setup.streamer.loadedChunks, []); await t.throwsAsync(() => setup.streamer.update(0, 0)); }); + +test("bulk placement prefetches four columns and groups ready columns in distance order", async t => { + t.timeout(3000); + const reads: { x: number; z: number; gate: ReturnType> }[] = []; + const started = deferred(), placing = deferred(), release = deferred(); + let first = true; + const setup = fixture({ loadRadius: 1, unloadRadius: 1 }, { + get: async (x, z) => { + if (reads.length < 4) { + const gate = deferred(); + reads.push({ x, z, gate }); + if (reads.length === 4) started.resolve(); + return gate.promise; + } + return chunk(x, z); + }, + placeChunks: async () => { + if (first) { + first = false; + placing.resolve(); + await release.promise; + } + } + }); + t.teardown(() => { reads.forEach(({ x, z, gate }) => gate.resolve(chunk(x, z))); release.resolve(); }); + const update = setup.streamer.update(0, 0); + await started.promise; + t.deepEqual(setup.placements, []); + for (const { x, z, gate } of reads) gate.resolve(chunk(x, z)); + await placing.promise; + t.deepEqual(setup.placements[0], reads.map(({ x, z }) => ({ x, z }))); + t.is(setup.placements[0].length, 4); + t.is(setup.events.filter(event => event.startsWith("get:")).length, 4); + release.resolve(); + await update; + const requested = setup.events.filter(event => event.startsWith("get:")).map(event => event.slice(4)); + t.deepEqual(setup.placements.flat().map(({ x, z }) => key(x, z)), requested); + const distances = setup.placements.flat().map(({ x, z }) => x * x + z * z); + t.deepEqual(distances, distances.slice().sort((a, b) => a - b)); + t.true(setup.placements.every(group => group.length > 0 && group.length <= 4)); + t.is(setup.streamer.loadedChunks.length, 9); + t.is(setup.peakReads(), 4); + t.is(setup.peakMutations(), 1); + t.false(setup.events.some(event => event.startsWith("place:"))); + await setup.streamer.dispose(); +}); + +test("a ready nearest column is placed before slow later source reads finish", async t => { + t.timeout(3000); + const placing = deferred(), release = deferred(); + const reads: { x: number; z: number; gate: ReturnType> }[] = []; + let allowReads = false, first = true; + const setup = fixture({ loadRadius: 1, unloadRadius: 1 }, { + get: async (x, z) => { + if (x === 0 && z === 0 || allowReads) return chunk(x, z); + const gate = deferred(); + reads.push({ x, z, gate }); + return gate.promise; + }, + placeChunks: async () => { + if (first) { + first = false; + placing.resolve(); + await release.promise; + } + } + }); + t.teardown(() => { reads.forEach(({ x, z, gate }) => gate.resolve(chunk(x, z))); release.resolve(); }); + const update = setup.streamer.update(0, 0); + await placing.promise; + t.deepEqual(setup.placements, [[{ x: 0, z: 0 }]]); + t.is(reads.length, 3); + t.deepEqual(setup.streamer.loadedChunks, []); + allowReads = true; + for (const { x, z, gate } of reads) gate.resolve(chunk(x, z)); + release.resolve(); + await update; + t.is(setup.streamer.loadedChunks.length, 9); + t.is(setup.peakMutations(), 1); + await setup.streamer.dispose(); +}); + +test("ready later columns occupy the prefetch window without bypassing an unresolved nearer column", async t => { + t.timeout(3000); + const initial = deferred(), firstPlacement = deferred(), fifthRead = deferred(); + const reads: { x: number; z: number; gate: ReturnType> }[] = []; + let allowReads = false; + const setup = fixture({ loadRadius: 1, unloadRadius: 1 }, { + get: async (x, z) => { + if (allowReads) return chunk(x, z); + const gate = deferred(); + reads.push({ x, z, gate }); + if (reads.length === 4) initial.resolve(); + if (reads.length === 5) fifthRead.resolve(); + return gate.promise; + }, + placeChunks: async () => { firstPlacement.resolve(); } + }); + t.teardown(() => reads.forEach(({ x, z, gate }) => gate.resolve(chunk(x, z)))); + const update = setup.streamer.update(0, 0); + await initial.promise; + for (const index of [0, 2, 3]) { + const { x, z, gate } = reads[index]; + gate.resolve(chunk(x, z)); + } + await firstPlacement.promise; + await fifthRead.promise; + await new Promise(resolve => setImmediate(resolve)); + t.is(reads.length, 5); + t.deepEqual(setup.placements, [[{ x: 0, z: 0 }]]); + allowReads = true; + for (const { x, z, gate } of reads) gate.resolve(chunk(x, z)); + await update; + const requested = setup.events.filter(event => event.startsWith("get:")).map(event => event.slice(4)); + t.deepEqual(setup.placements.flat().map(({ x, z }) => key(x, z)), requested); + t.is(setup.streamer.loadedChunks.length, 9); + await setup.streamer.dispose(); +}); + +test("moving the bulk target aborts every obsolete read and discards ignored late results", async t => { + t.timeout(3000); + const started = deferred(); + const reads: { x: number; z: number; gate: ReturnType>; signal?: AbortSignal }[] = []; + let allowReads = false; + const setup = fixture({ loadRadius: 1, unloadRadius: 1 }, { + get: async (x, z, signal) => { + if (allowReads) return chunk(x, z); + const gate = deferred(); + reads.push({ x, z, gate, signal }); + if (reads.length === 4) started.resolve(); + return gate.promise; + }, + placeChunks: async () => {} + }); + t.teardown(() => reads.forEach(({ x, z, gate }) => gate.resolve(chunk(x, z)))); + const first = setup.streamer.update(0, 0); + await started.promise; + const latest = setup.streamer.update(6, 0); + t.true(reads.every(read => read.signal?.aborted)); + t.is(setup.events.length, 4); + allowReads = true; + for (const { x, z, gate } of reads) gate.resolve(chunk(x, z)); + await Promise.all([first, latest]); + t.is(setup.streamer.loadedChunks.length, 9); + t.true(setup.placements.flat().every(({ x, z }) => Math.abs(x - 6) <= 1 && Math.abs(z) <= 1)); + t.deepEqual(setup.streamer.failedChunks, []); + t.is(setup.peakReads(), 4); + t.is(setup.peakMutations(), 1); + await setup.streamer.dispose(); +}); + +test("a center change during grouped placement unloads that whole group before placing the new area", async t => { + t.timeout(3000); + const started = deferred(), gate = deferred(); + let first = true; + const setup = fixture({ loadRadius: 1, unloadRadius: 1 }, { placeChunks: async () => { + if (first) { + first = false; + started.resolve(); + await gate.promise; + } + } }); + t.teardown(() => gate.resolve()); + const initial = setup.streamer.update(0, 0); + await started.promise; + const placed = setup.placements[0]; + const latest = setup.streamer.update(6, 0); + t.deepEqual(setup.bulkCalls, []); + gate.resolve(); + await Promise.all([initial, latest]); + t.deepEqual(setup.bulkCalls[0], placed); + t.true(setup.placements.slice(1).flat().every(({ x, z }) => Math.abs(x - 6) <= 1 && Math.abs(z) <= 1)); + t.is(setup.resident.size, 9); + t.is(setup.streamer.loadedChunks.length, 9); + t.is(setup.peakMutations(), 1); + await setup.streamer.dispose(); +}); + +test("disposal aborts bulk prefetch and waits for active grouped placement before cleanup", async t => { + t.timeout(3000); + const started = deferred(), placement = deferred(); + const signals: AbortSignal[] = []; + const reads: ReturnType>[] = []; + const setup = fixture({ loadRadius: 1, unloadRadius: 1 }, { + get: async (x, z, signal) => { + if (x === 0 && z === 0) return chunk(x, z); + const read = deferred(); + reads.push(read); + signals.push(signal!); + signal!.addEventListener("abort", () => read.reject(signal!.reason), { once: true }); + return read.promise; + }, + placeChunks: async () => { started.resolve(); await placement.promise; } + }); + t.teardown(() => { reads.forEach(read => read.resolve(chunk(0, 0))); placement.resolve(); }); + const update = setup.streamer.update(0, 0); + await started.promise; + const dispose = setup.streamer.dispose(); + t.true(signals.every(signal => signal.aborted)); + t.is(signals.length, 3); + t.deepEqual(setup.bulkCalls, []); + placement.resolve(); + await Promise.all([update, dispose]); + t.deepEqual(setup.bulkCalls, [[{ x: 0, z: 0 }]]); + t.deepEqual(setup.streamer.loadedChunks, []); + t.deepEqual(setup.streamer.failedChunks, []); + t.deepEqual([...setup.resident], []); + t.is(setup.streamer.pendingChunks, 0); + t.is(setup.peakMutations(), 1); +}); + +test("bulk source failures, missing columns, and coordinate mismatches do not prevent later ready columns", async t => { + const error = new Error("unreadable center"); + let fail = true; + const setup = fixture({ loadRadius: 1, unloadRadius: 1 }, { + get: async (x, z) => { + if (fail && x === 0 && z === 0) throw error; + if (x === 0 && z === -1) return undefined; + return chunk(fail && x === -1 && z === 0 ? x + 1 : x, z); + }, + placeChunks: async () => {} + }); + await setup.streamer.update(0, 0); + t.is(setup.streamer.loadedChunks.length, 6); + t.is(setup.streamer.failedChunks.length, 2); + t.is(setup.streamer.failedChunks[0].error, error); + t.is(setup.streamer.pendingChunks, 0); + const count = setup.events.filter(event => event.startsWith("get:")).length; + await setup.streamer.update(0, 0); + t.is(setup.events.filter(event => event.startsWith("get:")).length, count); + fail = false; + await setup.streamer.retryFailedChunks(); + t.is(setup.streamer.loadedChunks.length, 8); + t.deepEqual(setup.streamer.failedChunks, []); + t.is(setup.streamer.pendingChunks, 0); + await setup.streamer.dispose(); +}); + +test("a failed grouped placement cleans every attempted owner before retrying", async t => { + for (const error of [new Error("partial placement"), null, undefined]) { + let fail = true; + const setup = fixture({ loadRadius: 1, unloadRadius: 1 }, { placeChunks: async chunks => { + if (fail) { + fail = false; + for (const { x, z } of chunks.slice(1)) setup.resident.delete(key(x, z)); + throw error; + } + } }); + let rejected = false; + try { await setup.streamer.update(0, 0); } + catch (failure) { rejected = true; t.is(failure, error); } + t.true(rejected); + t.deepEqual(setup.bulkCalls[0], setup.placements[0]); + t.deepEqual(setup.streamer.loadedChunks, []); + t.deepEqual(setup.streamer.failedChunks, []); + t.deepEqual([...setup.resident], []); + await setup.streamer.update(0, 0); + t.is(setup.streamer.loadedChunks.length, 9); + t.is(setup.resident.size, 9); + t.is(setup.peakMutations(), 1); + await setup.streamer.dispose(); + } +}); + +test("failed grouped-placement cleanup retains even unattempted owners for a later retry", async t => { + const placementError = Object.freeze(new Error("partial placement")), cleanupError = new Error("cleanup failed"); + let failPlace = true, failUnload = true; + const setup = fixture({ loadRadius: 1, unloadRadius: 1 }, { + placeChunks: async chunks => { + if (failPlace) { + failPlace = false; + for (const { x, z } of chunks.slice(1)) setup.resident.delete(key(x, z)); + throw placementError; + } + }, + unloadColumns: async () => { + if (failUnload) { + failUnload = false; + throw cleanupError; + } + } + }); + const error = await t.throwsAsync(setup.streamer.update(0, 0), { message: placementError.message }); + t.is((error as Error & { cause?: unknown }).cause, cleanupError); + const attempted = setup.placements[0]; + t.is(attempted.length, 4); + t.deepEqual(setup.bulkCalls[0], attempted); + t.deepEqual(setup.streamer.loadedChunks, []); + t.is(setup.resident.size, 1); + const resumedAt = setup.events.length; + await setup.streamer.update(0, 0); + t.is(setup.events[resumedAt], "unload-many"); + t.deepEqual(setup.bulkCalls[1], attempted); + t.is(setup.streamer.loadedChunks.length, 9); + t.is(setup.resident.size, 9); + t.is(setup.peakMutations(), 1); + await setup.streamer.dispose(); +}); + +test("a placement rejection does not await unrelated reads, but disposal still waits for them", async t => { + t.timeout(3000); + const error = new Error("placement failed"); + const reads: { x: number; z: number; gate: ReturnType>; signal?: AbortSignal }[] = []; + const setup = fixture({ loadRadius: 1, unloadRadius: 1 }, { + get: async (x, z, signal) => { + if (x === 0 && z === 0) return chunk(x, z); + const gate = deferred(); + reads.push({ x, z, gate, signal }); + return gate.promise; + }, + placeChunks: async () => { throw error; } + }); + t.teardown(() => reads.forEach(({ x, z, gate }) => gate.resolve(chunk(x, z)))); + await t.throwsAsync(setup.streamer.update(0, 0), { is: error }); + t.is(reads.length, 3); + t.true(reads.every(read => read.signal?.aborted)); + t.deepEqual(setup.bulkCalls, [[{ x: 0, z: 0 }]]); + t.deepEqual([...setup.resident], []); + let disposed = false; + const dispose = setup.streamer.dispose().then(() => { disposed = true; }); + await new Promise(resolve => setImmediate(resolve)); + t.false(disposed); + for (const { x, z, gate } of reads) gate.resolve(chunk(x, z)); + await dispose; + t.true(disposed); + t.deepEqual(setup.streamer.loadedChunks, []); + t.deepEqual(setup.streamer.failedChunks, []); + t.is(setup.placements.length, 1); + t.is(setup.peakMutations(), 1); +}); + +test("retrying after placement failure keeps aborted reads bounded and discards their late in-area results", async t => { + t.timeout(3000); + const error = new Error("first placement failed"), placing = deferred(), release = deferred(); + const reads: { x: number; z: number; gate: ReturnType>; signal?: AbortSignal }[] = []; + let firstPass = true, firstRetry = true; + const setup = fixture({ loadRadius: 1, unloadRadius: 1 }, { + get: async (x, z, signal) => { + if (!firstPass || x === 0 && z === 0) return chunk(x, z); + const gate = deferred(); + reads.push({ x, z, gate, signal }); + return gate.promise; + }, + placeChunks: async () => { + if (firstPass) throw error; + if (firstRetry) { + firstRetry = false; + placing.resolve(); + await release.promise; + } + } + }); + t.teardown(() => { reads.forEach(({ x, z, gate }) => gate.resolve(chunk(x, z))); release.resolve(); }); + await t.throwsAsync(setup.streamer.update(0, 0), { is: error }); + t.is(reads.length, 3); + t.true(reads.every(read => read.signal?.aborted)); + firstPass = false; + const retry = setup.streamer.update(0, 0); + await placing.promise; + t.deepEqual(setup.placements, [[{ x: 0, z: 0 }], [{ x: 0, z: 0 }]]); + t.is(setup.events.filter(event => event.startsWith("get:")).length, 5); + t.is(setup.peakReads(), 4); + for (const { x, z, gate } of reads) gate.resolve(chunk(x, z)); + release.resolve(); + await retry; + for (const { x, z } of reads) t.is(setup.events.filter(event => event === `get:${x},${z}`).length, 2); + t.is(setup.events.filter(event => event.startsWith("get:")).length, 13); + t.is(setup.placements.slice(1).flat().length, 9); + t.is(setup.streamer.loadedChunks.length, 9); + t.is(setup.resident.size, 9); + t.deepEqual(setup.streamer.failedChunks, []); + t.is(setup.peakReads(), 4); + t.is(setup.peakMutations(), 1); + await setup.streamer.dispose(); +}); From e26a1b281fe7613e88a7eb7c88e9d738c6906347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Haylee=20Sch=C3=A4fer?= Date: Sat, 17 Oct 2026 15:19:45 +0200 Subject: [PATCH 4/5] Yield world work through a macrotask so grouped placement keeps frames scheduler.yield() continuations ran back to back for about 55 ms without a frame between them. Grouped placement chains 2-4 columns of placement, culling, and dispatch without real async waits, so its 8 ms yields produced 66-83 ms frame-callback gaps. MessageChannel lets rendering run between slices. Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 2 +- packages/minerender/src/util/util.ts | 7 ++++--- packages/minerender/test/Util.test.ts | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 packages/minerender/test/Util.test.ts diff --git a/ROADMAP.md b/ROADMAP.md index 896bde6a..6d9729c3 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -106,7 +106,7 @@ Immediate fixes: ~~fix `getChunkAt` to use `Map.get(key)`~~; ~~remove the 4×4× ~~Pick rendered mobs through `world.raycastEntities` and inspect their type and saved position in the world demo.~~ -~~Resolve each block state once per placement group and yield to the event loop instead of a timer-driven batch queue.~~ ~~Build section geometry in a browser worker.~~ ~~Group ready streamed columns with a four-column read window and refresh shared borders once per placement group.~~ Frame pacing during grouped placement remains unresolved: fixed-route measurements showed longer frame-callback gaps despite fewer section builds. Region decoding still runs on the main thread; wasm or worker-side chunk parsing remains a scale option once placement stops dominating. +~~Resolve each block state once per placement group and yield to the event loop instead of a timer-driven batch queue.~~ ~~Build section geometry in a browser worker.~~ ~~Group ready streamed columns with a four-column read window and refresh shared borders once per placement group.~~ Region decoding still runs on the main thread; wasm or worker-side chunk parsing remains a scale option once placement stops dominating. ~~Render sloped water/lava surfaces with still/flow textures and waterlogged blocks.~~ ~~Cover intrinsic water in kelp, seagrass, and bubble columns.~~ For remaining fluid parity, obtain vanilla block-state fluid definitions, face-occlusion shapes, and solidity/flow-blocking flags; add water overlays against glass/leaves. Keep 1 block = 16 units. diff --git a/packages/minerender/src/util/util.ts b/packages/minerender/src/util/util.ts index 013b38b3..b3547652 100644 --- a/packages/minerender/src/util/util.ts +++ b/packages/minerender/src/util/util.ts @@ -40,10 +40,11 @@ export async function sleep(timeout: number): Promise { }); } -/** Yields to the next macrotask without timer clamping when the platform supports it. */ +/** + * Yields to the next macrotask without timer clamping when the platform supports it. + * Not `scheduler.yield()`: Chrome runs its continuations ahead of rendering, so chained yields delay frames. + */ export function yieldToEventLoop(): Promise { - const scheduler = (globalThis as typeof globalThis & { scheduler?: { yield?: () => Promise } }).scheduler; - if (scheduler?.yield) return scheduler.yield(); return new Promise(resolve => { if (typeof MessageChannel !== "undefined") { const channel = new MessageChannel(); diff --git a/packages/minerender/test/Util.test.ts b/packages/minerender/test/Util.test.ts new file mode 100644 index 00000000..5c25ee50 --- /dev/null +++ b/packages/minerender/test/Util.test.ts @@ -0,0 +1,16 @@ +import test from "ava"; +import { yieldToEventLoop } from "../src/util/util"; + +test("yielding uses a macrotask instead of scheduler.yield, so chained yields let frames render", async t => { + const global = globalThis as typeof globalThis & { scheduler?: unknown }; + const previous = Object.getOwnPropertyDescriptor(global, "scheduler"); + let prioritized = 0; + global.scheduler = { yield: () => { prioritized++; return Promise.resolve(); } }; + try { + await yieldToEventLoop(); + t.is(prioritized, 0); + } finally { + if (previous) Object.defineProperty(global, "scheduler", previous); + else delete global.scheduler; + } +}); From 4bf8b10d2d67274162005933587785008846e7e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Haylee=20Sch=C3=A4fer?= Date: Sat, 17 Oct 2026 16:59:25 +0200 Subject: [PATCH 5/5] codex round 1: free a finished read when the nearest column has none, share placement cleanup, name the yield interval --- apps/web/benchmarks/README.md | 2 +- packages/minerender/reference/concepts.md | 8 +- .../minerender/src/world/MineRenderWorld.ts | 42 ++--- .../minerender/src/world/WorldStreamer.ts | 34 ++-- packages/minerender/test/Util.test.ts | 16 -- packages/minerender/test/WorldCulling.test.ts | 2 +- .../minerender/test/WorldStreamer.test.ts | 157 +++++++++++------- 7 files changed, 137 insertions(+), 124 deletions(-) delete mode 100644 packages/minerender/test/Util.test.ts diff --git a/apps/web/benchmarks/README.md b/apps/web/benchmarks/README.md index 79dd87ef..a27f29e0 100644 --- a/apps/web/benchmarks/README.md +++ b/apps/web/benchmarks/README.md @@ -66,7 +66,7 @@ The report separates these measurements: Each stop records loaded, pending, missing, and failed columns, drawing calls and triangles, Three.js geometry and texture counts, and evidence of worker replies or synchronous fallback. Cleanup is timed separately. Garbage collection runs after frame sampling, so its pause is excluded from streaming and frame measurements. Heap samples include shared assets and benchmark records that remain after world cleanup. The runner exits unsuccessfully for source failures or browser errors and preserves completed reports for inspection. -Placement and unloading record one operation per group of columns the streamer handles together, with its `columnCount` and `columns`. The streamer keeps at most four source reads or decoded results in its placement window and groups the ready prefix without waiting for a later unfinished read. Compare total columns as well as operation counts. Source reads can overlap; their summed wall times are not total streaming time. +Placement and unloading record one operation per group of columns the streamer handles together, with its `columnCount` and `columns`. Compare total columns as well as operation counts. Source reads can overlap; their summed wall times are not total streaming time. ## Compare runs diff --git a/packages/minerender/reference/concepts.md b/packages/minerender/reference/concepts.md index 7e2f6617..068189f1 100644 --- a/packages/minerender/reference/concepts.md +++ b/packages/minerender/reference/concepts.md @@ -193,17 +193,17 @@ For direct decoding, call `AnvilParser.parseEntityChunk(bytes, localX, localZ)`. Call `updatePosition` after the camera or view center moves. It accepts scene units; `update(x, z)` accepts absolute chunk coordinates. -With `MineRenderWorld`, the streamer reads ahead by at most four columns, counting both pending reads and decoded results. It places the nearest ready columns together and refreshes their shared borders once. An unfinished later read does not delay the ready prefix. Culling and mesh-build dispatch yield between sections after about 8 ms of work. World mutations remain sequential. Custom world adapters can provide `placeChunks` to use this path; adapters with only `placeChunk` retain one-at-a-time reads and placement. +With `MineRenderWorld`, the streamer reads ahead by at most four columns, counting both pending reads and decoded results. It places the nearest ready columns together and refreshes their shared borders once. An unfinished later read does not delay the ready prefix. Custom world adapters can provide `placeChunks` to use this path; adapters with only `placeChunk` retain one-at-a-time reads and placement. -The streamer aborts obsolete reads and active reads during disposal. Source callbacks must forward the optional signal to cancellable I/O to stop that work. Reads that ignore cancellation occupy their window slots until they settle. `dispose()` waits for them; reporting a placement failure does not. +The streamer aborts obsolete reads and active reads during disposal. Source callbacks must forward the optional signal to cancellable I/O to stop that work. Reads that ignore cancellation occupy their window slots until they settle. `dispose()` waits for them; a placement failure rejects without waiting for them. `readExternalChunk` handles oversized chunks stored beside their region file. Its coordinates are absolute chunk coordinates, and it reads from the same dimension as the region reader. The source caches region files within its configured limits; external payloads are read on demand without retaining them. A missing external file is a chunk error and can be retried. Source errors appear in `failedChunks` while other columns continue loading. Call `await stream.retryFailedChunks()` to retry them. Placement or unloading failures reject the update. -A failed placement group can have changed any of its columns. The streamer cleans the whole group before loading it again, and retains ownership if cleanup fails. An update to the center can cancel queued reads; an active placement finishes before eviction or disposal. +A failed placement group can have changed any of its columns. The streamer cleans the whole group before loading it again, and retains ownership if cleanup fails. An active placement finishes before eviction or disposal. -Call `await world.placeChunks(columns)` to replace already decoded columns together. The method validates and copies all column and section coordinates before mutation, preserves input order, and refreshes queued biomes, faces, fluids, saved lighting, and ambient occlusion after the group. Duplicate columns remain in input order. If a column fails, later columns are skipped and applied changes refresh before the promise rejects. The first placement error takes precedence over a refresh error. The input's section data remains caller-owned; keep it unchanged until placement finishes. `placeChunk(chunk, executor?)` retains its single-column and executor behavior. +Call `await world.placeChunks(columns)` to replace already decoded columns together. The method validates and copies all column and section coordinates before mutation, preserves input order, and refreshes queued biomes, faces, fluids, saved lighting, and ambient occlusion after the group. If a column fails, later columns are skipped and applied changes refresh before the promise rejects. The first placement error takes precedence over a refresh error. The input's section data remains caller-owned; keep it unchanged until placement finishes. Use `await world.unloadChunkColumns([{ x: -1, z: 0 }, { x: 0, z: 0 }])` to remove several columns and refresh surviving faces, fluids, saved lighting, ambient occlusion, and biome colors together. Coordinates are absolute chunk columns. The method validates and copies the whole input before removal, ignores duplicates and missing columns, and refreshes surviving neighbors once. diff --git a/packages/minerender/src/world/MineRenderWorld.ts b/packages/minerender/src/world/MineRenderWorld.ts index 19461c7f..8000a78b 100644 --- a/packages/minerender/src/world/MineRenderWorld.ts +++ b/packages/minerender/src/world/MineRenderWorld.ts @@ -22,6 +22,7 @@ import { loadBlockCulling } from "./_culling/BlockCulling"; import { logger } from "../util/log"; const log = logger("MineRenderWorld"); +const YIELD_INTERVAL_MS = 8; //TODO: maybe make this an Object3D to add children /** @@ -165,7 +166,7 @@ export class MineRenderWorld { failures.set(chunk, error); if (this._chunks.get(`${chunk.x}_${chunk.y}_${chunk.z}`) === chunk) this.pendingBiomes.set(chunk, undefined); } - if (performance.now() - sliceStart > 8) { + if (performance.now() - sliceStart > YIELD_INTERVAL_MS) { await yieldToEventLoop(); sliceStart = performance.now(); } @@ -366,7 +367,7 @@ export class MineRenderWorld { chunk.y * 16 + Math.floor(index / 256), chunk.z * 16 + Math.floor(index / 16) % 16)); for (const pos of positions) changes.set(`${pos.x},${pos.y},${pos.z}`, pos); } - if (performance.now() - sliceStart > 8) { + if (performance.now() - sliceStart > YIELD_INTERVAL_MS) { await yieldToEventLoop(); sliceStart = performance.now(); } @@ -403,7 +404,7 @@ export class MineRenderWorld { try { for (const column of columns) { await this.placeColumn(column, undefined, true); - if (performance.now() - sliceStart > 8) { + if (performance.now() - sliceStart > YIELD_INTERVAL_MS) { await yieldToEventLoop(); sliceStart = performance.now(); } @@ -490,7 +491,7 @@ export class MineRenderWorld { } finally { this.markSectionChanged(target); } - if (performance.now() - sliceStart > 8) { + if (performance.now() - sliceStart > YIELD_INTERVAL_MS) { await yieldToEventLoop(); sliceStart = performance.now(); } @@ -548,7 +549,7 @@ export class MineRenderWorld { for (const { x, z } of columns.values()) { this.entities?.clearColumn(x, z); await this.clearChunkColumn(x, z); - if (performance.now() - sliceStart > 8) { + if (performance.now() - sliceStart > YIELD_INTERVAL_MS) { await yieldToEventLoop(); sliceStart = performance.now(); } @@ -694,33 +695,26 @@ export class MineRenderWorld { for (const [index, mask] of masks) await chunk.setCullMaskIndex(index, mask); await chunk.refreshLight(indices === "all" ? undefined : indices); chunk.refreshAmbientOcclusion(this._ambientOcclusion); - if (performance.now() - sliceStart > 8) { + if (performance.now() - sliceStart > YIELD_INTERVAL_MS) { await yieldToEventLoop(); sliceStart = performance.now(); } } const pending: Promise[] = []; - let dispatchFailed = false, dispatchFailure: unknown; - try { - for (const chunk of [...batch.keys()]) { - const build = (async () => { - await chunk.rebuildSectionMesh(); - batch.delete(chunk); - })(); - // Handle rejections while dispatch yields; all started builds settle before their results are checked. - void build.catch(() => undefined); - pending.push(build); - if (performance.now() - sliceStart > 8) { - await yieldToEventLoop(); - sliceStart = performance.now(); - } + for (const chunk of [...batch.keys()]) { + const build = (async () => { + await chunk.rebuildSectionMesh(); + batch.delete(chunk); + })(); + // Handle rejections while dispatch yields; all started builds settle before their results are checked. + void build.catch(() => undefined); + pending.push(build); + if (performance.now() - sliceStart > YIELD_INTERVAL_MS) { + await yieldToEventLoop(); + sliceStart = performance.now(); } - } catch (error) { - dispatchFailed = true; - dispatchFailure = error; } const builds = await Promise.allSettled(pending); - if (dispatchFailed) throw dispatchFailure; const failure = builds.find(result => result.status === "rejected"); if (failure?.status === "rejected") throw failure.reason; } catch (error) { diff --git a/packages/minerender/src/world/WorldStreamer.ts b/packages/minerender/src/world/WorldStreamer.ts index 2081e8bc..e0126bde 100644 --- a/packages/minerender/src/world/WorldStreamer.ts +++ b/packages/minerender/src/world/WorldStreamer.ts @@ -190,23 +190,7 @@ export class WorldStreamer { continue; } this.owned.set(this.key(position), position); - try { - await this.world.placeChunk(chunk); - this.loaded.set(this.key(position), position); - } catch (error) { - try { - await this.unloadMany([position]); - } catch (cleanupError) { - const failure = error instanceof Error ? error : new Error(String(error)); - try { - Object.defineProperty(failure, "cause", { value: cleanupError, configurable: true }); - } catch { - throw Object.assign(new Error(failure.message), { cause: cleanupError }); - } - throw failure; - } - throw error; - } + await this.placeOwnedColumns([position], () => this.world.placeChunk(chunk!)); } } finally { for (const [key, read] of this.reads) { @@ -254,6 +238,13 @@ export class WorldStreamer { } const head = this.reads.get(this.key(position)); if (!head) { + for (let index = this.desired.length - 1; index >= 0; index--) { + const key = this.key(this.desired[index]); + if (this.reads.get(key)?.ready) { + this.reads.delete(key); + return true; + } + } // Aborted reads still occupy slots until their source work settles. await Promise.race([...this.reads.values()].map(read => read.promise)); return true; @@ -268,7 +259,6 @@ export class WorldStreamer { const key = this.key(desired), read = this.reads.get(key); if (!read?.ready) break; this.reads.delete(key); - if (read.controller.signal.aborted || this.disposed || !this.within(desired, this.loadRadius)) continue; if (read.failed) this.failed.set(key, { ...desired, error: read.error }); else if (!read.chunk) this.missing.set(key, desired); else { @@ -278,8 +268,13 @@ export class WorldStreamer { } if (!chunks.length) return true; for (const position of positions) this.owned.set(this.key(position), position); + await this.placeOwnedColumns(positions, () => this.world.placeChunks!(chunks)); + return true; + } + + private async placeOwnedColumns(positions: readonly ChunkPosition[], place: () => Promise): Promise { try { - await this.world.placeChunks!(chunks); + await place(); for (const position of positions) this.loaded.set(this.key(position), position); } catch (error) { try { @@ -295,7 +290,6 @@ export class WorldStreamer { } throw error; } - return true; } private needsLoad(position: ChunkPosition): boolean { diff --git a/packages/minerender/test/Util.test.ts b/packages/minerender/test/Util.test.ts deleted file mode 100644 index 5c25ee50..00000000 --- a/packages/minerender/test/Util.test.ts +++ /dev/null @@ -1,16 +0,0 @@ -import test from "ava"; -import { yieldToEventLoop } from "../src/util/util"; - -test("yielding uses a macrotask instead of scheduler.yield, so chained yields let frames render", async t => { - const global = globalThis as typeof globalThis & { scheduler?: unknown }; - const previous = Object.getOwnPropertyDescriptor(global, "scheduler"); - let prioritized = 0; - global.scheduler = { yield: () => { prioritized++; return Promise.resolve(); } }; - try { - await yieldToEventLoop(); - t.is(prioritized, 0); - } finally { - if (previous) Object.defineProperty(global, "scheduler", previous); - else delete global.scheduler; - } -}); diff --git a/packages/minerender/test/WorldCulling.test.ts b/packages/minerender/test/WorldCulling.test.ts index 74b7c270..16b43dcb 100644 --- a/packages/minerender/test/WorldCulling.test.ts +++ b/packages/minerender/test/WorldCulling.test.ts @@ -1465,7 +1465,7 @@ test.serial("column batches match sequential meshes across signed fluid, lightin t.is(world.getBiomeAt(0, 0, -8), "test:blue"); }); -for (const failure of [new Error("column asset failed"), undefined, null]) { +for (const failure of [new Error("column asset failed"), undefined]) { test.serial(`column batches flush partial writes before rejecting and leave later columns untouched (${String(failure)})`, async t => { const { world, scene, place } = fixture(t, { sectionMeshing: true }); const survivor = await place([15, 0, 0]), untouched = await place([48, 0, 0]); diff --git a/packages/minerender/test/WorldStreamer.test.ts b/packages/minerender/test/WorldStreamer.test.ts index 4e0b0d77..cabd9afc 100644 --- a/packages/minerender/test/WorldStreamer.test.ts +++ b/packages/minerender/test/WorldStreamer.test.ts @@ -895,75 +895,117 @@ test("bulk placement prefetches four columns and groups ready columns in distanc await setup.streamer.dispose(); }); -test("a ready nearest column is placed before slow later source reads finish", async t => { +test("a ready nearest column is placed before slow later reads, which stay within the prefetch window", async t => { t.timeout(3000); - const placing = deferred(), release = deferred(); + const initial = deferred(), firstPlacement = deferred(), fifthRead = deferred(); const reads: { x: number; z: number; gate: ReturnType> }[] = []; - let allowReads = false, first = true; + let allowReads = false; const setup = fixture({ loadRadius: 1, unloadRadius: 1 }, { get: async (x, z) => { - if (x === 0 && z === 0 || allowReads) return chunk(x, z); + if (allowReads) return chunk(x, z); const gate = deferred(); reads.push({ x, z, gate }); + if (reads.length === 4) initial.resolve(); + if (reads.length === 5) fifthRead.resolve(); return gate.promise; }, - placeChunks: async () => { - if (first) { - first = false; - placing.resolve(); - await release.promise; - } - } + placeChunks: async () => { firstPlacement.resolve(); } }); - t.teardown(() => { reads.forEach(({ x, z, gate }) => gate.resolve(chunk(x, z))); release.resolve(); }); + t.teardown(() => reads.forEach(({ x, z, gate }) => gate.resolve(chunk(x, z)))); const update = setup.streamer.update(0, 0); - await placing.promise; + await initial.promise; + for (const index of [0, 2, 3]) { + const { x, z, gate } = reads[index]; + gate.resolve(chunk(x, z)); + } + await firstPlacement.promise; + t.deepEqual(setup.placements, [[{ x: 0, z: 0 }]]); + t.is(reads.length, 4); + await fifthRead.promise; + await new Promise(resolve => setImmediate(resolve)); + t.is(reads.length, 5); t.deepEqual(setup.placements, [[{ x: 0, z: 0 }]]); - t.is(reads.length, 3); - t.deepEqual(setup.streamer.loadedChunks, []); allowReads = true; for (const { x, z, gate } of reads) gate.resolve(chunk(x, z)); - release.resolve(); await update; + const requested = setup.events.filter(event => event.startsWith("get:")).map(event => event.slice(4)); + t.deepEqual(setup.placements.flat().map(({ x, z }) => key(x, z)), requested); t.is(setup.streamer.loadedChunks.length, 9); - t.is(setup.peakMutations(), 1); await setup.streamer.dispose(); }); -test("ready later columns occupy the prefetch window without bypassing an unresolved nearer column", async t => { +test("a center move starts the nearest read when a finished read occupies the full window", async t => { t.timeout(3000); - const initial = deferred(), firstPlacement = deferred(), fifthRead = deferred(); + const started = deferred(), nearest = deferred(); + const reads: { x: number; z: number; gate: ReturnType>; signal?: AbortSignal }[] = []; + let allowReads = false; + const setup = fixture({ loadRadius: 1, unloadRadius: 1 }, { + get: async (x, z, signal) => { + if (allowReads) return chunk(x, z); + const gate = deferred(); + reads.push({ x, z, gate, signal }); + if (reads.length === 4) started.resolve(); + return gate.promise; + }, + placeChunks: async chunks => { + if (chunks.some(({ x, z }) => x === 0 && z === 1)) nearest.resolve(); + } + }); + t.teardown(() => reads.forEach(({ x, z, gate }) => gate.resolve(chunk(x, z)))); + const first = setup.streamer.update(0, 0); + await started.promise; + const latest = setup.streamer.update(0, 1); + t.true(reads.find(({ x, z }) => x === 0 && z === -1)!.signal?.aborted); + allowReads = true; + reads[0].gate.resolve(chunk(0, 0)); + await nearest.promise; + t.true(setup.placements.flat().some(({ x, z }) => x === 0 && z === 1)); + for (const { x, z, gate } of reads) gate.resolve(chunk(x, z)); + await Promise.all([first, latest]); + t.is(setup.streamer.loadedChunks.length, 9); + t.is(setup.streamer.pendingChunks, 0); + t.is(setup.peakReads(), 4); + await setup.streamer.dispose(); +}); + +test("retrying a failed column loads it when the full window holds a finished read", async t => { + t.timeout(3000); + const started = deferred(), full = deferred(), retried = deferred(); const reads: { x: number; z: number; gate: ReturnType> }[] = []; let allowReads = false; const setup = fixture({ loadRadius: 1, unloadRadius: 1 }, { get: async (x, z) => { - if (allowReads) return chunk(x, z); + if (allowReads) { + if (x === 0 && z === 0) retried.resolve(); + return chunk(x, z); + } const gate = deferred(); reads.push({ x, z, gate }); - if (reads.length === 4) initial.resolve(); - if (reads.length === 5) fifthRead.resolve(); + if (reads.length === 4) started.resolve(); + if (reads.length === 5) full.resolve(); return gate.promise; }, - placeChunks: async () => { firstPlacement.resolve(); } + placeChunks: async () => {} }); t.teardown(() => reads.forEach(({ x, z, gate }) => gate.resolve(chunk(x, z)))); const update = setup.streamer.update(0, 0); - await initial.promise; - for (const index of [0, 2, 3]) { - const { x, z, gate } = reads[index]; - gate.resolve(chunk(x, z)); - } - await firstPlacement.promise; - await fifthRead.promise; + await started.promise; + reads[0].gate.reject(new Error("source failed")); + await full.promise; + reads[3].gate.resolve(chunk(reads[3].x, reads[3].z)); await new Promise(resolve => setImmediate(resolve)); + t.is(setup.streamer.failedChunks.length, 1); t.is(reads.length, 5); - t.deepEqual(setup.placements, [[{ x: 0, z: 0 }]]); allowReads = true; + const retry = setup.streamer.retryFailedChunks(); + reads[1].gate.resolve(chunk(reads[1].x, reads[1].z)); + await retried.promise; for (const { x, z, gate } of reads) gate.resolve(chunk(x, z)); - await update; - const requested = setup.events.filter(event => event.startsWith("get:")).map(event => event.slice(4)); - t.deepEqual(setup.placements.flat().map(({ x, z }) => key(x, z)), requested); + await Promise.all([update, retry]); + t.true(setup.streamer.loadedChunks.some(({ x, z }) => x === 0 && z === 0)); t.is(setup.streamer.loadedChunks.length, 9); + t.deepEqual(setup.streamer.failedChunks, []); + t.is(setup.peakReads(), 4); await setup.streamer.dispose(); }); @@ -1087,33 +1129,32 @@ test("bulk source failures, missing columns, and coordinate mismatches do not pr }); test("a failed grouped placement cleans every attempted owner before retrying", async t => { - for (const error of [new Error("partial placement"), null, undefined]) { - let fail = true; - const setup = fixture({ loadRadius: 1, unloadRadius: 1 }, { placeChunks: async chunks => { - if (fail) { - fail = false; - for (const { x, z } of chunks.slice(1)) setup.resident.delete(key(x, z)); - throw error; - } - } }); - let rejected = false; - try { await setup.streamer.update(0, 0); } - catch (failure) { rejected = true; t.is(failure, error); } - t.true(rejected); - t.deepEqual(setup.bulkCalls[0], setup.placements[0]); - t.deepEqual(setup.streamer.loadedChunks, []); - t.deepEqual(setup.streamer.failedChunks, []); - t.deepEqual([...setup.resident], []); - await setup.streamer.update(0, 0); - t.is(setup.streamer.loadedChunks.length, 9); - t.is(setup.resident.size, 9); - t.is(setup.peakMutations(), 1); - await setup.streamer.dispose(); - } + const error = new Error("partial placement"); + let fail = true; + const setup = fixture({ loadRadius: 1, unloadRadius: 1 }, { placeChunks: async chunks => { + if (fail) { + fail = false; + for (const { x, z } of chunks.slice(1)) setup.resident.delete(key(x, z)); + throw error; + } + } }); + let rejected = false; + try { await setup.streamer.update(0, 0); } + catch (failure) { rejected = true; t.is(failure, error); } + t.true(rejected); + t.deepEqual(setup.bulkCalls[0], setup.placements[0]); + t.deepEqual(setup.streamer.loadedChunks, []); + t.deepEqual(setup.streamer.failedChunks, []); + t.deepEqual([...setup.resident], []); + await setup.streamer.update(0, 0); + t.is(setup.streamer.loadedChunks.length, 9); + t.is(setup.resident.size, 9); + t.is(setup.peakMutations(), 1); + await setup.streamer.dispose(); }); test("failed grouped-placement cleanup retains even unattempted owners for a later retry", async t => { - const placementError = Object.freeze(new Error("partial placement")), cleanupError = new Error("cleanup failed"); + const placementError = new Error("partial placement"), cleanupError = new Error("cleanup failed"); let failPlace = true, failUnload = true; const setup = fixture({ loadRadius: 1, unloadRadius: 1 }, { placeChunks: async chunks => {